From e0e75c46309eec2531b8613370cfbada259b5af3 Mon Sep 17 00:00:00 2001 From: Toby Date: Sat, 18 May 2024 15:01:16 -0700 Subject: [PATCH 1/3] wip: BBR experimental changes --- core/internal/congestion/bbr/bbr_sender.go | 59 +++++++++++++++++++--- 1 file changed, 52 insertions(+), 7 deletions(-) diff --git a/core/internal/congestion/bbr/bbr_sender.go b/core/internal/congestion/bbr/bbr_sender.go index 4afb07832c..eaaf0022a3 100644 --- a/core/internal/congestion/bbr/bbr_sender.go +++ b/core/internal/congestion/bbr/bbr_sender.go @@ -4,6 +4,8 @@ import ( "fmt" "math/rand" "net" + "os" + "strconv" "time" "github.com/apernet/quic-go/congestion" @@ -37,6 +39,8 @@ const ( derivedHighGain = 2.773 // The newly derived CWND gain for STARTUP, 2. derivedHighCWNDGain = 2.0 + + debugEnv = "HYSTERIA_BBR_DEBUG" ) // The cycle of gains used during the PROBE_BW stage. @@ -61,7 +65,7 @@ const ( // Flag. defaultStartupFullLossCount = 8 quicBbr2DefaultLossThreshold = 0.02 - maxBbrBurstPackets = 3 + maxBbrBurstPackets = 10 ) type bbrMode int @@ -237,6 +241,8 @@ type bbrSender struct { maxDatagramSize congestion.ByteCount // Recorded on packet sent. equivalent |unacked_packets_->bytes_in_flight()| bytesInFlight congestion.ByteCount + + debug bool } var _ congestion.CongestionControl = &bbrSender{} @@ -259,6 +265,7 @@ func newBbrSender( initialCongestionWindow, initialMaxCongestionWindow congestion.ByteCount, ) *bbrSender { + debug, _ := strconv.ParseBool(os.Getenv(debugEnv)) b := &bbrSender{ clock: clock, mode: bbrModeStartup, @@ -284,6 +291,7 @@ func newBbrSender( cwndToCalculateMinPacingRate: initialCongestionWindow, maxCongestionWindowWithNetworkParametersAdjusted: initialMaxCongestionWindow, maxDatagramSize: initialMaxDatagramSize, + debug: debug, } b.pacer = common.NewPacer(b.bandwidthForPacer) @@ -332,6 +340,8 @@ func (b *bbrSender) OnPacketSent( } b.sampler.OnPacketSent(sentTime, packetNumber, bytes, bytesInFlight, isRetransmittable) + + b.maybeAppLimited(bytesInFlight) } // CanSend implements the SendAlgorithm interface. @@ -411,8 +421,6 @@ func (b *bbrSender) OnCongestionEventEx(priorInFlight congestion.ByteCount, even // packet in lost_packets. var lastPacketSendState sendTimeState - b.maybeApplimited(priorInFlight) - // Update bytesInFlight b.bytesInFlight = priorInFlight for _, p := range ackedPackets { @@ -539,7 +547,7 @@ func (b *bbrSender) setDrainGain(drainGain float64) { b.drainGain = drainGain } -// What's the current estimated bandwidth in bytes per second. +// Get the current bandwidth estimate. Note that Bandwidth is in bits per second. func (b *bbrSender) bandwidthEstimate() Bandwidth { return b.maxBandwidth.GetBest() } @@ -607,6 +615,10 @@ func (b *bbrSender) enterStartupMode(now time.Time) { // b.maybeTraceStateChange(logging.CongestionStateStartup) b.pacingGain = b.highGain b.congestionWindowGain = b.highCwndGain + + if b.debug { + b.debugPrint("Phase: STARTUP") + } } // Enters the PROBE_BW mode. @@ -625,6 +637,10 @@ func (b *bbrSender) enterProbeBandwidthMode(now time.Time) { b.lastCycleStart = now b.pacingGain = pacingGain[b.cycleCurrentOffset] + + if b.debug { + b.debugPrint("Phase: PROBE_BW") + } } // Updates the round-trip counter if a round-trip has passed. Returns true if @@ -698,15 +714,17 @@ func (b *bbrSender) checkIfFullBandwidthReached(lastPacketSendState *sendTimeSta } } -func (b *bbrSender) maybeApplimited(bytesInFlight congestion.ByteCount) { +func (b *bbrSender) maybeAppLimited(bytesInFlight congestion.ByteCount) { congestionWindow := b.GetCongestionWindow() if bytesInFlight >= congestionWindow { return } availableBytes := congestionWindow - bytesInFlight - drainLimited := b.mode == bbrModeDrain && bytesInFlight > congestionWindow/2 - if !drainLimited || availableBytes > maxBbrBurstPackets*b.maxDatagramSize { + if availableBytes > maxBbrBurstPackets*b.maxDatagramSize { b.sampler.OnAppLimited() + if b.debug { + b.debugPrint("AppLimited, AvailableBytes: %d", availableBytes) + } } } @@ -718,6 +736,10 @@ func (b *bbrSender) maybeExitStartupOrDrain(now time.Time) { // b.maybeTraceStateChange(logging.CongestionStateDrain) b.pacingGain = b.drainGain b.congestionWindowGain = b.highCwndGain + + if b.debug { + b.debugPrint("Phase: DRAIN") + } } if b.mode == bbrModeDrain && b.bytesInFlight <= b.getTargetCongestionWindow(1) { b.enterProbeBandwidthMode(now) @@ -733,6 +755,12 @@ func (b *bbrSender) maybeEnterOrExitProbeRtt(now time.Time, isRoundStart, minRtt // Do not decide on the time to exit PROBE_RTT until the |bytes_in_flight| // is at the target small value. b.exitProbeRttAt = time.Time{} + + if b.debug { + b.debugPrint("BandwidthEstimate: %s, CongestionWindowGain: %.2f, PacingGain: %.2f, PacingRate: %s", + formatSpeed(b.bandwidthEstimate()), b.congestionWindowGain, b.pacingGain, formatSpeed(b.PacingRate())) + b.debugPrint("Phase: PROBE_RTT") + } } if b.mode == bbrModeProbeRtt { @@ -925,6 +953,12 @@ func (b *bbrSender) shouldExitStartupDueToLoss(lastPacketSendState *sendTimeStat return false } +func (b *bbrSender) debugPrint(format string, a ...any) { + fmt.Printf("[BBRSender] [%s] %s\n", + time.Now().Format("15:04:05"), + fmt.Sprintf(format, a...)) +} + func bdpFromRttAndBandwidth(rtt time.Duration, bandwidth Bandwidth) congestion.ByteCount { return congestion.ByteCount(rtt) * congestion.ByteCount(bandwidth) / congestion.ByteCount(BytesPerSecond) / congestion.ByteCount(time.Second) } @@ -942,3 +976,14 @@ func GetInitialPacketSize(addr net.Addr) congestion.ByteCount { return congestion.MinInitialPacketSize } } + +func formatSpeed(bw Bandwidth) string { + bwf := float64(bw) + units := []string{"bps", "Kbps", "Mbps", "Gbps"} + unitIndex := 0 + for bwf > 1024 && unitIndex < len(units)-1 { + bwf /= 1024 + unitIndex++ + } + return fmt.Sprintf("%.2f %s", bwf, units[unitIndex]) +} From cd512ce1c61716e0456a74ec0c78c0a072388085 Mon Sep 17 00:00:00 2001 From: Toby Date: Sun, 19 May 2024 11:46:52 -0700 Subject: [PATCH 2/3] chore: various tweaks --- core/internal/congestion/bbr/bbr_sender.go | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/core/internal/congestion/bbr/bbr_sender.go b/core/internal/congestion/bbr/bbr_sender.go index c4b22aee9c..6828dd6269 100644 --- a/core/internal/congestion/bbr/bbr_sender.go +++ b/core/internal/congestion/bbr/bbr_sender.go @@ -340,8 +340,6 @@ func (b *bbrSender) OnPacketSent( } b.sampler.OnPacketSent(sentTime, packetNumber, bytes, bytesInFlight, isRetransmittable) - - b.maybeAppLimited(bytesInFlight) } // CanSend implements the SendAlgorithm interface. @@ -421,6 +419,8 @@ func (b *bbrSender) OnCongestionEventEx(priorInFlight congestion.ByteCount, even // packet in lost_packets. var lastPacketSendState sendTimeState + b.maybeAppLimited(priorInFlight) + // Update bytesInFlight b.bytesInFlight = priorInFlight for _, p := range ackedPackets { @@ -716,16 +716,12 @@ func (b *bbrSender) checkIfFullBandwidthReached(lastPacketSendState *sendTimeSta func (b *bbrSender) maybeAppLimited(bytesInFlight congestion.ByteCount) { congestionWindow := b.GetCongestionWindow() - if bytesInFlight >= congestionWindow { + // HACK: consider it app-limited if bytes in flight is less than 90% of the congestion window. + if bytesInFlight >= congestionWindow*9/10 || + (b.mode == bbrModeDrain && bytesInFlight > congestionWindow/2) { return } - availableBytes := congestionWindow - bytesInFlight - if availableBytes > maxBbrBurstPackets*b.maxDatagramSize { - b.sampler.OnAppLimited() - if b.debug { - b.debugPrint("AppLimited, AvailableBytes: %d", availableBytes) - } - } + b.sampler.OnAppLimited() } // Transitions from STARTUP to DRAIN and from DRAIN to PROBE_BW if @@ -782,6 +778,9 @@ func (b *bbrSender) maybeEnterOrExitProbeRtt(now time.Time, isRoundStart, minRtt } if now.Sub(b.exitProbeRttAt) >= 0 && b.probeRttRoundPassed { b.minRttTimestamp = now + if b.debug { + b.debugPrint("MinRTT: %s", b.getMinRtt()) + } if !b.isAtFullBandwidth { b.enterStartupMode(now) } else { From 09b08fa494cc75416da191f18b46cd72f05121cf Mon Sep 17 00:00:00 2001 From: Toby Date: Mon, 20 May 2024 14:19:04 -0700 Subject: [PATCH 3/3] fix: try to fix maybeAppLimited 2 --- core/internal/congestion/bbr/bbr_sender.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/core/internal/congestion/bbr/bbr_sender.go b/core/internal/congestion/bbr/bbr_sender.go index 6828dd6269..62868ec718 100644 --- a/core/internal/congestion/bbr/bbr_sender.go +++ b/core/internal/congestion/bbr/bbr_sender.go @@ -715,13 +715,9 @@ func (b *bbrSender) checkIfFullBandwidthReached(lastPacketSendState *sendTimeSta } func (b *bbrSender) maybeAppLimited(bytesInFlight congestion.ByteCount) { - congestionWindow := b.GetCongestionWindow() - // HACK: consider it app-limited if bytes in flight is less than 90% of the congestion window. - if bytesInFlight >= congestionWindow*9/10 || - (b.mode == bbrModeDrain && bytesInFlight > congestionWindow/2) { - return + if bytesInFlight < b.getTargetCongestionWindow(1) { + b.sampler.OnAppLimited() } - b.sampler.OnAppLimited() } // Transitions from STARTUP to DRAIN and from DRAIN to PROBE_BW if