FreeBSD kernel IPv4 code
bbr.c
Go to the documentation of this file.
1/*-
2 * Copyright (c) 2016-2020 Netflix, Inc.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 *
25 */
33#include <sys/cdefs.h>
34__FBSDID("$FreeBSD$");
35
36#include "opt_inet.h"
37#include "opt_inet6.h"
38#include "opt_ipsec.h"
39#include "opt_tcpdebug.h"
40#include "opt_ratelimit.h"
41#include <sys/param.h>
42#include <sys/arb.h>
43#include <sys/module.h>
44#include <sys/kernel.h>
45#include <sys/libkern.h>
46#ifdef TCP_HHOOK
47#include <sys/hhook.h>
48#endif
49#include <sys/malloc.h>
50#include <sys/mbuf.h>
51#include <sys/proc.h>
52#include <sys/socket.h>
53#include <sys/socketvar.h>
54#include <sys/sysctl.h>
55#include <sys/systm.h>
56#ifdef STATS
57#include <sys/qmath.h>
58#include <sys/tree.h>
59#include <sys/stats.h> /* Must come after qmath.h and tree.h */
60#endif
61#include <sys/refcount.h>
62#include <sys/queue.h>
63#include <sys/eventhandler.h>
64#include <sys/smp.h>
65#include <sys/kthread.h>
66#include <sys/lock.h>
67#include <sys/mutex.h>
68#include <sys/tim_filter.h>
69#include <sys/time.h>
70#include <sys/protosw.h>
71#include <vm/uma.h>
72#include <sys/kern_prefetch.h>
73
74#include <net/route.h>
75#include <net/route/nhop.h>
76#include <net/vnet.h>
77
78#define TCPSTATES /* for logging */
79
80#include <netinet/in.h>
81#include <netinet/in_kdtrace.h>
82#include <netinet/in_pcb.h>
83#include <netinet/ip.h>
84#include <netinet/ip_icmp.h> /* required for icmp_var.h */
85#include <netinet/icmp_var.h> /* for ICMP_BANDLIM */
86#include <netinet/ip_var.h>
87#include <netinet/ip6.h>
88#include <netinet6/in6_pcb.h>
89#include <netinet6/ip6_var.h>
90#define TCPOUTFLAGS
91#include <netinet/tcp.h>
92#include <netinet/tcp_fsm.h>
93#include <netinet/tcp_seq.h>
94#include <netinet/tcp_timer.h>
95#include <netinet/tcp_var.h>
96#include <netinet/tcpip.h>
97#include <netinet/tcp_hpts.h>
98#include <netinet/cc/cc.h>
99#include <netinet/tcp_log_buf.h>
100#include <netinet/tcp_ratelimit.h>
101#include <netinet/tcp_lro.h>
102#ifdef TCPDEBUG
103#include <netinet/tcp_debug.h>
104#endif /* TCPDEBUG */
105#ifdef TCP_OFFLOAD
106#include <netinet/tcp_offload.h>
107#endif
108#ifdef INET6
109#include <netinet6/tcp6_var.h>
110#endif
111#include <netinet/tcp_fastopen.h>
112
113#include <netipsec/ipsec_support.h>
114#include <net/if.h>
115#include <net/if_var.h>
116#include <net/ethernet.h>
117
118#if defined(IPSEC) || defined(IPSEC_SUPPORT)
119#include <netipsec/ipsec.h>
120#include <netipsec/ipsec6.h>
121#endif /* IPSEC */
122
123#include <netinet/udp.h>
124#include <netinet/udp_var.h>
125#include <machine/in_cksum.h>
126
127#ifdef MAC
128#include <security/mac/mac_framework.h>
129#endif
130
131#include "sack_filter.h"
132#include "tcp_bbr.h"
133#include "rack_bbr_common.h"
134uma_zone_t bbr_zone;
135uma_zone_t bbr_pcb_zone;
136
137struct sysctl_ctx_list bbr_sysctl_ctx;
138struct sysctl_oid *bbr_sysctl_root;
139
140#define TCPT_RANGESET_NOSLOP(tv, value, tvmin, tvmax) do { \
141 (tv) = (value); \
142 if ((u_long)(tv) < (u_long)(tvmin)) \
143 (tv) = (tvmin); \
144 if ((u_long)(tv) > (u_long)(tvmax)) \
145 (tv) = (tvmax); \
146} while(0)
147
148/*#define BBR_INVARIANT 1*/
149
150/*
151 * initial window
152 */
153static uint32_t bbr_def_init_win = 10;
154static int32_t bbr_persist_min = 250000; /* 250ms */
155static int32_t bbr_persist_max = 1000000; /* 1 Second */
156static int32_t bbr_cwnd_may_shrink = 0;
157static int32_t bbr_cwndtarget_rtt_touse = BBR_RTT_PROP;
158static int32_t bbr_num_pktepo_for_del_limit = BBR_NUM_RTTS_FOR_DEL_LIMIT;
159static int32_t bbr_hardware_pacing_limit = 8000;
160static int32_t bbr_quanta = 3; /* How much extra quanta do we get? */
161static int32_t bbr_no_retran = 0;
162
163static int32_t bbr_error_base_paceout = 10000; /* usec to pace */
164static int32_t bbr_max_net_error_cnt = 10;
165/* Should the following be dynamic too -- loss wise */
166static int32_t bbr_rtt_gain_thresh = 0;
167/* Measurement controls */
168static int32_t bbr_use_google_algo = 1;
169static int32_t bbr_ts_limiting = 1;
170static int32_t bbr_ts_can_raise = 0;
171static int32_t bbr_do_red = 600;
172static int32_t bbr_red_scale = 20000;
173static int32_t bbr_red_mul = 1;
174static int32_t bbr_red_div = 2;
175static int32_t bbr_red_growth_restrict = 1;
176static int32_t bbr_target_is_bbunit = 0;
177static int32_t bbr_drop_limit = 0;
178/*
179 * How much gain do we need to see to
180 * stay in startup?
181 */
182static int32_t bbr_marks_rxt_sack_passed = 0;
183static int32_t bbr_start_exit = 25;
184static int32_t bbr_low_start_exit = 25; /* When we are in reduced gain */
185static int32_t bbr_startup_loss_thresh = 2000; /* 20.00% loss */
186static int32_t bbr_hptsi_max_mul = 1; /* These two mul/div assure a min pacing */
187static int32_t bbr_hptsi_max_div = 2; /* time, 0 means turned off. We need this
188 * if we go back ever to where the pacer
189 * has priority over timers.
190 */
191static int32_t bbr_policer_call_from_rack_to = 0;
192static int32_t bbr_policer_detection_enabled = 1;
193static int32_t bbr_min_measurements_req = 1; /* We need at least 2
194 * measurements before we are
195 * "good" note that 2 == 1.
196 * This is because we use a >
197 * comparison. This means if
198 * min_measure was 0, it takes
199 * num-measures > min(0) and
200 * you get 1 measurement and
201 * you are good. Set to 1, you
202 * have to have two
203 * measurements (this is done
204 * to prevent it from being ok
205 * to have no measurements). */
206static int32_t bbr_no_pacing_until = 4;
207
208static int32_t bbr_min_usec_delta = 20000; /* 20,000 usecs */
209static int32_t bbr_min_peer_delta = 20; /* 20 units */
210static int32_t bbr_delta_percent = 150; /* 15.0 % */
211
212static int32_t bbr_target_cwnd_mult_limit = 8;
213/*
214 * bbr_cwnd_min_val is the number of
215 * segments we hold to in the RTT probe
216 * state typically 4.
217 */
218static int32_t bbr_cwnd_min_val = BBR_PROBERTT_NUM_MSS;
219
220static int32_t bbr_cwnd_min_val_hs = BBR_HIGHSPEED_NUM_MSS;
221
222static int32_t bbr_gain_to_target = 1;
223static int32_t bbr_gain_gets_extra_too = 1;
224/*
225 * bbr_high_gain is the 2/ln(2) value we need
226 * to double the sending rate in startup. This
227 * is used for both cwnd and hptsi gain's.
228 */
229static int32_t bbr_high_gain = BBR_UNIT * 2885 / 1000 + 1;
230static int32_t bbr_startup_lower = BBR_UNIT * 1500 / 1000 + 1;
231static int32_t bbr_use_lower_gain_in_startup = 1;
232
233/* thresholds for reduction on drain in sub-states/drain */
234static int32_t bbr_drain_rtt = BBR_SRTT;
235static int32_t bbr_drain_floor = 88;
236static int32_t google_allow_early_out = 1;
237static int32_t google_consider_lost = 1;
238static int32_t bbr_drain_drop_mul = 4;
239static int32_t bbr_drain_drop_div = 5;
240static int32_t bbr_rand_ot = 50;
241static int32_t bbr_can_force_probertt = 0;
242static int32_t bbr_can_adjust_probertt = 1;
243static int32_t bbr_probertt_sets_rtt = 0;
244static int32_t bbr_can_use_ts_for_rtt = 1;
245static int32_t bbr_is_ratio = 0;
246static int32_t bbr_sub_drain_app_limit = 1;
247static int32_t bbr_prtt_slam_cwnd = 1;
248static int32_t bbr_sub_drain_slam_cwnd = 1;
249static int32_t bbr_slam_cwnd_in_main_drain = 1;
250static int32_t bbr_filter_len_sec = 6; /* How long does the rttProp filter
251 * hold */
252static uint32_t bbr_rtt_probe_limit = (USECS_IN_SECOND * 4);
253/*
254 * bbr_drain_gain is the reverse of the high_gain
255 * designed to drain back out the standing queue
256 * that is formed in startup by causing a larger
257 * hptsi gain and thus drainging the packets
258 * in flight.
259 */
260static int32_t bbr_drain_gain = BBR_UNIT * 1000 / 2885;
261static int32_t bbr_rttprobe_gain = 192;
262
263/*
264 * The cwnd_gain is the default cwnd gain applied when
265 * calculating a target cwnd. Note that the cwnd is
266 * a secondary factor in the way BBR works (see the
267 * paper and think about it, it will take some time).
268 * Basically the hptsi_gain spreads the packets out
269 * so you never get more than BDP to the peer even
270 * if the cwnd is high. In our implemenation that
271 * means in non-recovery/retransmission scenarios
272 * cwnd will never be reached by the flight-size.
273 */
274static int32_t bbr_cwnd_gain = BBR_UNIT * 2;
275static int32_t bbr_tlp_type_to_use = BBR_SRTT;
276static int32_t bbr_delack_time = 100000; /* 100ms in useconds */
277static int32_t bbr_sack_not_required = 0; /* set to one to allow non-sack to use bbr */
278static int32_t bbr_initial_bw_bps = 62500; /* 500kbps in bytes ps */
279static int32_t bbr_ignore_data_after_close = 1;
280static int16_t bbr_hptsi_gain[] = {
281 (BBR_UNIT *5 / 4),
282 (BBR_UNIT * 3 / 4),
283 BBR_UNIT,
284 BBR_UNIT,
285 BBR_UNIT,
286 BBR_UNIT,
287 BBR_UNIT,
288 BBR_UNIT
289};
290int32_t bbr_use_rack_resend_cheat = 1;
291int32_t bbr_sends_full_iwnd = 1;
292
293#define BBR_HPTSI_GAIN_MAX 8
294/*
295 * The BBR module incorporates a number of
296 * TCP ideas that have been put out into the IETF
297 * over the last few years:
298 * - Yuchung Cheng's RACK TCP (for which its named) that
299 * will stop us using the number of dup acks and instead
300 * use time as the gage of when we retransmit.
301 * - Reorder Detection of RFC4737 and the Tail-Loss probe draft
302 * of Dukkipati et.al.
303 * - Van Jacobson's et.al BBR.
304 *
305 * RACK depends on SACK, so if an endpoint arrives that
306 * cannot do SACK the state machine below will shuttle the
307 * connection back to using the "default" TCP stack that is
308 * in FreeBSD.
309 *
310 * To implement BBR and RACK the original TCP stack was first decomposed
311 * into a functional state machine with individual states
312 * for each of the possible TCP connection states. The do_segment
313 * functions role in life is to mandate the connection supports SACK
314 * initially and then assure that the RACK state matches the conenction
315 * state before calling the states do_segment function. Data processing
316 * of inbound segments also now happens in the hpts_do_segment in general
317 * with only one exception. This is so we can keep the connection on
318 * a single CPU.
319 *
320 * Each state is simplified due to the fact that the original do_segment
321 * has been decomposed and we *know* what state we are in (no
322 * switches on the state) and all tests for SACK are gone. This
323 * greatly simplifies what each state does.
324 *
325 * TCP output is also over-written with a new version since it
326 * must maintain the new rack scoreboard and has had hptsi
327 * integrated as a requirment. Still todo is to eliminate the
328 * use of the callout_() system and use the hpts for all
329 * timers as well.
330 */
331static uint32_t bbr_rtt_probe_time = 200000; /* 200ms in micro seconds */
332static uint32_t bbr_rtt_probe_cwndtarg = 4; /* How many mss's outstanding */
333static const int32_t bbr_min_req_free = 2; /* The min we must have on the
334 * free list */
335static int32_t bbr_tlp_thresh = 1;
336static int32_t bbr_reorder_thresh = 2;
337static int32_t bbr_reorder_fade = 60000000; /* 0 - never fade, def
338 * 60,000,000 - 60 seconds */
339static int32_t bbr_pkt_delay = 1000;
340static int32_t bbr_min_to = 1000; /* Number of usec's minimum timeout */
341static int32_t bbr_incr_timers = 1;
342
343static int32_t bbr_tlp_min = 10000; /* 10ms in usecs */
344static int32_t bbr_delayed_ack_time = 200000; /* 200ms in usecs */
345static int32_t bbr_exit_startup_at_loss = 1;
346
347/*
348 * bbr_lt_bw_ratio is 1/8th
349 * bbr_lt_bw_diff is < 4 Kbit/sec
350 */
351static uint64_t bbr_lt_bw_diff = 4000 / 8; /* In bytes per second */
352static uint64_t bbr_lt_bw_ratio = 8; /* For 1/8th */
353static uint32_t bbr_lt_bw_max_rtts = 48; /* How many rtt's do we use
354 * the lt_bw for */
355static uint32_t bbr_lt_intvl_min_rtts = 4; /* Min num of RTT's to measure
356 * lt_bw */
357static int32_t bbr_lt_intvl_fp = 0; /* False positive epoch diff */
358static int32_t bbr_lt_loss_thresh = 196; /* Lost vs delivered % */
359static int32_t bbr_lt_fd_thresh = 100; /* false detection % */
360
361static int32_t bbr_verbose_logging = 0;
362/*
363 * Currently regular tcp has a rto_min of 30ms
364 * the backoff goes 12 times so that ends up
365 * being a total of 122.850 seconds before a
366 * connection is killed.
367 */
368static int32_t bbr_rto_min_ms = 30; /* 30ms same as main freebsd */
369static int32_t bbr_rto_max_sec = 4; /* 4 seconds */
370
371/****************************************************/
372/* DEFAULT TSO SIZING (cpu performance impacting) */
373/****************************************************/
374/* What amount is our formula using to get TSO size */
375static int32_t bbr_hptsi_per_second = 1000;
376
377/*
378 * For hptsi under bbr_cross_over connections what is delay
379 * target 7ms (in usec) combined with a seg_max of 2
380 * gets us close to identical google behavior in
381 * TSO size selection (possibly more 1MSS sends).
382 */
383static int32_t bbr_hptsi_segments_delay_tar = 7000;
384
385/* Does pacing delay include overhead's in its time calculations? */
386static int32_t bbr_include_enet_oh = 0;
387static int32_t bbr_include_ip_oh = 1;
388static int32_t bbr_include_tcp_oh = 1;
389static int32_t bbr_google_discount = 10;
390
391/* Do we use (nf mode) pkt-epoch to drive us or rttProp? */
392static int32_t bbr_state_is_pkt_epoch = 0;
393static int32_t bbr_state_drain_2_tar = 1;
394/* What is the max the 0 - bbr_cross_over MBPS TSO target
395 * can reach using our delay target. Note that this
396 * value becomes the floor for the cross over
397 * algorithm.
398 */
399static int32_t bbr_hptsi_segments_max = 2;
400static int32_t bbr_hptsi_segments_floor = 1;
401static int32_t bbr_hptsi_utter_max = 0;
402
403/* What is the min the 0 - bbr_cross-over MBPS TSO target can be */
404static int32_t bbr_hptsi_bytes_min = 1460;
405static int32_t bbr_all_get_min = 0;
406
407/* Cross over point from algo-a to algo-b */
408static uint32_t bbr_cross_over = TWENTY_THREE_MBPS;
409
410/* Do we deal with our restart state? */
411static int32_t bbr_uses_idle_restart = 0;
412static int32_t bbr_idle_restart_threshold = 100000; /* 100ms in useconds */
413
414/* Do we allow hardware pacing? */
415static int32_t bbr_allow_hdwr_pacing = 0;
416static int32_t bbr_hdwr_pace_adjust = 2; /* multipler when we calc the tso size */
417static int32_t bbr_hdwr_pace_floor = 1;
418static int32_t bbr_hdwr_pacing_delay_cnt = 10;
419
420/****************************************************/
421static int32_t bbr_resends_use_tso = 0;
422static int32_t bbr_tlp_max_resend = 2;
423static int32_t bbr_sack_block_limit = 128;
424
425#define BBR_MAX_STAT 19
426counter_u64_t bbr_state_time[BBR_MAX_STAT];
427counter_u64_t bbr_state_lost[BBR_MAX_STAT];
428counter_u64_t bbr_state_resend[BBR_MAX_STAT];
429counter_u64_t bbr_stat_arry[BBR_STAT_SIZE];
430counter_u64_t bbr_opts_arry[BBR_OPTS_SIZE];
431counter_u64_t bbr_out_size[TCP_MSS_ACCT_SIZE];
432counter_u64_t bbr_flows_whdwr_pacing;
433counter_u64_t bbr_flows_nohdwr_pacing;
434
435counter_u64_t bbr_nohdwr_pacing_enobuf;
436counter_u64_t bbr_hdwr_pacing_enobuf;
437
438static inline uint64_t bbr_get_bw(struct tcp_bbr *bbr);
439
440/*
441 * Static defintions we need for forward declarations.
442 */
443static uint32_t
444bbr_get_pacing_length(struct tcp_bbr *bbr, uint16_t gain,
445 uint32_t useconds_time, uint64_t bw);
446static uint32_t
447bbr_get_a_state_target(struct tcp_bbr *bbr, uint32_t gain);
448static void
449 bbr_set_state(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t win);
450static void
451bbr_set_probebw_gains(struct tcp_bbr *bbr, uint32_t cts, uint32_t losses);
452static void
453bbr_substate_change(struct tcp_bbr *bbr, uint32_t cts, int line,
454 int dolog);
455static uint32_t
456bbr_get_target_cwnd(struct tcp_bbr *bbr, uint64_t bw, uint32_t gain);
457static void
458bbr_state_change(struct tcp_bbr *bbr, uint32_t cts, int32_t epoch,
459 int32_t pkt_epoch, uint32_t losses);
460static uint32_t
461bbr_calc_thresh_rack(struct tcp_bbr *bbr, uint32_t srtt, uint32_t cts, struct bbr_sendmap *rsm);
462static uint32_t bbr_initial_cwnd(struct tcp_bbr *bbr, struct tcpcb *tp);
463static uint32_t
464bbr_calc_thresh_tlp(struct tcpcb *tp, struct tcp_bbr *bbr,
465 struct bbr_sendmap *rsm, uint32_t srtt,
466 uint32_t cts);
467static void
468bbr_exit_persist(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts,
469 int32_t line);
470static void
471 bbr_set_state_target(struct tcp_bbr *bbr, int line);
472static void
473 bbr_enter_probe_rtt(struct tcp_bbr *bbr, uint32_t cts, int32_t line);
474
475static void
476 bbr_log_progress_event(struct tcp_bbr *bbr, struct tcpcb *tp, uint32_t tick, int event, int line);
477
478static void
479 tcp_bbr_tso_size_check(struct tcp_bbr *bbr, uint32_t cts);
480
481static void
482 bbr_setup_red_bw(struct tcp_bbr *bbr, uint32_t cts);
483
484static void
485 bbr_log_rtt_shrinks(struct tcp_bbr *bbr, uint32_t cts, uint32_t applied, uint32_t rtt,
486 uint32_t line, uint8_t is_start, uint16_t set);
487
488static struct bbr_sendmap *
489 bbr_find_lowest_rsm(struct tcp_bbr *bbr);
490static __inline uint32_t
491bbr_get_rtt(struct tcp_bbr *bbr, int32_t rtt_type);
492static void
493 bbr_log_to_start(struct tcp_bbr *bbr, uint32_t cts, uint32_t to, int32_t slot, uint8_t which);
494
495static void
496bbr_log_timer_var(struct tcp_bbr *bbr, int mode, uint32_t cts, uint32_t time_since_sent, uint32_t srtt,
497 uint32_t thresh, uint32_t to);
498static void
499 bbr_log_hpts_diag(struct tcp_bbr *bbr, uint32_t cts, struct hpts_diag *diag);
500
501static void
502bbr_log_type_bbrsnd(struct tcp_bbr *bbr, uint32_t len, uint32_t slot,
503 uint32_t del_by, uint32_t cts, uint32_t sloton, uint32_t prev_delay);
504
505static void
506bbr_enter_persist(struct tcpcb *tp, struct tcp_bbr *bbr,
507 uint32_t cts, int32_t line);
508static void
509 bbr_stop_all_timers(struct tcpcb *tp);
510static void
511 bbr_exit_probe_rtt(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts);
512static void
513 bbr_check_probe_rtt_limits(struct tcp_bbr *bbr, uint32_t cts);
514static void
515 bbr_timer_cancel(struct tcp_bbr *bbr, int32_t line, uint32_t cts);
516
517static void
518bbr_log_pacing_delay_calc(struct tcp_bbr *bbr, uint16_t gain, uint32_t len,
519 uint32_t cts, uint32_t usecs, uint64_t bw, uint32_t override, int mod);
520
521static int
522bbr_ctloutput(struct inpcb *inp, struct sockopt *sopt);
523
524static inline uint8_t
525bbr_state_val(struct tcp_bbr *bbr)
526{
527 return(bbr->rc_bbr_substate);
528}
529
530static inline uint32_t
531get_min_cwnd(struct tcp_bbr *bbr)
532{
533 int mss;
534
535 mss = min((bbr->rc_tp->t_maxseg - bbr->rc_last_options), bbr->r_ctl.rc_pace_max_segs);
536 if (bbr_get_rtt(bbr, BBR_RTT_PROP) < BBR_HIGH_SPEED)
537 return (bbr_cwnd_min_val_hs * mss);
538 else
539 return (bbr_cwnd_min_val * mss);
540}
541
542static uint32_t
543bbr_get_persists_timer_val(struct tcpcb *tp, struct tcp_bbr *bbr)
544{
545 uint64_t srtt, var;
546 uint64_t ret_val;
547
548 bbr->r_ctl.rc_hpts_flags |= PACE_TMR_PERSIT;
549 if (tp->t_srtt == 0) {
550 srtt = (uint64_t)BBR_INITIAL_RTO;
551 var = 0;
552 } else {
553 srtt = ((uint64_t)TICKS_2_USEC(tp->t_srtt) >> TCP_RTT_SHIFT);
554 var = ((uint64_t)TICKS_2_USEC(tp->t_rttvar) >> TCP_RTT_SHIFT);
555 }
556 TCPT_RANGESET_NOSLOP(ret_val, ((srtt + var) * tcp_backoff[tp->t_rxtshift]),
557 bbr_persist_min, bbr_persist_max);
558 return ((uint32_t)ret_val);
559}
560
561static uint32_t
562bbr_timer_start(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts)
563{
564 /*
565 * Start the FR timer, we do this based on getting the first one in
566 * the rc_tmap. Note that if its NULL we must stop the timer. in all
567 * events we need to stop the running timer (if its running) before
568 * starting the new one.
569 */
570 uint32_t thresh, exp, to, srtt, time_since_sent, tstmp_touse;
571 int32_t idx;
572 int32_t is_tlp_timer = 0;
573 struct bbr_sendmap *rsm;
574
575 if (bbr->rc_all_timers_stopped) {
576 /* All timers have been stopped none are to run */
577 return (0);
578 }
579 if (bbr->rc_in_persist) {
580 /* We can't start any timer in persists */
581 return (bbr_get_persists_timer_val(tp, bbr));
582 }
583 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap);
584 if ((rsm == NULL) ||
585 ((tp->t_flags & TF_SACK_PERMIT) == 0) ||
586 (tp->t_state < TCPS_ESTABLISHED)) {
587 /* Nothing on the send map */
588activate_rxt:
589 if (SEQ_LT(tp->snd_una, tp->snd_max) || sbavail(&(tp->t_inpcb->inp_socket->so_snd))) {
590 uint64_t tov;
591
592 time_since_sent = 0;
593 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap);
594 if (rsm) {
595 idx = rsm->r_rtr_cnt - 1;
596 if (TSTMP_GEQ(rsm->r_tim_lastsent[idx], bbr->r_ctl.rc_tlp_rxt_last_time))
597 tstmp_touse = rsm->r_tim_lastsent[idx];
598 else
599 tstmp_touse = bbr->r_ctl.rc_tlp_rxt_last_time;
600 if (TSTMP_GT(tstmp_touse, cts))
601 time_since_sent = cts - tstmp_touse;
602 }
603 bbr->r_ctl.rc_hpts_flags |= PACE_TMR_RXT;
604 if (tp->t_srtt == 0)
605 tov = BBR_INITIAL_RTO;
606 else
607 tov = ((uint64_t)(TICKS_2_USEC(tp->t_srtt) +
608 ((uint64_t)TICKS_2_USEC(tp->t_rttvar) * (uint64_t)4)) >> TCP_RTT_SHIFT);
609 if (tp->t_rxtshift)
610 tov *= tcp_backoff[tp->t_rxtshift];
611 if (tov > time_since_sent)
612 tov -= time_since_sent;
613 else
614 tov = bbr->r_ctl.rc_min_to;
615 TCPT_RANGESET_NOSLOP(to, tov,
616 (bbr->r_ctl.rc_min_rto_ms * MS_IN_USEC),
617 (bbr->rc_max_rto_sec * USECS_IN_SECOND));
618 bbr_log_timer_var(bbr, 2, cts, 0, srtt, 0, to);
619 return (to);
620 }
621 return (0);
622 }
623 if (rsm->r_flags & BBR_ACKED) {
624 rsm = bbr_find_lowest_rsm(bbr);
625 if (rsm == NULL) {
626 /* No lowest? */
627 goto activate_rxt;
628 }
629 }
630 /* Convert from ms to usecs */
631 if (rsm->r_flags & BBR_SACK_PASSED) {
632 if ((tp->t_flags & TF_SENTFIN) &&
633 ((tp->snd_max - tp->snd_una) == 1) &&
634 (rsm->r_flags & BBR_HAS_FIN)) {
635 /*
636 * We don't start a bbr rack timer if all we have is
637 * a FIN outstanding.
638 */
639 goto activate_rxt;
640 }
641 srtt = bbr_get_rtt(bbr, BBR_RTT_RACK);
642 thresh = bbr_calc_thresh_rack(bbr, srtt, cts, rsm);
643 idx = rsm->r_rtr_cnt - 1;
644 exp = rsm->r_tim_lastsent[idx] + thresh;
645 if (SEQ_GEQ(exp, cts)) {
646 to = exp - cts;
647 if (to < bbr->r_ctl.rc_min_to) {
648 to = bbr->r_ctl.rc_min_to;
649 }
650 } else {
651 to = bbr->r_ctl.rc_min_to;
652 }
653 } else {
654 /* Ok we need to do a TLP not RACK */
655 if (bbr->rc_tlp_in_progress != 0) {
656 /*
657 * The previous send was a TLP.
658 */
659 goto activate_rxt;
660 }
661 rsm = TAILQ_LAST_FAST(&bbr->r_ctl.rc_tmap, bbr_sendmap, r_tnext);
662 if (rsm == NULL) {
663 /* We found no rsm to TLP with. */
664 goto activate_rxt;
665 }
666 if (rsm->r_flags & BBR_HAS_FIN) {
667 /* If its a FIN we don't do TLP */
668 rsm = NULL;
669 goto activate_rxt;
670 }
671 time_since_sent = 0;
672 idx = rsm->r_rtr_cnt - 1;
673 if (TSTMP_GEQ(rsm->r_tim_lastsent[idx], bbr->r_ctl.rc_tlp_rxt_last_time))
674 tstmp_touse = rsm->r_tim_lastsent[idx];
675 else
676 tstmp_touse = bbr->r_ctl.rc_tlp_rxt_last_time;
677 if (TSTMP_GT(tstmp_touse, cts))
678 time_since_sent = cts - tstmp_touse;
679 is_tlp_timer = 1;
680 srtt = bbr_get_rtt(bbr, bbr_tlp_type_to_use);
681 thresh = bbr_calc_thresh_tlp(tp, bbr, rsm, srtt, cts);
682 if (thresh > time_since_sent)
683 to = thresh - time_since_sent;
684 else
685 to = bbr->r_ctl.rc_min_to;
686 if (to > (((uint32_t)bbr->rc_max_rto_sec) * USECS_IN_SECOND)) {
687 /*
688 * If the TLP time works out to larger than the max
689 * RTO lets not do TLP.. just RTO.
690 */
691 goto activate_rxt;
692 }
693 if ((bbr->rc_tlp_rtx_out == 1) &&
694 (rsm->r_start == bbr->r_ctl.rc_last_tlp_seq)) {
695 /*
696 * Second retransmit of the same TLP
697 * lets not.
698 */
699 bbr->rc_tlp_rtx_out = 0;
700 goto activate_rxt;
701 }
702 if (rsm->r_start != bbr->r_ctl.rc_last_tlp_seq) {
703 /*
704 * The tail is no longer the last one I did a probe
705 * on
706 */
707 bbr->r_ctl.rc_tlp_seg_send_cnt = 0;
708 bbr->r_ctl.rc_last_tlp_seq = rsm->r_start;
709 }
710 }
711 if (is_tlp_timer == 0) {
712 BBR_STAT_INC(bbr_to_arm_rack);
713 bbr->r_ctl.rc_hpts_flags |= PACE_TMR_RACK;
714 } else {
715 bbr_log_timer_var(bbr, 1, cts, time_since_sent, srtt, thresh, to);
716 if (bbr->r_ctl.rc_tlp_seg_send_cnt > bbr_tlp_max_resend) {
717 /*
718 * We have exceeded how many times we can retran the
719 * current TLP timer, switch to the RTO timer.
720 */
721 goto activate_rxt;
722 } else {
723 BBR_STAT_INC(bbr_to_arm_tlp);
724 bbr->r_ctl.rc_hpts_flags |= PACE_TMR_TLP;
725 }
726 }
727 return (to);
728}
729
730static inline int32_t
731bbr_minseg(struct tcp_bbr *bbr)
732{
733 return (bbr->r_ctl.rc_pace_min_segs - bbr->rc_last_options);
734}
735
736static void
737bbr_start_hpts_timer(struct tcp_bbr *bbr, struct tcpcb *tp, uint32_t cts, int32_t frm, int32_t slot, uint32_t tot_len)
738{
739 struct inpcb *inp;
740 struct hpts_diag diag;
741 uint32_t delayed_ack = 0;
742 uint32_t left = 0;
743 uint32_t hpts_timeout;
744 uint8_t stopped;
745 int32_t delay_calc = 0;
746 uint32_t prev_delay = 0;
747
748 inp = tp->t_inpcb;
749 if (tcp_in_hpts(inp)) {
750 /* A previous call is already set up */
751 return;
752 }
753 if ((tp->t_state == TCPS_CLOSED) ||
754 (tp->t_state == TCPS_LISTEN)) {
755 return;
756 }
757 stopped = bbr->rc_tmr_stopped;
758 if (stopped && TSTMP_GT(bbr->r_ctl.rc_timer_exp, cts)) {
759 left = bbr->r_ctl.rc_timer_exp - cts;
760 }
761 bbr->r_ctl.rc_hpts_flags = 0;
762 bbr->r_ctl.rc_timer_exp = 0;
763 prev_delay = bbr->r_ctl.rc_last_delay_val;
764 if (bbr->r_ctl.rc_last_delay_val &&
765 (slot == 0)) {
766 /*
767 * If a previous pacer delay was in place we
768 * are not coming from the output side (where
769 * we calculate a delay, more likely a timer).
770 */
771 slot = bbr->r_ctl.rc_last_delay_val;
772 if (TSTMP_GT(cts, bbr->rc_pacer_started)) {
773 /* Compensate for time passed */
774 delay_calc = cts - bbr->rc_pacer_started;
775 if (delay_calc <= slot)
776 slot -= delay_calc;
777 }
778 }
779 /* Do we have early to make up for by pushing out the pacing time? */
780 if (bbr->r_agg_early_set) {
781 bbr_log_pacing_delay_calc(bbr, 0, bbr->r_ctl.rc_agg_early, cts, slot, 0, bbr->r_agg_early_set, 2);
782 slot += bbr->r_ctl.rc_agg_early;
783 bbr->r_ctl.rc_agg_early = 0;
784 bbr->r_agg_early_set = 0;
785 }
786 /* Are we running a total debt that needs to be compensated for? */
787 if (bbr->r_ctl.rc_hptsi_agg_delay) {
788 if (slot > bbr->r_ctl.rc_hptsi_agg_delay) {
789 /* We nuke the delay */
790 slot -= bbr->r_ctl.rc_hptsi_agg_delay;
791 bbr->r_ctl.rc_hptsi_agg_delay = 0;
792 } else {
793 /* We nuke some of the delay, put in a minimal 100usecs */
794 bbr->r_ctl.rc_hptsi_agg_delay -= slot;
795 bbr->r_ctl.rc_last_delay_val = slot = 100;
796 }
797 }
798 bbr->r_ctl.rc_last_delay_val = slot;
799 hpts_timeout = bbr_timer_start(tp, bbr, cts);
800 if (tp->t_flags & TF_DELACK) {
801 if (bbr->rc_in_persist == 0) {
802 delayed_ack = bbr_delack_time;
803 } else {
804 /*
805 * We are in persists and have
806 * gotten a new data element.
807 */
808 if (hpts_timeout > bbr_delack_time) {
809 /*
810 * Lets make the persists timer (which acks)
811 * be the smaller of hpts_timeout and bbr_delack_time.
812 */
813 hpts_timeout = bbr_delack_time;
814 }
815 }
816 }
817 if (delayed_ack &&
818 ((hpts_timeout == 0) ||
819 (delayed_ack < hpts_timeout))) {
820 /* We need a Delayed ack timer */
821 bbr->r_ctl.rc_hpts_flags = PACE_TMR_DELACK;
822 hpts_timeout = delayed_ack;
823 }
824 if (slot) {
825 /* Mark that we have a pacing timer up */
826 BBR_STAT_INC(bbr_paced_segments);
827 bbr->r_ctl.rc_hpts_flags |= PACE_PKT_OUTPUT;
828 }
829 /*
830 * If no timers are going to run and we will fall off thfe hptsi
831 * wheel, we resort to a keep-alive timer if its configured.
832 */
833 if ((hpts_timeout == 0) &&
834 (slot == 0)) {
835 if ((V_tcp_always_keepalive || inp->inp_socket->so_options & SO_KEEPALIVE) &&
836 (tp->t_state <= TCPS_CLOSING)) {
837 /*
838 * Ok we have no timer (persists, rack, tlp, rxt or
839 * del-ack), we don't have segments being paced. So
840 * all that is left is the keepalive timer.
841 */
842 if (TCPS_HAVEESTABLISHED(tp->t_state)) {
843 hpts_timeout = TICKS_2_USEC(TP_KEEPIDLE(tp));
844 } else {
845 hpts_timeout = TICKS_2_USEC(TP_KEEPINIT(tp));
846 }
847 bbr->r_ctl.rc_hpts_flags |= PACE_TMR_KEEP;
848 }
849 }
850 if (left && (stopped & (PACE_TMR_KEEP | PACE_TMR_DELACK)) ==
851 (bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK)) {
852 /*
853 * RACK, TLP, persists and RXT timers all are restartable
854 * based on actions input .. i.e we received a packet (ack
855 * or sack) and that changes things (rw, or snd_una etc).
856 * Thus we can restart them with a new value. For
857 * keep-alive, delayed_ack we keep track of what was left
858 * and restart the timer with a smaller value.
859 */
860 if (left < hpts_timeout)
861 hpts_timeout = left;
862 }
863 if (bbr->r_ctl.rc_incr_tmrs && slot &&
864 (bbr->r_ctl.rc_hpts_flags & (PACE_TMR_TLP|PACE_TMR_RXT))) {
865 /*
866 * If configured to do so, and the timer is either
867 * the TLP or RXT timer, we need to increase the timeout
868 * by the pacing time. Consider the bottleneck at my
869 * machine as an example, we are sending something
870 * to start a TLP on. The last packet won't be emitted
871 * fully until the pacing time (the bottleneck will hold
872 * the data in place). Once the packet is emitted that
873 * is when we want to start waiting for the TLP. This
874 * is most evident with hardware pacing (where the nic
875 * is holding the packet(s) before emitting). But it
876 * can also show up in the network so we do it for all
877 * cases. Technically we would take off one packet from
878 * this extra delay but this is easier and being more
879 * conservative is probably better.
880 */
881 hpts_timeout += slot;
882 }
883 if (hpts_timeout) {
884 /*
885 * Hack alert for now we can't time-out over 2147 seconds (a
886 * bit more than 35min)
887 */
888 if (hpts_timeout > 0x7ffffffe)
889 hpts_timeout = 0x7ffffffe;
890 bbr->r_ctl.rc_timer_exp = cts + hpts_timeout;
891 } else
892 bbr->r_ctl.rc_timer_exp = 0;
893 if ((slot) &&
894 (bbr->rc_use_google ||
895 bbr->output_error_seen ||
896 (slot <= hpts_timeout)) ) {
897 /*
898 * Tell LRO that it can queue packets while
899 * we pace.
900 */
901 bbr->rc_inp->inp_flags2 |= INP_MBUF_QUEUE_READY;
902 if ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_RACK) &&
903 (bbr->rc_cwnd_limited == 0)) {
904 /*
905 * If we are not cwnd limited and we
906 * are running a rack timer we put on
907 * the do not disturbe even for sack.
908 */
909 inp->inp_flags2 |= INP_DONT_SACK_QUEUE;
910 } else
911 inp->inp_flags2 &= ~INP_DONT_SACK_QUEUE;
912 bbr->rc_pacer_started = cts;
913
914 (void)tcp_hpts_insert_diag(tp->t_inpcb, HPTS_USEC_TO_SLOTS(slot),
915 __LINE__, &diag);
916 bbr->rc_timer_first = 0;
917 bbr->bbr_timer_src = frm;
918 bbr_log_to_start(bbr, cts, hpts_timeout, slot, 1);
919 bbr_log_hpts_diag(bbr, cts, &diag);
920 } else if (hpts_timeout) {
921 (void)tcp_hpts_insert_diag(tp->t_inpcb, HPTS_USEC_TO_SLOTS(hpts_timeout),
922 __LINE__, &diag);
923 /*
924 * We add the flag here as well if the slot is set,
925 * since hpts will call in to clear the queue first before
926 * calling the output routine (which does our timers).
927 * We don't want to set the flag if its just a timer
928 * else the arrival of data might (that causes us
929 * to send more) might get delayed. Imagine being
930 * on a keep-alive timer and a request comes in for
931 * more data.
932 */
933 if (slot)
934 bbr->rc_pacer_started = cts;
935 if ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_RACK) &&
936 (bbr->rc_cwnd_limited == 0)) {
937 /*
938 * For a rack timer, don't wake us even
939 * if a sack arrives as long as we are
940 * not cwnd limited.
941 */
942 bbr->rc_inp->inp_flags2 |= INP_MBUF_QUEUE_READY;
943 inp->inp_flags2 |= INP_DONT_SACK_QUEUE;
944 } else {
945 /* All other timers wake us up */
946 bbr->rc_inp->inp_flags2 &= ~INP_MBUF_QUEUE_READY;
947 inp->inp_flags2 &= ~INP_DONT_SACK_QUEUE;
948 }
949 bbr->bbr_timer_src = frm;
950 bbr_log_to_start(bbr, cts, hpts_timeout, slot, 0);
951 bbr_log_hpts_diag(bbr, cts, &diag);
952 bbr->rc_timer_first = 1;
953 }
954 bbr->rc_tmr_stopped = 0;
955 bbr_log_type_bbrsnd(bbr, tot_len, slot, delay_calc, cts, frm, prev_delay);
956}
957
958static void
959bbr_timer_audit(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts, struct sockbuf *sb)
960{
961 /*
962 * We received an ack, and then did not call send or were bounced
963 * out due to the hpts was running. Now a timer is up as well, is it
964 * the right timer?
965 */
966 struct inpcb *inp;
967 struct bbr_sendmap *rsm;
968 uint32_t hpts_timeout;
969 int tmr_up;
970
971 tmr_up = bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK;
972 if (bbr->rc_in_persist && (tmr_up == PACE_TMR_PERSIT))
973 return;
974 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap);
975 if (((rsm == NULL) || (tp->t_state < TCPS_ESTABLISHED)) &&
976 (tmr_up == PACE_TMR_RXT)) {
977 /* Should be an RXT */
978 return;
979 }
980 inp = bbr->rc_inp;
981 if (rsm == NULL) {
982 /* Nothing outstanding? */
983 if (tp->t_flags & TF_DELACK) {
984 if (tmr_up == PACE_TMR_DELACK)
985 /*
986 * We are supposed to have delayed ack up
987 * and we do
988 */
989 return;
990 } else if (sbavail(&inp->inp_socket->so_snd) &&
991 (tmr_up == PACE_TMR_RXT)) {
992 /*
993 * if we hit enobufs then we would expect the
994 * possiblity of nothing outstanding and the RXT up
995 * (and the hptsi timer).
996 */
997 return;
998 } else if (((V_tcp_always_keepalive ||
999 inp->inp_socket->so_options & SO_KEEPALIVE) &&
1000 (tp->t_state <= TCPS_CLOSING)) &&
1001 (tmr_up == PACE_TMR_KEEP) &&
1002 (tp->snd_max == tp->snd_una)) {
1003 /* We should have keep alive up and we do */
1004 return;
1005 }
1006 }
1007 if (rsm && (rsm->r_flags & BBR_SACK_PASSED)) {
1008 if ((tp->t_flags & TF_SENTFIN) &&
1009 ((tp->snd_max - tp->snd_una) == 1) &&
1010 (rsm->r_flags & BBR_HAS_FIN)) {
1011 /* needs to be a RXT */
1012 if (tmr_up == PACE_TMR_RXT)
1013 return;
1014 else
1015 goto wrong_timer;
1016 } else if (tmr_up == PACE_TMR_RACK)
1017 return;
1018 else
1019 goto wrong_timer;
1020 } else if (rsm && (tmr_up == PACE_TMR_RACK)) {
1021 /* Rack timer has priority if we have data out */
1022 return;
1023 } else if (SEQ_GT(tp->snd_max, tp->snd_una) &&
1024 ((tmr_up == PACE_TMR_TLP) ||
1025 (tmr_up == PACE_TMR_RXT))) {
1026 /*
1027 * Either a TLP or RXT is fine if no sack-passed is in place
1028 * and data is outstanding.
1029 */
1030 return;
1031 } else if (tmr_up == PACE_TMR_DELACK) {
1032 /*
1033 * If the delayed ack was going to go off before the
1034 * rtx/tlp/rack timer were going to expire, then that would
1035 * be the timer in control. Note we don't check the time
1036 * here trusting the code is correct.
1037 */
1038 return;
1039 }
1040 if (SEQ_GT(tp->snd_max, tp->snd_una) &&
1041 ((tmr_up == PACE_TMR_RXT) ||
1042 (tmr_up == PACE_TMR_TLP) ||
1043 (tmr_up == PACE_TMR_RACK))) {
1044 /*
1045 * We have outstanding data and
1046 * we *do* have a RACK, TLP or RXT
1047 * timer running. We won't restart
1048 * anything here since thats probably ok we
1049 * will get called with some timer here shortly.
1050 */
1051 return;
1052 }
1053 /*
1054 * Ok the timer originally started is not what we want now. We will
1055 * force the hpts to be stopped if any, and restart with the slot
1056 * set to what was in the saved slot.
1057 */
1058wrong_timer:
1059 if ((bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0) {
1060 if (tcp_in_hpts(inp))
1061 tcp_hpts_remove(inp);
1062 bbr_timer_cancel(bbr, __LINE__, cts);
1063 bbr_start_hpts_timer(bbr, tp, cts, 1, bbr->r_ctl.rc_last_delay_val,
1064 0);
1065 } else {
1066 /*
1067 * Output is hptsi so we just need to switch the type of
1068 * timer. We don't bother with keep-alive, since when we
1069 * jump through the output, it will start the keep-alive if
1070 * nothing is sent.
1071 *
1072 * We only need a delayed-ack added and or the hpts_timeout.
1073 */
1074 hpts_timeout = bbr_timer_start(tp, bbr, cts);
1075 if (tp->t_flags & TF_DELACK) {
1076 if (hpts_timeout == 0) {
1077 hpts_timeout = bbr_delack_time;
1078 bbr->r_ctl.rc_hpts_flags = PACE_TMR_DELACK;
1079 }
1080 else if (hpts_timeout > bbr_delack_time) {
1081 hpts_timeout = bbr_delack_time;
1082 bbr->r_ctl.rc_hpts_flags = PACE_TMR_DELACK;
1083 }
1084 }
1085 if (hpts_timeout) {
1086 if (hpts_timeout > 0x7ffffffe)
1087 hpts_timeout = 0x7ffffffe;
1088 bbr->r_ctl.rc_timer_exp = cts + hpts_timeout;
1089 }
1090 }
1091}
1092
1093int32_t bbr_clear_lost = 0;
1094
1095/*
1096 * Considers the two time values now (cts) and earlier.
1097 * If cts is smaller than earlier, we could have
1098 * had a sequence wrap (our counter wraps every
1099 * 70 min or so) or it could be just clock skew
1100 * getting us two different time values. Clock skew
1101 * will show up within 10ms or so. So in such
1102 * a case (where cts is behind earlier time by
1103 * less than 10ms) we return 0. Otherwise we
1104 * return the true difference between them.
1105 */
1106static inline uint32_t
1107bbr_calc_time(uint32_t cts, uint32_t earlier_time) {
1108 /*
1109 * Given two timestamps, the current time stamp cts, and some other
1110 * time-stamp taken in theory earlier return the difference. The
1111 * trick is here sometimes locking will get the other timestamp
1112 * after the cts. If this occurs we need to return 0.
1113 */
1114 if (TSTMP_GEQ(cts, earlier_time))
1115 return (cts - earlier_time);
1116 /*
1117 * cts is behind earlier_time if its less than 10ms consider it 0.
1118 * If its more than 10ms difference then we had a time wrap. Else
1119 * its just the normal locking foo. I wonder if we should not go to
1120 * 64bit TS and get rid of this issue.
1121 */
1122 if (TSTMP_GEQ((cts + 10000), earlier_time))
1123 return (0);
1124 /*
1125 * Ok the time must have wrapped. So we need to answer a large
1126 * amount of time, which the normal subtraction should do.
1127 */
1128 return (cts - earlier_time);
1129}
1130
1131static int
1132sysctl_bbr_clear_lost(SYSCTL_HANDLER_ARGS)
1133{
1134 uint32_t stat;
1135 int32_t error;
1136
1137 error = SYSCTL_OUT(req, &bbr_clear_lost, sizeof(uint32_t));
1138 if (error || req->newptr == NULL)
1139 return error;
1140
1141 error = SYSCTL_IN(req, &stat, sizeof(uint32_t));
1142 if (error)
1143 return (error);
1144 if (stat == 1) {
1145#ifdef BBR_INVARIANTS
1146 printf("Clearing BBR lost counters\n");
1147#endif
1148 COUNTER_ARRAY_ZERO(bbr_state_lost, BBR_MAX_STAT);
1149 COUNTER_ARRAY_ZERO(bbr_state_time, BBR_MAX_STAT);
1150 COUNTER_ARRAY_ZERO(bbr_state_resend, BBR_MAX_STAT);
1151 } else if (stat == 2) {
1152#ifdef BBR_INVARIANTS
1153 printf("Clearing BBR option counters\n");
1154#endif
1155 COUNTER_ARRAY_ZERO(bbr_opts_arry, BBR_OPTS_SIZE);
1156 } else if (stat == 3) {
1157#ifdef BBR_INVARIANTS
1158 printf("Clearing BBR stats counters\n");
1159#endif
1160 COUNTER_ARRAY_ZERO(bbr_stat_arry, BBR_STAT_SIZE);
1161 } else if (stat == 4) {
1162#ifdef BBR_INVARIANTS
1163 printf("Clearing BBR out-size counters\n");
1164#endif
1165 COUNTER_ARRAY_ZERO(bbr_out_size, TCP_MSS_ACCT_SIZE);
1166 }
1167 bbr_clear_lost = 0;
1168 return (0);
1169}
1170
1171static void
1172bbr_init_sysctls(void)
1173{
1174 struct sysctl_oid *bbr_probertt;
1175 struct sysctl_oid *bbr_hptsi;
1176 struct sysctl_oid *bbr_measure;
1177 struct sysctl_oid *bbr_cwnd;
1178 struct sysctl_oid *bbr_timeout;
1179 struct sysctl_oid *bbr_states;
1180 struct sysctl_oid *bbr_startup;
1181 struct sysctl_oid *bbr_policer;
1182
1183 /* Probe rtt controls */
1184 bbr_probertt = SYSCTL_ADD_NODE(&bbr_sysctl_ctx,
1185 SYSCTL_CHILDREN(bbr_sysctl_root),
1186 OID_AUTO,
1187 "probertt",
1188 CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
1189 "");
1190 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1191 SYSCTL_CHILDREN(bbr_probertt),
1192 OID_AUTO, "gain", CTLFLAG_RW,
1193 &bbr_rttprobe_gain, 192,
1194 "What is the filter gain drop in probe_rtt (0=disable)?");
1195 SYSCTL_ADD_U32(&bbr_sysctl_ctx,
1196 SYSCTL_CHILDREN(bbr_probertt),
1197 OID_AUTO, "cwnd", CTLFLAG_RW,
1198 &bbr_rtt_probe_cwndtarg, 4,
1199 "How many mss's are outstanding during probe-rtt");
1200 SYSCTL_ADD_U32(&bbr_sysctl_ctx,
1201 SYSCTL_CHILDREN(bbr_probertt),
1202 OID_AUTO, "int", CTLFLAG_RW,
1203 &bbr_rtt_probe_limit, 4000000,
1204 "If RTT has not shrank in this many micro-seconds enter probe-rtt");
1205 SYSCTL_ADD_U32(&bbr_sysctl_ctx,
1206 SYSCTL_CHILDREN(bbr_probertt),
1207 OID_AUTO, "mintime", CTLFLAG_RW,
1208 &bbr_rtt_probe_time, 200000,
1209 "How many microseconds in probe-rtt");
1210 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1211 SYSCTL_CHILDREN(bbr_probertt),
1212 OID_AUTO, "filter_len_sec", CTLFLAG_RW,
1213 &bbr_filter_len_sec, 6,
1214 "How long in seconds does the rttProp filter run?");
1215 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1216 SYSCTL_CHILDREN(bbr_probertt),
1217 OID_AUTO, "drain_rtt", CTLFLAG_RW,
1218 &bbr_drain_rtt, BBR_SRTT,
1219 "What is the drain rtt to use in probeRTT (rtt_prop=0, rtt_rack=1, rtt_pkt=2, rtt_srtt=3?");
1220 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1221 SYSCTL_CHILDREN(bbr_probertt),
1222 OID_AUTO, "can_force", CTLFLAG_RW,
1223 &bbr_can_force_probertt, 0,
1224 "If we keep setting new low rtt's but delay going in probe-rtt can we force in??");
1225 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1226 SYSCTL_CHILDREN(bbr_probertt),
1227 OID_AUTO, "enter_sets_force", CTLFLAG_RW,
1228 &bbr_probertt_sets_rtt, 0,
1229 "In NF mode, do we imitate google_mode and set the rttProp on entry to probe-rtt?");
1230 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1231 SYSCTL_CHILDREN(bbr_probertt),
1232 OID_AUTO, "can_adjust", CTLFLAG_RW,
1233 &bbr_can_adjust_probertt, 1,
1234 "Can we dynamically adjust the probe-rtt limits and times?");
1235 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1236 SYSCTL_CHILDREN(bbr_probertt),
1237 OID_AUTO, "is_ratio", CTLFLAG_RW,
1238 &bbr_is_ratio, 0,
1239 "is the limit to filter a ratio?");
1240 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1241 SYSCTL_CHILDREN(bbr_probertt),
1242 OID_AUTO, "use_cwnd", CTLFLAG_RW,
1243 &bbr_prtt_slam_cwnd, 0,
1244 "Should we set/recover cwnd?");
1245 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1246 SYSCTL_CHILDREN(bbr_probertt),
1247 OID_AUTO, "can_use_ts", CTLFLAG_RW,
1248 &bbr_can_use_ts_for_rtt, 1,
1249 "Can we use the ms timestamp if available for retransmistted rtt calculations?");
1250
1251 /* Pacing controls */
1252 bbr_hptsi = SYSCTL_ADD_NODE(&bbr_sysctl_ctx,
1253 SYSCTL_CHILDREN(bbr_sysctl_root),
1254 OID_AUTO,
1255 "pacing",
1256 CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
1257 "");
1258 SYSCTL_ADD_U32(&bbr_sysctl_ctx,
1259 SYSCTL_CHILDREN(bbr_hptsi),
1260 OID_AUTO, "hw_pacing", CTLFLAG_RW,
1261 &bbr_allow_hdwr_pacing, 1,
1262 "Do we allow hardware pacing?");
1263 SYSCTL_ADD_U32(&bbr_sysctl_ctx,
1264 SYSCTL_CHILDREN(bbr_hptsi),
1265 OID_AUTO, "hw_pacing_limit", CTLFLAG_RW,
1266 &bbr_hardware_pacing_limit, 4000,
1267 "Do we have a limited number of connections for pacing chelsio (0=no limit)?");
1268 SYSCTL_ADD_U32(&bbr_sysctl_ctx,
1269 SYSCTL_CHILDREN(bbr_hptsi),
1270 OID_AUTO, "hw_pacing_adj", CTLFLAG_RW,
1271 &bbr_hdwr_pace_adjust, 2,
1272 "Multiplier to calculated tso size?");
1273 SYSCTL_ADD_U32(&bbr_sysctl_ctx,
1274 SYSCTL_CHILDREN(bbr_hptsi),
1275 OID_AUTO, "hw_pacing_floor", CTLFLAG_RW,
1276 &bbr_hdwr_pace_floor, 1,
1277 "Do we invoke the hardware pacing floor?");
1278 SYSCTL_ADD_U32(&bbr_sysctl_ctx,
1279 SYSCTL_CHILDREN(bbr_hptsi),
1280 OID_AUTO, "hw_pacing_delay_cnt", CTLFLAG_RW,
1281 &bbr_hdwr_pacing_delay_cnt, 10,
1282 "How many packets must be sent after hdwr pacing is enabled");
1283 SYSCTL_ADD_U32(&bbr_sysctl_ctx,
1284 SYSCTL_CHILDREN(bbr_hptsi),
1285 OID_AUTO, "bw_cross", CTLFLAG_RW,
1286 &bbr_cross_over, 3000000,
1287 "What is the point where we cross over to linux like TSO size set");
1288 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1289 SYSCTL_CHILDREN(bbr_hptsi),
1290 OID_AUTO, "seg_deltarg", CTLFLAG_RW,
1291 &bbr_hptsi_segments_delay_tar, 7000,
1292 "What is the worse case delay target for hptsi < 48Mbp connections");
1293 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1294 SYSCTL_CHILDREN(bbr_hptsi),
1295 OID_AUTO, "enet_oh", CTLFLAG_RW,
1296 &bbr_include_enet_oh, 0,
1297 "Do we include the ethernet overhead in calculating pacing delay?");
1298 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1299 SYSCTL_CHILDREN(bbr_hptsi),
1300 OID_AUTO, "ip_oh", CTLFLAG_RW,
1301 &bbr_include_ip_oh, 1,
1302 "Do we include the IP overhead in calculating pacing delay?");
1303 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1304 SYSCTL_CHILDREN(bbr_hptsi),
1305 OID_AUTO, "tcp_oh", CTLFLAG_RW,
1306 &bbr_include_tcp_oh, 0,
1307 "Do we include the TCP overhead in calculating pacing delay?");
1308 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1309 SYSCTL_CHILDREN(bbr_hptsi),
1310 OID_AUTO, "google_discount", CTLFLAG_RW,
1311 &bbr_google_discount, 10,
1312 "What is the default google discount percentage wise for pacing (11 = 1.1%%)?");
1313 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1314 SYSCTL_CHILDREN(bbr_hptsi),
1315 OID_AUTO, "all_get_min", CTLFLAG_RW,
1316 &bbr_all_get_min, 0,
1317 "If you are less than a MSS do you just get the min?");
1318 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1319 SYSCTL_CHILDREN(bbr_hptsi),
1320 OID_AUTO, "tso_min", CTLFLAG_RW,
1321 &bbr_hptsi_bytes_min, 1460,
1322 "For 0 -> 24Mbps what is floor number of segments for TSO");
1323 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1324 SYSCTL_CHILDREN(bbr_hptsi),
1325 OID_AUTO, "seg_tso_max", CTLFLAG_RW,
1326 &bbr_hptsi_segments_max, 6,
1327 "For 0 -> 24Mbps what is top number of segments for TSO");
1328 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1329 SYSCTL_CHILDREN(bbr_hptsi),
1330 OID_AUTO, "seg_floor", CTLFLAG_RW,
1331 &bbr_hptsi_segments_floor, 1,
1332 "Minimum TSO size we will fall too in segments");
1333 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1334 SYSCTL_CHILDREN(bbr_hptsi),
1335 OID_AUTO, "utter_max", CTLFLAG_RW,
1336 &bbr_hptsi_utter_max, 0,
1337 "The absolute maximum that any pacing (outside of hardware) can be");
1338 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1339 SYSCTL_CHILDREN(bbr_hptsi),
1340 OID_AUTO, "seg_divisor", CTLFLAG_RW,
1341 &bbr_hptsi_per_second, 100,
1342 "What is the divisor in our hptsi TSO calculation 512Mbps < X > 24Mbps ");
1343 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1344 SYSCTL_CHILDREN(bbr_hptsi),
1345 OID_AUTO, "srtt_mul", CTLFLAG_RW,
1346 &bbr_hptsi_max_mul, 1,
1347 "The multiplier for pace len max");
1348 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1349 SYSCTL_CHILDREN(bbr_hptsi),
1350 OID_AUTO, "srtt_div", CTLFLAG_RW,
1351 &bbr_hptsi_max_div, 2,
1352 "The divisor for pace len max");
1353 /* Measurement controls */
1354 bbr_measure = SYSCTL_ADD_NODE(&bbr_sysctl_ctx,
1355 SYSCTL_CHILDREN(bbr_sysctl_root),
1356 OID_AUTO,
1357 "measure",
1358 CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
1359 "Measurement controls");
1360 SYSCTL_ADD_U32(&bbr_sysctl_ctx,
1361 SYSCTL_CHILDREN(bbr_measure),
1362 OID_AUTO, "min_i_bw", CTLFLAG_RW,
1363 &bbr_initial_bw_bps, 62500,
1364 "Minimum initial b/w in bytes per second");
1365 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1366 SYSCTL_CHILDREN(bbr_measure),
1367 OID_AUTO, "no_sack_needed", CTLFLAG_RW,
1368 &bbr_sack_not_required, 0,
1369 "Do we allow bbr to run on connections not supporting SACK?");
1370 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1371 SYSCTL_CHILDREN(bbr_measure),
1372 OID_AUTO, "use_google", CTLFLAG_RW,
1373 &bbr_use_google_algo, 0,
1374 "Use has close to google V1.0 has possible?");
1375 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1376 SYSCTL_CHILDREN(bbr_measure),
1377 OID_AUTO, "ts_limiting", CTLFLAG_RW,
1378 &bbr_ts_limiting, 1,
1379 "Do we attempt to use the peers timestamp to limit b/w caculations?");
1380 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1381 SYSCTL_CHILDREN(bbr_measure),
1382 OID_AUTO, "ts_can_raise", CTLFLAG_RW,
1383 &bbr_ts_can_raise, 0,
1384 "Can we raise the b/w via timestamp b/w calculation?");
1385 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1386 SYSCTL_CHILDREN(bbr_measure),
1387 OID_AUTO, "ts_delta", CTLFLAG_RW,
1388 &bbr_min_usec_delta, 20000,
1389 "How long in usec between ts of our sends in ts validation code?");
1390 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1391 SYSCTL_CHILDREN(bbr_measure),
1392 OID_AUTO, "ts_peer_delta", CTLFLAG_RW,
1393 &bbr_min_peer_delta, 20,
1394 "What min numerical value should be between the peer deltas?");
1395 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1396 SYSCTL_CHILDREN(bbr_measure),
1397 OID_AUTO, "ts_delta_percent", CTLFLAG_RW,
1398 &bbr_delta_percent, 150,
1399 "What percentage (150 = 15.0) do we allow variance for?");
1400 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1401 SYSCTL_CHILDREN(bbr_measure),
1402 OID_AUTO, "min_measure_good_bw", CTLFLAG_RW,
1403 &bbr_min_measurements_req, 1,
1404 "What is the minimum measurement count we need before we switch to our b/w estimate");
1405 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1406 SYSCTL_CHILDREN(bbr_measure),
1407 OID_AUTO, "min_measure_before_pace", CTLFLAG_RW,
1408 &bbr_no_pacing_until, 4,
1409 "How many pkt-epoch's (0 is off) do we need before pacing is on?");
1410 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1411 SYSCTL_CHILDREN(bbr_measure),
1412 OID_AUTO, "quanta", CTLFLAG_RW,
1413 &bbr_quanta, 2,
1414 "Extra quanta to add when calculating the target (ID section 4.2.3.2).");
1415 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1416 SYSCTL_CHILDREN(bbr_measure),
1417 OID_AUTO, "noretran", CTLFLAG_RW,
1418 &bbr_no_retran, 0,
1419 "Should google mode not use retransmission measurements for the b/w estimation?");
1420 /* State controls */
1421 bbr_states = SYSCTL_ADD_NODE(&bbr_sysctl_ctx,
1422 SYSCTL_CHILDREN(bbr_sysctl_root),
1423 OID_AUTO,
1424 "states",
1425 CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
1426 "State controls");
1427 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1428 SYSCTL_CHILDREN(bbr_states),
1429 OID_AUTO, "idle_restart", CTLFLAG_RW,
1430 &bbr_uses_idle_restart, 0,
1431 "Do we use a new special idle_restart state to ramp back up quickly?");
1432 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1433 SYSCTL_CHILDREN(bbr_states),
1434 OID_AUTO, "idle_restart_threshold", CTLFLAG_RW,
1435 &bbr_idle_restart_threshold, 100000,
1436 "How long must we be idle before we restart??");
1437 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1438 SYSCTL_CHILDREN(bbr_states),
1439 OID_AUTO, "use_pkt_epoch", CTLFLAG_RW,
1440 &bbr_state_is_pkt_epoch, 0,
1441 "Do we use a pkt-epoch for substate if 0 rttProp?");
1442 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1443 SYSCTL_CHILDREN(bbr_states),
1444 OID_AUTO, "startup_rtt_gain", CTLFLAG_RW,
1445 &bbr_rtt_gain_thresh, 0,
1446 "What increase in RTT triggers us to stop ignoring no-loss and possibly exit startup?");
1447 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1448 SYSCTL_CHILDREN(bbr_states),
1449 OID_AUTO, "drain_floor", CTLFLAG_RW,
1450 &bbr_drain_floor, 88,
1451 "What is the lowest we can drain (pg) too?");
1452 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1453 SYSCTL_CHILDREN(bbr_states),
1454 OID_AUTO, "drain_2_target", CTLFLAG_RW,
1455 &bbr_state_drain_2_tar, 1,
1456 "Do we drain to target in drain substate?");
1457 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1458 SYSCTL_CHILDREN(bbr_states),
1459 OID_AUTO, "gain_2_target", CTLFLAG_RW,
1460 &bbr_gain_to_target, 1,
1461 "Does probe bw gain to target??");
1462 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1463 SYSCTL_CHILDREN(bbr_states),
1464 OID_AUTO, "gain_extra_time", CTLFLAG_RW,
1465 &bbr_gain_gets_extra_too, 1,
1466 "Does probe bw gain get the extra time too?");
1467 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1468 SYSCTL_CHILDREN(bbr_states),
1469 OID_AUTO, "ld_div", CTLFLAG_RW,
1470 &bbr_drain_drop_div, 5,
1471 "Long drain drop divider?");
1472 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1473 SYSCTL_CHILDREN(bbr_states),
1474 OID_AUTO, "ld_mul", CTLFLAG_RW,
1475 &bbr_drain_drop_mul, 4,
1476 "Long drain drop multiplier?");
1477 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1478 SYSCTL_CHILDREN(bbr_states),
1479 OID_AUTO, "rand_ot_disc", CTLFLAG_RW,
1480 &bbr_rand_ot, 50,
1481 "Random discount of the ot?");
1482 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1483 SYSCTL_CHILDREN(bbr_states),
1484 OID_AUTO, "dr_filter_life", CTLFLAG_RW,
1485 &bbr_num_pktepo_for_del_limit, BBR_NUM_RTTS_FOR_DEL_LIMIT,
1486 "How many packet-epochs does the b/w delivery rate last?");
1487 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1488 SYSCTL_CHILDREN(bbr_states),
1489 OID_AUTO, "subdrain_applimited", CTLFLAG_RW,
1490 &bbr_sub_drain_app_limit, 0,
1491 "Does our sub-state drain invoke app limited if its long?");
1492 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1493 SYSCTL_CHILDREN(bbr_states),
1494 OID_AUTO, "use_cwnd_subdrain", CTLFLAG_RW,
1495 &bbr_sub_drain_slam_cwnd, 0,
1496 "Should we set/recover cwnd for sub-state drain?");
1497 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1498 SYSCTL_CHILDREN(bbr_states),
1499 OID_AUTO, "use_cwnd_maindrain", CTLFLAG_RW,
1500 &bbr_slam_cwnd_in_main_drain, 0,
1501 "Should we set/recover cwnd for main-state drain?");
1502 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1503 SYSCTL_CHILDREN(bbr_states),
1504 OID_AUTO, "google_gets_earlyout", CTLFLAG_RW,
1505 &google_allow_early_out, 1,
1506 "Should we allow google probe-bw/drain to exit early at flight target?");
1507 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1508 SYSCTL_CHILDREN(bbr_states),
1509 OID_AUTO, "google_exit_loss", CTLFLAG_RW,
1510 &google_consider_lost, 1,
1511 "Should we have losses exit gain of probebw in google mode??");
1512 /* Startup controls */
1513 bbr_startup = SYSCTL_ADD_NODE(&bbr_sysctl_ctx,
1514 SYSCTL_CHILDREN(bbr_sysctl_root),
1515 OID_AUTO,
1516 "startup",
1517 CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
1518 "Startup controls");
1519 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1520 SYSCTL_CHILDREN(bbr_startup),
1521 OID_AUTO, "cheat_iwnd", CTLFLAG_RW,
1522 &bbr_sends_full_iwnd, 1,
1523 "Do we not pace but burst out initial windows has our TSO size?");
1524 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1525 SYSCTL_CHILDREN(bbr_startup),
1526 OID_AUTO, "loss_threshold", CTLFLAG_RW,
1527 &bbr_startup_loss_thresh, 2000,
1528 "In startup what is the loss threshold in a pe that will exit us from startup?");
1529 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1530 SYSCTL_CHILDREN(bbr_startup),
1531 OID_AUTO, "use_lowerpg", CTLFLAG_RW,
1532 &bbr_use_lower_gain_in_startup, 1,
1533 "Should we use a lower hptsi gain if we see loss in startup?");
1534 SYSCTL_ADD_U32(&bbr_sysctl_ctx,
1535 SYSCTL_CHILDREN(bbr_startup),
1536 OID_AUTO, "gain", CTLFLAG_RW,
1537 &bbr_start_exit, 25,
1538 "What gain percent do we need to see to stay in startup??");
1539 SYSCTL_ADD_U32(&bbr_sysctl_ctx,
1540 SYSCTL_CHILDREN(bbr_startup),
1541 OID_AUTO, "low_gain", CTLFLAG_RW,
1542 &bbr_low_start_exit, 15,
1543 "What gain percent do we need to see to stay in the lower gain startup??");
1544 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1545 SYSCTL_CHILDREN(bbr_startup),
1546 OID_AUTO, "loss_exit", CTLFLAG_RW,
1547 &bbr_exit_startup_at_loss, 1,
1548 "Should we exit startup at loss in an epoch if we are not gaining?");
1549 /* CWND controls */
1550 bbr_cwnd = SYSCTL_ADD_NODE(&bbr_sysctl_ctx,
1551 SYSCTL_CHILDREN(bbr_sysctl_root),
1552 OID_AUTO,
1553 "cwnd",
1554 CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
1555 "Cwnd controls");
1556 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1557 SYSCTL_CHILDREN(bbr_cwnd),
1558 OID_AUTO, "tar_rtt", CTLFLAG_RW,
1559 &bbr_cwndtarget_rtt_touse, 0,
1560 "Target cwnd rtt measurement to use (0=rtt_prop, 1=rtt_rack, 2=pkt_rtt, 3=srtt)?");
1561 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1562 SYSCTL_CHILDREN(bbr_cwnd),
1563 OID_AUTO, "may_shrink", CTLFLAG_RW,
1564 &bbr_cwnd_may_shrink, 0,
1565 "Can the cwnd shrink if it would grow to more than the target?");
1566 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1567 SYSCTL_CHILDREN(bbr_cwnd),
1568 OID_AUTO, "max_target_limit", CTLFLAG_RW,
1569 &bbr_target_cwnd_mult_limit, 8,
1570 "Do we limit the cwnd to some multiple of the cwnd target if cwnd can't shrink 0=no?");
1571 SYSCTL_ADD_U32(&bbr_sysctl_ctx,
1572 SYSCTL_CHILDREN(bbr_cwnd),
1573 OID_AUTO, "highspeed_min", CTLFLAG_RW,
1574 &bbr_cwnd_min_val_hs, BBR_HIGHSPEED_NUM_MSS,
1575 "What is the high-speed min cwnd (rttProp under 1ms)");
1576 SYSCTL_ADD_U32(&bbr_sysctl_ctx,
1577 SYSCTL_CHILDREN(bbr_cwnd),
1578 OID_AUTO, "lowspeed_min", CTLFLAG_RW,
1579 &bbr_cwnd_min_val, BBR_PROBERTT_NUM_MSS,
1580 "What is the min cwnd (rttProp > 1ms)");
1581 SYSCTL_ADD_U32(&bbr_sysctl_ctx,
1582 SYSCTL_CHILDREN(bbr_cwnd),
1583 OID_AUTO, "initwin", CTLFLAG_RW,
1584 &bbr_def_init_win, 10,
1585 "What is the BBR initial window, if 0 use tcp version");
1586 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1587 SYSCTL_CHILDREN(bbr_cwnd),
1588 OID_AUTO, "do_loss_red", CTLFLAG_RW,
1589 &bbr_do_red, 600,
1590 "Do we reduce the b/w at exit from recovery based on ratio of prop/srtt (800=80.0, 0=off)?");
1591 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1592 SYSCTL_CHILDREN(bbr_cwnd),
1593 OID_AUTO, "red_scale", CTLFLAG_RW,
1594 &bbr_red_scale, 20000,
1595 "What RTT do we scale with?");
1596 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1597 SYSCTL_CHILDREN(bbr_cwnd),
1598 OID_AUTO, "red_growslow", CTLFLAG_RW,
1599 &bbr_red_growth_restrict, 1,
1600 "Do we restrict cwnd growth for whats in flight?");
1601 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1602 SYSCTL_CHILDREN(bbr_cwnd),
1603 OID_AUTO, "red_div", CTLFLAG_RW,
1604 &bbr_red_div, 2,
1605 "If we reduce whats the divisor?");
1606 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1607 SYSCTL_CHILDREN(bbr_cwnd),
1608 OID_AUTO, "red_mul", CTLFLAG_RW,
1609 &bbr_red_mul, 1,
1610 "If we reduce whats the mulitiplier?");
1611 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1612 SYSCTL_CHILDREN(bbr_cwnd),
1613 OID_AUTO, "target_is_unit", CTLFLAG_RW,
1614 &bbr_target_is_bbunit, 0,
1615 "Is the state target the pacing_gain or BBR_UNIT?");
1616 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1617 SYSCTL_CHILDREN(bbr_cwnd),
1618 OID_AUTO, "drop_limit", CTLFLAG_RW,
1619 &bbr_drop_limit, 0,
1620 "Number of segments limit for drop (0=use min_cwnd w/flight)?");
1621
1622 /* Timeout controls */
1623 bbr_timeout = SYSCTL_ADD_NODE(&bbr_sysctl_ctx,
1624 SYSCTL_CHILDREN(bbr_sysctl_root),
1625 OID_AUTO,
1626 "timeout",
1627 CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
1628 "Time out controls");
1629 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1630 SYSCTL_CHILDREN(bbr_timeout),
1631 OID_AUTO, "delack", CTLFLAG_RW,
1632 &bbr_delack_time, 100000,
1633 "BBR's delayed ack time");
1634 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1635 SYSCTL_CHILDREN(bbr_timeout),
1636 OID_AUTO, "tlp_uses", CTLFLAG_RW,
1637 &bbr_tlp_type_to_use, 3,
1638 "RTT that TLP uses in its calculations, 0=rttProp, 1=Rack_rtt, 2=pkt_rtt and 3=srtt");
1639 SYSCTL_ADD_U32(&bbr_sysctl_ctx,
1640 SYSCTL_CHILDREN(bbr_timeout),
1641 OID_AUTO, "persmin", CTLFLAG_RW,
1642 &bbr_persist_min, 250000,
1643 "What is the minimum time in microseconds between persists");
1644 SYSCTL_ADD_U32(&bbr_sysctl_ctx,
1645 SYSCTL_CHILDREN(bbr_timeout),
1646 OID_AUTO, "persmax", CTLFLAG_RW,
1647 &bbr_persist_max, 1000000,
1648 "What is the largest delay in microseconds between persists");
1649 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1650 SYSCTL_CHILDREN(bbr_timeout),
1651 OID_AUTO, "tlp_minto", CTLFLAG_RW,
1652 &bbr_tlp_min, 10000,
1653 "TLP Min timeout in usecs");
1654 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1655 SYSCTL_CHILDREN(bbr_timeout),
1656 OID_AUTO, "tlp_dack_time", CTLFLAG_RW,
1657 &bbr_delayed_ack_time, 200000,
1658 "TLP delayed ack compensation value");
1659 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1660 SYSCTL_CHILDREN(bbr_sysctl_root),
1661 OID_AUTO, "minrto", CTLFLAG_RW,
1662 &bbr_rto_min_ms, 30,
1663 "Minimum RTO in ms");
1664 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1665 SYSCTL_CHILDREN(bbr_timeout),
1666 OID_AUTO, "maxrto", CTLFLAG_RW,
1667 &bbr_rto_max_sec, 4,
1668 "Maximum RTO in seconds -- should be at least as large as min_rto");
1669 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1670 SYSCTL_CHILDREN(bbr_timeout),
1671 OID_AUTO, "tlp_retry", CTLFLAG_RW,
1672 &bbr_tlp_max_resend, 2,
1673 "How many times does TLP retry a single segment or multiple with no ACK");
1674 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1675 SYSCTL_CHILDREN(bbr_timeout),
1676 OID_AUTO, "minto", CTLFLAG_RW,
1677 &bbr_min_to, 1000,
1678 "Minimum rack timeout in useconds");
1679 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1680 SYSCTL_CHILDREN(bbr_timeout),
1681 OID_AUTO, "pktdelay", CTLFLAG_RW,
1682 &bbr_pkt_delay, 1000,
1683 "Extra RACK time (in useconds) besides reordering thresh");
1684 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1685 SYSCTL_CHILDREN(bbr_timeout),
1686 OID_AUTO, "incr_tmrs", CTLFLAG_RW,
1687 &bbr_incr_timers, 1,
1688 "Increase the RXT/TLP timer by the pacing time used?");
1689 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1690 SYSCTL_CHILDREN(bbr_timeout),
1691 OID_AUTO, "rxtmark_sackpassed", CTLFLAG_RW,
1692 &bbr_marks_rxt_sack_passed, 0,
1693 "Mark sack passed on all those not ack'd when a RXT hits?");
1694 /* Policer controls */
1695 bbr_policer = SYSCTL_ADD_NODE(&bbr_sysctl_ctx,
1696 SYSCTL_CHILDREN(bbr_sysctl_root),
1697 OID_AUTO,
1698 "policer",
1699 CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
1700 "Policer controls");
1701 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1702 SYSCTL_CHILDREN(bbr_policer),
1703 OID_AUTO, "detect_enable", CTLFLAG_RW,
1704 &bbr_policer_detection_enabled, 1,
1705 "Is policer detection enabled??");
1706 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1707 SYSCTL_CHILDREN(bbr_policer),
1708 OID_AUTO, "min_pes", CTLFLAG_RW,
1709 &bbr_lt_intvl_min_rtts, 4,
1710 "Minimum number of PE's?");
1711 SYSCTL_ADD_U64(&bbr_sysctl_ctx,
1712 SYSCTL_CHILDREN(bbr_policer),
1713 OID_AUTO, "bwdiff", CTLFLAG_RW,
1714 &bbr_lt_bw_diff, (4000/8),
1715 "Minimal bw diff?");
1716 SYSCTL_ADD_U64(&bbr_sysctl_ctx,
1717 SYSCTL_CHILDREN(bbr_policer),
1718 OID_AUTO, "bwratio", CTLFLAG_RW,
1719 &bbr_lt_bw_ratio, 8,
1720 "Minimal bw diff?");
1721 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1722 SYSCTL_CHILDREN(bbr_policer),
1723 OID_AUTO, "from_rack_rxt", CTLFLAG_RW,
1724 &bbr_policer_call_from_rack_to, 0,
1725 "Do we call the policer detection code from a rack-timeout?");
1726 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1727 SYSCTL_CHILDREN(bbr_policer),
1728 OID_AUTO, "false_postive", CTLFLAG_RW,
1729 &bbr_lt_intvl_fp, 0,
1730 "What packet epoch do we do false-postive detection at (0=no)?");
1731 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1732 SYSCTL_CHILDREN(bbr_policer),
1733 OID_AUTO, "loss_thresh", CTLFLAG_RW,
1734 &bbr_lt_loss_thresh, 196,
1735 "Loss threshold 196 = 19.6%?");
1736 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1737 SYSCTL_CHILDREN(bbr_policer),
1738 OID_AUTO, "false_postive_thresh", CTLFLAG_RW,
1739 &bbr_lt_fd_thresh, 100,
1740 "What percentage is the false detection threshold (150=15.0)?");
1741 /* All the rest */
1742 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1743 SYSCTL_CHILDREN(bbr_sysctl_root),
1744 OID_AUTO, "cheat_rxt", CTLFLAG_RW,
1745 &bbr_use_rack_resend_cheat, 0,
1746 "Do we burst 1ms between sends on retransmissions (like rack)?");
1747 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1748 SYSCTL_CHILDREN(bbr_sysctl_root),
1749 OID_AUTO, "error_paceout", CTLFLAG_RW,
1750 &bbr_error_base_paceout, 10000,
1751 "When we hit an error what is the min to pace out in usec's?");
1752 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1753 SYSCTL_CHILDREN(bbr_sysctl_root),
1754 OID_AUTO, "kill_paceout", CTLFLAG_RW,
1755 &bbr_max_net_error_cnt, 10,
1756 "When we hit this many errors in a row, kill the session?");
1757 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1758 SYSCTL_CHILDREN(bbr_sysctl_root),
1759 OID_AUTO, "data_after_close", CTLFLAG_RW,
1760 &bbr_ignore_data_after_close, 1,
1761 "Do we hold off sending a RST until all pending data is ack'd");
1762 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1763 SYSCTL_CHILDREN(bbr_sysctl_root),
1764 OID_AUTO, "resend_use_tso", CTLFLAG_RW,
1765 &bbr_resends_use_tso, 0,
1766 "Can resends use TSO?");
1767 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1768 SYSCTL_CHILDREN(bbr_sysctl_root),
1769 OID_AUTO, "sblklimit", CTLFLAG_RW,
1770 &bbr_sack_block_limit, 128,
1771 "When do we start ignoring small sack blocks");
1772 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1773 SYSCTL_CHILDREN(bbr_sysctl_root),
1774 OID_AUTO, "bb_verbose", CTLFLAG_RW,
1775 &bbr_verbose_logging, 0,
1776 "Should BBR black box logging be verbose");
1777 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1778 SYSCTL_CHILDREN(bbr_sysctl_root),
1779 OID_AUTO, "reorder_thresh", CTLFLAG_RW,
1780 &bbr_reorder_thresh, 2,
1781 "What factor for rack will be added when seeing reordering (shift right)");
1782 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1783 SYSCTL_CHILDREN(bbr_sysctl_root),
1784 OID_AUTO, "reorder_fade", CTLFLAG_RW,
1785 &bbr_reorder_fade, 0,
1786 "Does reorder detection fade, if so how many ms (0 means never)");
1787 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1788 SYSCTL_CHILDREN(bbr_sysctl_root),
1789 OID_AUTO, "rtt_tlp_thresh", CTLFLAG_RW,
1790 &bbr_tlp_thresh, 1,
1791 "what divisor for TLP rtt/retran will be added (1=rtt, 2=1/2 rtt etc)");
1792 /* Stats and counters */
1793 /* The pacing counters for hdwr/software can't be in the array */
1794 bbr_nohdwr_pacing_enobuf = counter_u64_alloc(M_WAITOK);
1795 bbr_hdwr_pacing_enobuf = counter_u64_alloc(M_WAITOK);
1796 SYSCTL_ADD_COUNTER_U64(&bbr_sysctl_ctx,
1797 SYSCTL_CHILDREN(bbr_sysctl_root),
1798 OID_AUTO, "enob_hdwr_pacing", CTLFLAG_RD,
1799 &bbr_hdwr_pacing_enobuf,
1800 "Total number of enobufs for hardware paced flows");
1801 SYSCTL_ADD_COUNTER_U64(&bbr_sysctl_ctx,
1802 SYSCTL_CHILDREN(bbr_sysctl_root),
1803 OID_AUTO, "enob_no_hdwr_pacing", CTLFLAG_RD,
1804 &bbr_nohdwr_pacing_enobuf,
1805 "Total number of enobufs for non-hardware paced flows");
1806
1807 bbr_flows_whdwr_pacing = counter_u64_alloc(M_WAITOK);
1808 SYSCTL_ADD_COUNTER_U64(&bbr_sysctl_ctx,
1809 SYSCTL_CHILDREN(bbr_sysctl_root),
1810 OID_AUTO, "hdwr_pacing", CTLFLAG_RD,
1811 &bbr_flows_whdwr_pacing,
1812 "Total number of hardware paced flows");
1813 bbr_flows_nohdwr_pacing = counter_u64_alloc(M_WAITOK);
1814 SYSCTL_ADD_COUNTER_U64(&bbr_sysctl_ctx,
1815 SYSCTL_CHILDREN(bbr_sysctl_root),
1816 OID_AUTO, "software_pacing", CTLFLAG_RD,
1817 &bbr_flows_nohdwr_pacing,
1818 "Total number of software paced flows");
1819 COUNTER_ARRAY_ALLOC(bbr_stat_arry, BBR_STAT_SIZE, M_WAITOK);
1820 SYSCTL_ADD_COUNTER_U64_ARRAY(&bbr_sysctl_ctx, SYSCTL_CHILDREN(bbr_sysctl_root),
1821 OID_AUTO, "stats", CTLFLAG_RD,
1822 bbr_stat_arry, BBR_STAT_SIZE, "BBR Stats");
1823 COUNTER_ARRAY_ALLOC(bbr_opts_arry, BBR_OPTS_SIZE, M_WAITOK);
1824 SYSCTL_ADD_COUNTER_U64_ARRAY(&bbr_sysctl_ctx, SYSCTL_CHILDREN(bbr_sysctl_root),
1825 OID_AUTO, "opts", CTLFLAG_RD,
1826 bbr_opts_arry, BBR_OPTS_SIZE, "BBR Option Stats");
1827 COUNTER_ARRAY_ALLOC(bbr_state_lost, BBR_MAX_STAT, M_WAITOK);
1828 SYSCTL_ADD_COUNTER_U64_ARRAY(&bbr_sysctl_ctx, SYSCTL_CHILDREN(bbr_sysctl_root),
1829 OID_AUTO, "lost", CTLFLAG_RD,
1830 bbr_state_lost, BBR_MAX_STAT, "Stats of when losses occur");
1831 COUNTER_ARRAY_ALLOC(bbr_state_resend, BBR_MAX_STAT, M_WAITOK);
1832 SYSCTL_ADD_COUNTER_U64_ARRAY(&bbr_sysctl_ctx, SYSCTL_CHILDREN(bbr_sysctl_root),
1833 OID_AUTO, "stateresend", CTLFLAG_RD,
1834 bbr_state_resend, BBR_MAX_STAT, "Stats of what states resend");
1835 COUNTER_ARRAY_ALLOC(bbr_state_time, BBR_MAX_STAT, M_WAITOK);
1836 SYSCTL_ADD_COUNTER_U64_ARRAY(&bbr_sysctl_ctx, SYSCTL_CHILDREN(bbr_sysctl_root),
1837 OID_AUTO, "statetime", CTLFLAG_RD,
1838 bbr_state_time, BBR_MAX_STAT, "Stats of time spent in the states");
1839 COUNTER_ARRAY_ALLOC(bbr_out_size, TCP_MSS_ACCT_SIZE, M_WAITOK);
1840 SYSCTL_ADD_COUNTER_U64_ARRAY(&bbr_sysctl_ctx, SYSCTL_CHILDREN(bbr_sysctl_root),
1841 OID_AUTO, "outsize", CTLFLAG_RD,
1842 bbr_out_size, TCP_MSS_ACCT_SIZE, "Size of output calls");
1843 SYSCTL_ADD_PROC(&bbr_sysctl_ctx,
1844 SYSCTL_CHILDREN(bbr_sysctl_root),
1845 OID_AUTO, "clrlost", CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE,
1846 &bbr_clear_lost, 0, sysctl_bbr_clear_lost, "IU", "Clear lost counters");
1847}
1848
1849static void
1850bbr_counter_destroy(void)
1851{
1852 COUNTER_ARRAY_FREE(bbr_stat_arry, BBR_STAT_SIZE);
1853 COUNTER_ARRAY_FREE(bbr_opts_arry, BBR_OPTS_SIZE);
1854 COUNTER_ARRAY_FREE(bbr_out_size, TCP_MSS_ACCT_SIZE);
1855 COUNTER_ARRAY_FREE(bbr_state_lost, BBR_MAX_STAT);
1856 COUNTER_ARRAY_FREE(bbr_state_time, BBR_MAX_STAT);
1857 COUNTER_ARRAY_FREE(bbr_state_resend, BBR_MAX_STAT);
1858 counter_u64_free(bbr_nohdwr_pacing_enobuf);
1859 counter_u64_free(bbr_hdwr_pacing_enobuf);
1860 counter_u64_free(bbr_flows_whdwr_pacing);
1861 counter_u64_free(bbr_flows_nohdwr_pacing);
1862
1863}
1864
1865static __inline void
1866bbr_fill_in_logging_data(struct tcp_bbr *bbr, struct tcp_log_bbr *l, uint32_t cts)
1867{
1868 memset(l, 0, sizeof(union tcp_log_stackspecific));
1869 l->cur_del_rate = bbr->r_ctl.rc_bbr_cur_del_rate;
1870 l->delRate = get_filter_value(&bbr->r_ctl.rc_delrate);
1871 l->rttProp = get_filter_value_small(&bbr->r_ctl.rc_rttprop);
1872 l->bw_inuse = bbr_get_bw(bbr);
1873 l->inflight = ctf_flight_size(bbr->rc_tp,
1874 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
1875 l->applimited = bbr->r_ctl.r_app_limited_until;
1876 l->delivered = bbr->r_ctl.rc_delivered;
1877 l->timeStamp = cts;
1878 l->lost = bbr->r_ctl.rc_lost;
1879 l->bbr_state = bbr->rc_bbr_state;
1880 l->bbr_substate = bbr_state_val(bbr);
1881 l->epoch = bbr->r_ctl.rc_rtt_epoch;
1882 l->lt_epoch = bbr->r_ctl.rc_lt_epoch;
1883 l->pacing_gain = bbr->r_ctl.rc_bbr_hptsi_gain;
1884 l->cwnd_gain = bbr->r_ctl.rc_bbr_cwnd_gain;
1885 l->inhpts = tcp_in_hpts(bbr->rc_inp);
1886 l->use_lt_bw = bbr->rc_lt_use_bw;
1887 l->pkts_out = bbr->r_ctl.rc_flight_at_input;
1888 l->pkt_epoch = bbr->r_ctl.rc_pkt_epoch;
1889}
1890
1891static void
1892bbr_log_type_bw_reduce(struct tcp_bbr *bbr, int reason)
1893{
1894 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
1895 union tcp_log_stackspecific log;
1896
1897 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);
1898 log.u_bbr.flex1 = 0;
1899 log.u_bbr.flex2 = 0;
1900 log.u_bbr.flex5 = 0;
1901 log.u_bbr.flex3 = 0;
1902 log.u_bbr.flex4 = bbr->r_ctl.rc_pkt_epoch_loss_rate;
1903 log.u_bbr.flex7 = reason;
1904 log.u_bbr.flex6 = bbr->r_ctl.rc_bbr_enters_probertt;
1905 log.u_bbr.flex8 = 0;
1906 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
1907 &bbr->rc_inp->inp_socket->so_rcv,
1908 &bbr->rc_inp->inp_socket->so_snd,
1909 BBR_LOG_BW_RED_EV, 0,
1910 0, &log, false, &bbr->rc_tv);
1911 }
1912}
1913
1914static void
1915bbr_log_type_rwnd_collapse(struct tcp_bbr *bbr, int seq, int mode, uint32_t count)
1916{
1917 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
1918 union tcp_log_stackspecific log;
1919
1920 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);
1921 log.u_bbr.flex1 = seq;
1922 log.u_bbr.flex2 = count;
1923 log.u_bbr.flex8 = mode;
1924 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
1925 &bbr->rc_inp->inp_socket->so_rcv,
1926 &bbr->rc_inp->inp_socket->so_snd,
1927 BBR_LOG_LOWGAIN, 0,
1928 0, &log, false, &bbr->rc_tv);
1929 }
1930}
1931
1932static void
1933bbr_log_type_just_return(struct tcp_bbr *bbr, uint32_t cts, uint32_t tlen, uint8_t hpts_calling,
1934 uint8_t reason, uint32_t p_maxseg, int len)
1935{
1936 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
1937 union tcp_log_stackspecific log;
1938
1939 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
1940 log.u_bbr.flex1 = p_maxseg;
1941 log.u_bbr.flex2 = bbr->r_ctl.rc_hpts_flags;
1942 log.u_bbr.flex3 = bbr->r_ctl.rc_timer_exp;
1943 log.u_bbr.flex4 = reason;
1944 log.u_bbr.flex5 = bbr->rc_in_persist;
1945 log.u_bbr.flex6 = bbr->r_ctl.rc_last_delay_val;
1946 log.u_bbr.flex7 = p_maxseg;
1947 log.u_bbr.flex8 = bbr->rc_in_persist;
1948 log.u_bbr.pkts_out = 0;
1949 log.u_bbr.applimited = len;
1950 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
1951 &bbr->rc_inp->inp_socket->so_rcv,
1952 &bbr->rc_inp->inp_socket->so_snd,
1953 BBR_LOG_JUSTRET, 0,
1954 tlen, &log, false, &bbr->rc_tv);
1955 }
1956}
1957
1958static void
1959bbr_log_type_enter_rec(struct tcp_bbr *bbr, uint32_t seq)
1960{
1961 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
1962 union tcp_log_stackspecific log;
1963
1964 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);
1965 log.u_bbr.flex1 = seq;
1966 log.u_bbr.flex2 = bbr->r_ctl.rc_cwnd_on_ent;
1967 log.u_bbr.flex3 = bbr->r_ctl.rc_recovery_start;
1968 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
1969 &bbr->rc_inp->inp_socket->so_rcv,
1970 &bbr->rc_inp->inp_socket->so_snd,
1971 BBR_LOG_ENTREC, 0,
1972 0, &log, false, &bbr->rc_tv);
1973 }
1974}
1975
1976static void
1977bbr_log_msgsize_fail(struct tcp_bbr *bbr, struct tcpcb *tp, uint32_t len, uint32_t maxseg, uint32_t mtu, int32_t csum_flags, int32_t tso, uint32_t cts)
1978{
1979 if (tp->t_logstate != TCP_LOG_STATE_OFF) {
1980 union tcp_log_stackspecific log;
1981
1982 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
1983 log.u_bbr.flex1 = tso;
1984 log.u_bbr.flex2 = maxseg;
1985 log.u_bbr.flex3 = mtu;
1986 log.u_bbr.flex4 = csum_flags;
1987 TCP_LOG_EVENTP(tp, NULL,
1988 &bbr->rc_inp->inp_socket->so_rcv,
1989 &bbr->rc_inp->inp_socket->so_snd,
1990 BBR_LOG_MSGSIZE, 0,
1991 0, &log, false, &bbr->rc_tv);
1992 }
1993}
1994
1995static void
1996bbr_log_flowend(struct tcp_bbr *bbr)
1997{
1998 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
1999 union tcp_log_stackspecific log;
2000 struct sockbuf *r, *s;
2001 struct timeval tv;
2002
2003 if (bbr->rc_inp->inp_socket) {
2004 r = &bbr->rc_inp->inp_socket->so_rcv;
2005 s = &bbr->rc_inp->inp_socket->so_snd;
2006 } else {
2007 r = s = NULL;
2008 }
2009 bbr_fill_in_logging_data(bbr, &log.u_bbr, tcp_get_usecs(&tv));
2010 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2011 r, s,
2012 TCP_LOG_FLOWEND, 0,
2013 0, &log, false, &tv);
2014 }
2015}
2016
2017static void
2018bbr_log_pkt_epoch(struct tcp_bbr *bbr, uint32_t cts, uint32_t line,
2019 uint32_t lost, uint32_t del)
2020{
2021 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2022 union tcp_log_stackspecific log;
2023
2024 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2025 log.u_bbr.flex1 = lost;
2026 log.u_bbr.flex2 = del;
2027 log.u_bbr.flex3 = bbr->r_ctl.rc_bbr_lastbtlbw;
2028 log.u_bbr.flex4 = bbr->r_ctl.rc_pkt_epoch_rtt;
2029 log.u_bbr.flex5 = bbr->r_ctl.rc_bbr_last_startup_epoch;
2030 log.u_bbr.flex6 = bbr->r_ctl.rc_lost_at_startup;
2031 log.u_bbr.flex7 = line;
2032 log.u_bbr.flex8 = 0;
2033 log.u_bbr.inflight = bbr->r_ctl.r_measurement_count;
2034 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2035 &bbr->rc_inp->inp_socket->so_rcv,
2036 &bbr->rc_inp->inp_socket->so_snd,
2037 BBR_LOG_PKT_EPOCH, 0,
2038 0, &log, false, &bbr->rc_tv);
2039 }
2040}
2041
2042static void
2043bbr_log_time_epoch(struct tcp_bbr *bbr, uint32_t cts, uint32_t line, uint32_t epoch_time)
2044{
2045 if (bbr_verbose_logging && (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF)) {
2046 union tcp_log_stackspecific log;
2047
2048 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2049 log.u_bbr.flex1 = bbr->r_ctl.rc_lost;
2050 log.u_bbr.flex2 = bbr->rc_inp->inp_socket->so_snd.sb_lowat;
2051 log.u_bbr.flex3 = bbr->rc_inp->inp_socket->so_snd.sb_hiwat;
2052 log.u_bbr.flex7 = line;
2053 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2054 &bbr->rc_inp->inp_socket->so_rcv,
2055 &bbr->rc_inp->inp_socket->so_snd,
2056 BBR_LOG_TIME_EPOCH, 0,
2057 0, &log, false, &bbr->rc_tv);
2058 }
2059}
2060
2061static void
2062bbr_log_set_of_state_target(struct tcp_bbr *bbr, uint32_t new_tar, int line, int meth)
2063{
2064 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2065 union tcp_log_stackspecific log;
2066
2067 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);
2068 log.u_bbr.flex1 = bbr->r_ctl.rc_target_at_state;
2069 log.u_bbr.flex2 = new_tar;
2070 log.u_bbr.flex3 = line;
2071 log.u_bbr.flex4 = bbr->r_ctl.rc_pace_max_segs;
2072 log.u_bbr.flex5 = bbr_quanta;
2073 log.u_bbr.flex6 = bbr->r_ctl.rc_pace_min_segs;
2074 log.u_bbr.flex7 = bbr->rc_last_options;
2075 log.u_bbr.flex8 = meth;
2076 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2077 &bbr->rc_inp->inp_socket->so_rcv,
2078 &bbr->rc_inp->inp_socket->so_snd,
2079 BBR_LOG_STATE_TARGET, 0,
2080 0, &log, false, &bbr->rc_tv);
2081 }
2082
2083}
2084
2085static void
2086bbr_log_type_statechange(struct tcp_bbr *bbr, uint32_t cts, int32_t line)
2087{
2088 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2089 union tcp_log_stackspecific log;
2090
2091 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2092 log.u_bbr.flex1 = line;
2093 log.u_bbr.flex2 = bbr->r_ctl.rc_rtt_shrinks;
2094 log.u_bbr.flex3 = bbr->r_ctl.rc_probertt_int;
2095 if (bbr_state_is_pkt_epoch)
2096 log.u_bbr.flex4 = bbr_get_rtt(bbr, BBR_RTT_PKTRTT);
2097 else
2098 log.u_bbr.flex4 = bbr_get_rtt(bbr, BBR_RTT_PROP);
2099 log.u_bbr.flex5 = bbr->r_ctl.rc_bbr_last_startup_epoch;
2100 log.u_bbr.flex6 = bbr->r_ctl.rc_lost_at_startup;
2101 log.u_bbr.flex7 = (bbr->r_ctl.rc_target_at_state/1000);
2102 log.u_bbr.lt_epoch = bbr->r_ctl.rc_level_state_extra;
2103 log.u_bbr.pkts_out = bbr->r_ctl.rc_target_at_state;
2104 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2105 &bbr->rc_inp->inp_socket->so_rcv,
2106 &bbr->rc_inp->inp_socket->so_snd,
2107 BBR_LOG_STATE, 0,
2108 0, &log, false, &bbr->rc_tv);
2109 }
2110}
2111
2112static void
2113bbr_log_rtt_shrinks(struct tcp_bbr *bbr, uint32_t cts, uint32_t applied,
2114 uint32_t rtt, uint32_t line, uint8_t reas, uint16_t cond)
2115{
2116 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2117 union tcp_log_stackspecific log;
2118
2119 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2120 log.u_bbr.flex1 = line;
2121 log.u_bbr.flex2 = bbr->r_ctl.rc_rtt_shrinks;
2122 log.u_bbr.flex3 = bbr->r_ctl.last_in_probertt;
2123 log.u_bbr.flex4 = applied;
2124 log.u_bbr.flex5 = rtt;
2125 log.u_bbr.flex6 = bbr->r_ctl.rc_target_at_state;
2126 log.u_bbr.flex7 = cond;
2127 log.u_bbr.flex8 = reas;
2128 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2129 &bbr->rc_inp->inp_socket->so_rcv,
2130 &bbr->rc_inp->inp_socket->so_snd,
2131 BBR_LOG_RTT_SHRINKS, 0,
2132 0, &log, false, &bbr->rc_tv);
2133 }
2134}
2135
2136static void
2137bbr_log_type_exit_rec(struct tcp_bbr *bbr)
2138{
2139 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2140 union tcp_log_stackspecific log;
2141
2142 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);
2143 log.u_bbr.flex1 = bbr->r_ctl.rc_recovery_start;
2144 log.u_bbr.flex2 = bbr->r_ctl.rc_cwnd_on_ent;
2145 log.u_bbr.flex5 = bbr->r_ctl.rc_target_at_state;
2146 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2147 &bbr->rc_inp->inp_socket->so_rcv,
2148 &bbr->rc_inp->inp_socket->so_snd,
2149 BBR_LOG_EXITREC, 0,
2150 0, &log, false, &bbr->rc_tv);
2151 }
2152}
2153
2154static void
2155bbr_log_type_cwndupd(struct tcp_bbr *bbr, uint32_t bytes_this_ack, uint32_t chg,
2156 uint32_t prev_acked, int32_t meth, uint32_t target, uint32_t th_ack, int32_t line)
2157{
2158 if (bbr_verbose_logging && (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF)) {
2159 union tcp_log_stackspecific log;
2160
2161 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);
2162 log.u_bbr.flex1 = line;
2163 log.u_bbr.flex2 = prev_acked;
2164 log.u_bbr.flex3 = bytes_this_ack;
2165 log.u_bbr.flex4 = chg;
2166 log.u_bbr.flex5 = th_ack;
2167 log.u_bbr.flex6 = target;
2168 log.u_bbr.flex8 = meth;
2169 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2170 &bbr->rc_inp->inp_socket->so_rcv,
2171 &bbr->rc_inp->inp_socket->so_snd,
2172 BBR_LOG_CWND, 0,
2173 0, &log, false, &bbr->rc_tv);
2174 }
2175}
2176
2177static void
2178bbr_log_rtt_sample(struct tcp_bbr *bbr, uint32_t rtt, uint32_t tsin)
2179{
2180 /*
2181 * Log the rtt sample we are applying to the srtt algorithm in
2182 * useconds.
2183 */
2184 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2185 union tcp_log_stackspecific log;
2186
2187 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);
2188 log.u_bbr.flex1 = rtt;
2189 log.u_bbr.flex2 = bbr->r_ctl.rc_bbr_state_time;
2190 log.u_bbr.flex3 = bbr->r_ctl.rc_ack_hdwr_delay;
2191 log.u_bbr.flex4 = bbr->rc_tp->ts_offset;
2192 log.u_bbr.flex5 = bbr->r_ctl.rc_target_at_state;
2193 log.u_bbr.pkts_out = tcp_tv_to_mssectick(&bbr->rc_tv);
2194 log.u_bbr.flex6 = tsin;
2195 log.u_bbr.flex7 = 0;
2196 log.u_bbr.flex8 = bbr->rc_ack_was_delayed;
2197 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2198 &bbr->rc_inp->inp_socket->so_rcv,
2199 &bbr->rc_inp->inp_socket->so_snd,
2200 TCP_LOG_RTT, 0,
2201 0, &log, false, &bbr->rc_tv);
2202 }
2203}
2204
2205static void
2206bbr_log_type_pesist(struct tcp_bbr *bbr, uint32_t cts, uint32_t time_in, int32_t line, uint8_t enter_exit)
2207{
2208 if (bbr_verbose_logging && (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF)) {
2209 union tcp_log_stackspecific log;
2210
2211 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2212 log.u_bbr.flex1 = time_in;
2213 log.u_bbr.flex2 = line;
2214 log.u_bbr.flex8 = enter_exit;
2215 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2216 &bbr->rc_inp->inp_socket->so_rcv,
2217 &bbr->rc_inp->inp_socket->so_snd,
2218 BBR_LOG_PERSIST, 0,
2219 0, &log, false, &bbr->rc_tv);
2220 }
2221}
2222static void
2223bbr_log_ack_clear(struct tcp_bbr *bbr, uint32_t cts)
2224{
2225 if (bbr_verbose_logging && (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF)) {
2226 union tcp_log_stackspecific log;
2227
2228 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2229 log.u_bbr.flex1 = bbr->rc_tp->ts_recent_age;
2230 log.u_bbr.flex2 = bbr->r_ctl.rc_rtt_shrinks;
2231 log.u_bbr.flex3 = bbr->r_ctl.rc_probertt_int;
2232 log.u_bbr.flex4 = bbr->r_ctl.rc_went_idle_time;
2233 log.u_bbr.flex5 = bbr->r_ctl.rc_target_at_state;
2234 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2235 &bbr->rc_inp->inp_socket->so_rcv,
2236 &bbr->rc_inp->inp_socket->so_snd,
2237 BBR_LOG_ACKCLEAR, 0,
2238 0, &log, false, &bbr->rc_tv);
2239 }
2240}
2241
2242static void
2243bbr_log_ack_event(struct tcp_bbr *bbr, struct tcphdr *th, struct tcpopt *to, uint32_t tlen,
2244 uint16_t nsegs, uint32_t cts, int32_t nxt_pkt, struct mbuf *m)
2245{
2246 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2247 union tcp_log_stackspecific log;
2248 struct timeval tv;
2249
2250 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2251 log.u_bbr.flex1 = nsegs;
2252 log.u_bbr.flex2 = bbr->r_ctl.rc_lost_bytes;
2253 if (m) {
2254 struct timespec ts;
2255
2256 log.u_bbr.flex3 = m->m_flags;
2257 if (m->m_flags & M_TSTMP) {
2258 mbuf_tstmp2timespec(m, &ts);
2259 tv.tv_sec = ts.tv_sec;
2260 tv.tv_usec = ts.tv_nsec / 1000;
2261 log.u_bbr.lt_epoch = tcp_tv_to_usectick(&tv);
2262 } else {
2263 log.u_bbr.lt_epoch = 0;
2264 }
2265 if (m->m_flags & M_TSTMP_LRO) {
2266 tv.tv_sec = m->m_pkthdr.rcv_tstmp / 1000000000;
2267 tv.tv_usec = (m->m_pkthdr.rcv_tstmp % 1000000000) / 1000;
2268 log.u_bbr.flex5 = tcp_tv_to_usectick(&tv);
2269 } else {
2270 /* No arrival timestamp */
2271 log.u_bbr.flex5 = 0;
2272 }
2273
2274 log.u_bbr.pkts_out = tcp_get_usecs(&tv);
2275 } else {
2276 log.u_bbr.flex3 = 0;
2277 log.u_bbr.flex5 = 0;
2278 log.u_bbr.flex6 = 0;
2279 log.u_bbr.pkts_out = 0;
2280 }
2281 log.u_bbr.flex4 = bbr->r_ctl.rc_target_at_state;
2282 log.u_bbr.flex7 = bbr->r_wanted_output;
2283 log.u_bbr.flex8 = bbr->rc_in_persist;
2284 TCP_LOG_EVENTP(bbr->rc_tp, th,
2285 &bbr->rc_inp->inp_socket->so_rcv,
2286 &bbr->rc_inp->inp_socket->so_snd,
2287 TCP_LOG_IN, 0,
2288 tlen, &log, true, &bbr->rc_tv);
2289 }
2290}
2291
2292static void
2293bbr_log_doseg_done(struct tcp_bbr *bbr, uint32_t cts, int32_t nxt_pkt, int32_t did_out)
2294{
2295 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2296 union tcp_log_stackspecific log;
2297
2298 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2299 log.u_bbr.flex1 = did_out;
2300 log.u_bbr.flex2 = nxt_pkt;
2301 log.u_bbr.flex3 = bbr->r_ctl.rc_last_delay_val;
2302 log.u_bbr.flex4 = bbr->r_ctl.rc_hpts_flags;
2303 log.u_bbr.flex5 = bbr->r_ctl.rc_timer_exp;
2304 log.u_bbr.flex6 = bbr->r_ctl.rc_lost_bytes;
2305 log.u_bbr.flex7 = bbr->r_wanted_output;
2306 log.u_bbr.flex8 = bbr->rc_in_persist;
2307 log.u_bbr.pkts_out = bbr->r_ctl.highest_hdwr_delay;
2308 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2309 &bbr->rc_inp->inp_socket->so_rcv,
2310 &bbr->rc_inp->inp_socket->so_snd,
2311 BBR_LOG_DOSEG_DONE, 0,
2312 0, &log, true, &bbr->rc_tv);
2313 }
2314}
2315
2316static void
2317bbr_log_enobuf_jmp(struct tcp_bbr *bbr, uint32_t len, uint32_t cts,
2318 int32_t line, uint32_t o_len, uint32_t segcnt, uint32_t segsiz)
2319{
2320 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2321 union tcp_log_stackspecific log;
2322
2323 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2324 log.u_bbr.flex1 = line;
2325 log.u_bbr.flex2 = o_len;
2326 log.u_bbr.flex3 = segcnt;
2327 log.u_bbr.flex4 = segsiz;
2328 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2329 &bbr->rc_inp->inp_socket->so_rcv,
2330 &bbr->rc_inp->inp_socket->so_snd,
2331 BBR_LOG_ENOBUF_JMP, ENOBUFS,
2332 len, &log, true, &bbr->rc_tv);
2333 }
2334}
2335
2336static void
2337bbr_log_to_processing(struct tcp_bbr *bbr, uint32_t cts, int32_t ret, int32_t timers, uint8_t hpts_calling)
2338{
2339 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2340 union tcp_log_stackspecific log;
2341
2342 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2343 log.u_bbr.flex1 = timers;
2344 log.u_bbr.flex2 = ret;
2345 log.u_bbr.flex3 = bbr->r_ctl.rc_timer_exp;
2346 log.u_bbr.flex4 = bbr->r_ctl.rc_hpts_flags;
2347 log.u_bbr.flex5 = cts;
2348 log.u_bbr.flex6 = bbr->r_ctl.rc_target_at_state;
2349 log.u_bbr.flex8 = hpts_calling;
2350 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2351 &bbr->rc_inp->inp_socket->so_rcv,
2352 &bbr->rc_inp->inp_socket->so_snd,
2353 BBR_LOG_TO_PROCESS, 0,
2354 0, &log, false, &bbr->rc_tv);
2355 }
2356}
2357
2358static void
2359bbr_log_to_event(struct tcp_bbr *bbr, uint32_t cts, int32_t to_num)
2360{
2361 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2362 union tcp_log_stackspecific log;
2363 uint64_t ar;
2364
2365 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2366 log.u_bbr.flex1 = bbr->bbr_timer_src;
2367 log.u_bbr.flex2 = 0;
2368 log.u_bbr.flex3 = bbr->r_ctl.rc_hpts_flags;
2369 ar = (uint64_t)(bbr->r_ctl.rc_resend);
2370 ar >>= 32;
2371 ar &= 0x00000000ffffffff;
2372 log.u_bbr.flex4 = (uint32_t)ar;
2373 ar = (uint64_t)bbr->r_ctl.rc_resend;
2374 ar &= 0x00000000ffffffff;
2375 log.u_bbr.flex5 = (uint32_t)ar;
2376 log.u_bbr.flex6 = TICKS_2_USEC(bbr->rc_tp->t_rxtcur);
2377 log.u_bbr.flex8 = to_num;
2378 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2379 &bbr->rc_inp->inp_socket->so_rcv,
2380 &bbr->rc_inp->inp_socket->so_snd,
2381 BBR_LOG_RTO, 0,
2382 0, &log, false, &bbr->rc_tv);
2383 }
2384}
2385
2386static void
2387bbr_log_startup_event(struct tcp_bbr *bbr, uint32_t cts, uint32_t flex1, uint32_t flex2, uint32_t flex3, uint8_t reason)
2388{
2389 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2390 union tcp_log_stackspecific log;
2391
2392 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2393 log.u_bbr.flex1 = flex1;
2394 log.u_bbr.flex2 = flex2;
2395 log.u_bbr.flex3 = flex3;
2396 log.u_bbr.flex4 = 0;
2397 log.u_bbr.flex5 = bbr->r_ctl.rc_target_at_state;
2398 log.u_bbr.flex6 = bbr->r_ctl.rc_lost_at_startup;
2399 log.u_bbr.flex8 = reason;
2400 log.u_bbr.cur_del_rate = bbr->r_ctl.rc_bbr_lastbtlbw;
2401 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2402 &bbr->rc_inp->inp_socket->so_rcv,
2403 &bbr->rc_inp->inp_socket->so_snd,
2404 BBR_LOG_REDUCE, 0,
2405 0, &log, false, &bbr->rc_tv);
2406 }
2407}
2408
2409static void
2410bbr_log_hpts_diag(struct tcp_bbr *bbr, uint32_t cts, struct hpts_diag *diag)
2411{
2412 if (bbr_verbose_logging && (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF)) {
2413 union tcp_log_stackspecific log;
2414
2415 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2416 log.u_bbr.flex1 = diag->p_nxt_slot;
2417 log.u_bbr.flex2 = diag->p_cur_slot;
2418 log.u_bbr.flex3 = diag->slot_req;
2419 log.u_bbr.flex4 = diag->inp_hptsslot;
2420 log.u_bbr.flex5 = diag->slot_remaining;
2421 log.u_bbr.flex6 = diag->need_new_to;
2422 log.u_bbr.flex7 = diag->p_hpts_active;
2423 log.u_bbr.flex8 = diag->p_on_min_sleep;
2424 /* Hijack other fields as needed */
2425 log.u_bbr.epoch = diag->have_slept;
2426 log.u_bbr.lt_epoch = diag->yet_to_sleep;
2427 log.u_bbr.pkts_out = diag->co_ret;
2428 log.u_bbr.applimited = diag->hpts_sleep_time;
2429 log.u_bbr.delivered = diag->p_prev_slot;
2430 log.u_bbr.inflight = diag->p_runningslot;
2431 log.u_bbr.bw_inuse = diag->wheel_slot;
2432 log.u_bbr.rttProp = diag->wheel_cts;
2433 log.u_bbr.delRate = diag->maxslots;
2434 log.u_bbr.cur_del_rate = diag->p_curtick;
2435 log.u_bbr.cur_del_rate <<= 32;
2436 log.u_bbr.cur_del_rate |= diag->p_lasttick;
2437 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2438 &bbr->rc_inp->inp_socket->so_rcv,
2439 &bbr->rc_inp->inp_socket->so_snd,
2440 BBR_LOG_HPTSDIAG, 0,
2441 0, &log, false, &bbr->rc_tv);
2442 }
2443}
2444
2445static void
2446bbr_log_timer_var(struct tcp_bbr *bbr, int mode, uint32_t cts, uint32_t time_since_sent, uint32_t srtt,
2447 uint32_t thresh, uint32_t to)
2448{
2449 if (bbr_verbose_logging && (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF)) {
2450 union tcp_log_stackspecific log;
2451
2452 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2453 log.u_bbr.flex1 = bbr->rc_tp->t_rttvar;
2454 log.u_bbr.flex2 = time_since_sent;
2455 log.u_bbr.flex3 = srtt;
2456 log.u_bbr.flex4 = thresh;
2457 log.u_bbr.flex5 = to;
2458 log.u_bbr.flex6 = bbr->rc_tp->t_srtt;
2459 log.u_bbr.flex8 = mode;
2460 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2461 &bbr->rc_inp->inp_socket->so_rcv,
2462 &bbr->rc_inp->inp_socket->so_snd,
2463 BBR_LOG_TIMERPREP, 0,
2464 0, &log, false, &bbr->rc_tv);
2465 }
2466}
2467
2468static void
2469bbr_log_pacing_delay_calc(struct tcp_bbr *bbr, uint16_t gain, uint32_t len,
2470 uint32_t cts, uint32_t usecs, uint64_t bw, uint32_t override, int mod)
2471{
2472 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2473 union tcp_log_stackspecific log;
2474
2475 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2476 log.u_bbr.flex1 = usecs;
2477 log.u_bbr.flex2 = len;
2478 log.u_bbr.flex3 = (uint32_t)((bw >> 32) & 0x00000000ffffffff);
2479 log.u_bbr.flex4 = (uint32_t)(bw & 0x00000000ffffffff);
2480 if (override)
2481 log.u_bbr.flex5 = (1 << 2);
2482 else
2483 log.u_bbr.flex5 = 0;
2484 log.u_bbr.flex6 = override;
2485 log.u_bbr.flex7 = gain;
2486 log.u_bbr.flex8 = mod;
2487 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2488 &bbr->rc_inp->inp_socket->so_rcv,
2489 &bbr->rc_inp->inp_socket->so_snd,
2490 BBR_LOG_HPTSI_CALC, 0,
2491 len, &log, false, &bbr->rc_tv);
2492 }
2493}
2494
2495static void
2496bbr_log_to_start(struct tcp_bbr *bbr, uint32_t cts, uint32_t to, int32_t slot, uint8_t which)
2497{
2498 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2499 union tcp_log_stackspecific log;
2500
2501 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2502
2503 log.u_bbr.flex1 = bbr->bbr_timer_src;
2504 log.u_bbr.flex2 = to;
2505 log.u_bbr.flex3 = bbr->r_ctl.rc_hpts_flags;
2506 log.u_bbr.flex4 = slot;
2507 log.u_bbr.flex5 = bbr->rc_inp->inp_hptsslot;
2508 log.u_bbr.flex6 = TICKS_2_USEC(bbr->rc_tp->t_rxtcur);
2509 log.u_bbr.pkts_out = bbr->rc_inp->inp_flags2;
2510 log.u_bbr.flex8 = which;
2511 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2512 &bbr->rc_inp->inp_socket->so_rcv,
2513 &bbr->rc_inp->inp_socket->so_snd,
2514 BBR_LOG_TIMERSTAR, 0,
2515 0, &log, false, &bbr->rc_tv);
2516 }
2517}
2518
2519static void
2520bbr_log_thresh_choice(struct tcp_bbr *bbr, uint32_t cts, uint32_t thresh, uint32_t lro, uint32_t srtt, struct bbr_sendmap *rsm, uint8_t frm)
2521{
2522 if (bbr_verbose_logging && (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF)) {
2523 union tcp_log_stackspecific log;
2524
2525 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2526 log.u_bbr.flex1 = thresh;
2527 log.u_bbr.flex2 = lro;
2528 log.u_bbr.flex3 = bbr->r_ctl.rc_reorder_ts;
2529 log.u_bbr.flex4 = rsm->r_tim_lastsent[(rsm->r_rtr_cnt - 1)];
2530 log.u_bbr.flex5 = TICKS_2_USEC(bbr->rc_tp->t_rxtcur);
2531 log.u_bbr.flex6 = srtt;
2532 log.u_bbr.flex7 = bbr->r_ctl.rc_reorder_shift;
2533 log.u_bbr.flex8 = frm;
2534 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2535 &bbr->rc_inp->inp_socket->so_rcv,
2536 &bbr->rc_inp->inp_socket->so_snd,
2537 BBR_LOG_THRESH_CALC, 0,
2538 0, &log, false, &bbr->rc_tv);
2539 }
2540}
2541
2542static void
2543bbr_log_to_cancel(struct tcp_bbr *bbr, int32_t line, uint32_t cts, uint8_t hpts_removed)
2544{
2545 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2546 union tcp_log_stackspecific log;
2547
2548 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2549 log.u_bbr.flex1 = line;
2550 log.u_bbr.flex2 = bbr->bbr_timer_src;
2551 log.u_bbr.flex3 = bbr->r_ctl.rc_hpts_flags;
2552 log.u_bbr.flex4 = bbr->rc_in_persist;
2553 log.u_bbr.flex5 = bbr->r_ctl.rc_target_at_state;
2554 log.u_bbr.flex6 = TICKS_2_USEC(bbr->rc_tp->t_rxtcur);
2555 log.u_bbr.flex8 = hpts_removed;
2556 log.u_bbr.pkts_out = bbr->rc_pacer_started;
2557 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2558 &bbr->rc_inp->inp_socket->so_rcv,
2559 &bbr->rc_inp->inp_socket->so_snd,
2560 BBR_LOG_TIMERCANC, 0,
2561 0, &log, false, &bbr->rc_tv);
2562 }
2563}
2564
2565static void
2566bbr_log_tstmp_validation(struct tcp_bbr *bbr, uint64_t peer_delta, uint64_t delta)
2567{
2568 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2569 union tcp_log_stackspecific log;
2570
2571 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);
2572 log.u_bbr.flex1 = bbr->r_ctl.bbr_peer_tsratio;
2573 log.u_bbr.flex2 = (peer_delta >> 32);
2574 log.u_bbr.flex3 = (peer_delta & 0x00000000ffffffff);
2575 log.u_bbr.flex4 = (delta >> 32);
2576 log.u_bbr.flex5 = (delta & 0x00000000ffffffff);
2577 log.u_bbr.flex7 = bbr->rc_ts_clock_set;
2578 log.u_bbr.flex8 = bbr->rc_ts_cant_be_used;
2579 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2580 &bbr->rc_inp->inp_socket->so_rcv,
2581 &bbr->rc_inp->inp_socket->so_snd,
2582 BBR_LOG_TSTMP_VAL, 0,
2583 0, &log, false, &bbr->rc_tv);
2584 }
2585}
2586
2587static void
2588bbr_log_type_tsosize(struct tcp_bbr *bbr, uint32_t cts, uint32_t tsosz, uint32_t tls, uint32_t old_val, uint32_t maxseg, int hdwr)
2589{
2590 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2591 union tcp_log_stackspecific log;
2592
2593 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2594 log.u_bbr.flex1 = tsosz;
2595 log.u_bbr.flex2 = tls;
2596 log.u_bbr.flex3 = tcp_min_hptsi_time;
2597 log.u_bbr.flex4 = bbr->r_ctl.bbr_hptsi_bytes_min;
2598 log.u_bbr.flex5 = old_val;
2599 log.u_bbr.flex6 = maxseg;
2600 log.u_bbr.flex7 = bbr->rc_no_pacing;
2601 log.u_bbr.flex7 <<= 1;
2602 log.u_bbr.flex7 |= bbr->rc_past_init_win;
2603 if (hdwr)
2604 log.u_bbr.flex8 = 0x80 | bbr->rc_use_google;
2605 else
2606 log.u_bbr.flex8 = bbr->rc_use_google;
2607 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2608 &bbr->rc_inp->inp_socket->so_rcv,
2609 &bbr->rc_inp->inp_socket->so_snd,
2610 BBR_LOG_BBRTSO, 0,
2611 0, &log, false, &bbr->rc_tv);
2612 }
2613}
2614
2615static void
2616bbr_log_type_rsmclear(struct tcp_bbr *bbr, uint32_t cts, struct bbr_sendmap *rsm,
2617 uint32_t flags, uint32_t line)
2618{
2619 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2620 union tcp_log_stackspecific log;
2621
2622 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2623 log.u_bbr.flex1 = line;
2624 log.u_bbr.flex2 = rsm->r_start;
2625 log.u_bbr.flex3 = rsm->r_end;
2626 log.u_bbr.flex4 = rsm->r_delivered;
2627 log.u_bbr.flex5 = rsm->r_rtr_cnt;
2628 log.u_bbr.flex6 = rsm->r_dupack;
2629 log.u_bbr.flex7 = rsm->r_tim_lastsent[0];
2630 log.u_bbr.flex8 = rsm->r_flags;
2631 /* Hijack the pkts_out fids */
2632 log.u_bbr.applimited = flags;
2633 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2634 &bbr->rc_inp->inp_socket->so_rcv,
2635 &bbr->rc_inp->inp_socket->so_snd,
2636 BBR_RSM_CLEARED, 0,
2637 0, &log, false, &bbr->rc_tv);
2638 }
2639}
2640
2641static void
2642bbr_log_type_bbrupd(struct tcp_bbr *bbr, uint8_t flex8, uint32_t cts,
2643 uint32_t flex3, uint32_t flex2, uint32_t flex5,
2644 uint32_t flex6, uint32_t pkts_out, int flex7,
2645 uint32_t flex4, uint32_t flex1)
2646{
2647
2648 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2649 union tcp_log_stackspecific log;
2650
2651 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2652 log.u_bbr.flex1 = flex1;
2653 log.u_bbr.flex2 = flex2;
2654 log.u_bbr.flex3 = flex3;
2655 log.u_bbr.flex4 = flex4;
2656 log.u_bbr.flex5 = flex5;
2657 log.u_bbr.flex6 = flex6;
2658 log.u_bbr.flex7 = flex7;
2659 /* Hijack the pkts_out fids */
2660 log.u_bbr.pkts_out = pkts_out;
2661 log.u_bbr.flex8 = flex8;
2662 if (bbr->rc_ack_was_delayed)
2663 log.u_bbr.epoch = bbr->r_ctl.rc_ack_hdwr_delay;
2664 else
2665 log.u_bbr.epoch = 0;
2666 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2667 &bbr->rc_inp->inp_socket->so_rcv,
2668 &bbr->rc_inp->inp_socket->so_snd,
2669 BBR_LOG_BBRUPD, 0,
2670 flex2, &log, false, &bbr->rc_tv);
2671 }
2672}
2673
2674static void
2675bbr_log_type_ltbw(struct tcp_bbr *bbr, uint32_t cts, int32_t reason,
2676 uint32_t newbw, uint32_t obw, uint32_t diff,
2677 uint32_t tim)
2678{
2679 if (/*bbr_verbose_logging && */(bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF)) {
2680 union tcp_log_stackspecific log;
2681
2682 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2683 log.u_bbr.flex1 = reason;
2684 log.u_bbr.flex2 = newbw;
2685 log.u_bbr.flex3 = obw;
2686 log.u_bbr.flex4 = diff;
2687 log.u_bbr.flex5 = bbr->r_ctl.rc_lt_lost;
2688 log.u_bbr.flex6 = bbr->r_ctl.rc_lt_del;
2689 log.u_bbr.flex7 = bbr->rc_lt_is_sampling;
2690 log.u_bbr.pkts_out = tim;
2691 log.u_bbr.bw_inuse = bbr->r_ctl.rc_lt_bw;
2692 if (bbr->rc_lt_use_bw == 0)
2693 log.u_bbr.epoch = bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_lt_epoch;
2694 else
2695 log.u_bbr.epoch = bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_lt_epoch_use;
2696 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2697 &bbr->rc_inp->inp_socket->so_rcv,
2698 &bbr->rc_inp->inp_socket->so_snd,
2699 BBR_LOG_BWSAMP, 0,
2700 0, &log, false, &bbr->rc_tv);
2701 }
2702}
2703
2704static inline void
2705bbr_log_progress_event(struct tcp_bbr *bbr, struct tcpcb *tp, uint32_t tick, int event, int line)
2706{
2707 if (bbr_verbose_logging && (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF)) {
2708 union tcp_log_stackspecific log;
2709
2710 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);
2711 log.u_bbr.flex1 = line;
2712 log.u_bbr.flex2 = tick;
2713 log.u_bbr.flex3 = tp->t_maxunacktime;
2714 log.u_bbr.flex4 = tp->t_acktime;
2715 log.u_bbr.flex8 = event;
2716 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2717 &bbr->rc_inp->inp_socket->so_rcv,
2718 &bbr->rc_inp->inp_socket->so_snd,
2719 BBR_LOG_PROGRESS, 0,
2720 0, &log, false, &bbr->rc_tv);
2721 }
2722}
2723
2724static void
2725bbr_type_log_hdwr_pacing(struct tcp_bbr *bbr, const struct ifnet *ifp,
2726 uint64_t rate, uint64_t hw_rate, int line, uint32_t cts,
2727 int error)
2728{
2729 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2730 union tcp_log_stackspecific log;
2731
2732 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2733 log.u_bbr.flex1 = ((hw_rate >> 32) & 0x00000000ffffffff);
2734 log.u_bbr.flex2 = (hw_rate & 0x00000000ffffffff);
2735 log.u_bbr.flex3 = (((uint64_t)ifp >> 32) & 0x00000000ffffffff);
2736 log.u_bbr.flex4 = ((uint64_t)ifp & 0x00000000ffffffff);
2737 log.u_bbr.bw_inuse = rate;
2738 log.u_bbr.flex5 = line;
2739 log.u_bbr.flex6 = error;
2740 log.u_bbr.flex8 = bbr->skip_gain;
2741 log.u_bbr.flex8 <<= 1;
2742 log.u_bbr.flex8 |= bbr->gain_is_limited;
2743 log.u_bbr.flex8 <<= 1;
2744 log.u_bbr.flex8 |= bbr->bbr_hdrw_pacing;
2745 log.u_bbr.pkts_out = bbr->rc_tp->t_maxseg;
2746 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2747 &bbr->rc_inp->inp_socket->so_rcv,
2748 &bbr->rc_inp->inp_socket->so_snd,
2749 BBR_LOG_HDWR_PACE, 0,
2750 0, &log, false, &bbr->rc_tv);
2751 }
2752}
2753
2754static void
2755bbr_log_type_bbrsnd(struct tcp_bbr *bbr, uint32_t len, uint32_t slot, uint32_t del_by, uint32_t cts, uint32_t line, uint32_t prev_delay)
2756{
2757 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2758 union tcp_log_stackspecific log;
2759
2760 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2761 log.u_bbr.flex1 = slot;
2762 log.u_bbr.flex2 = del_by;
2763 log.u_bbr.flex3 = prev_delay;
2764 log.u_bbr.flex4 = line;
2765 log.u_bbr.flex5 = bbr->r_ctl.rc_last_delay_val;
2766 log.u_bbr.flex6 = bbr->r_ctl.rc_hptsi_agg_delay;
2767 log.u_bbr.flex7 = (0x0000ffff & bbr->r_ctl.rc_hpts_flags);
2768 log.u_bbr.flex8 = bbr->rc_in_persist;
2769 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2770 &bbr->rc_inp->inp_socket->so_rcv,
2771 &bbr->rc_inp->inp_socket->so_snd,
2772 BBR_LOG_BBRSND, 0,
2773 len, &log, false, &bbr->rc_tv);
2774 }
2775}
2776
2777static void
2778bbr_log_type_bbrrttprop(struct tcp_bbr *bbr, uint32_t t, uint32_t end, uint32_t tsconv, uint32_t cts, int32_t match, uint32_t seq, uint8_t flags)
2779{
2780 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2781 union tcp_log_stackspecific log;
2782
2783 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2784 log.u_bbr.flex1 = bbr->r_ctl.rc_delivered;
2785 log.u_bbr.flex2 = 0;
2786 log.u_bbr.flex3 = bbr->r_ctl.rc_lowest_rtt;
2787 log.u_bbr.flex4 = end;
2788 log.u_bbr.flex5 = seq;
2789 log.u_bbr.flex6 = t;
2790 log.u_bbr.flex7 = match;
2791 log.u_bbr.flex8 = flags;
2792 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2793 &bbr->rc_inp->inp_socket->so_rcv,
2794 &bbr->rc_inp->inp_socket->so_snd,
2795 BBR_LOG_BBRRTT, 0,
2796 0, &log, false, &bbr->rc_tv);
2797 }
2798}
2799
2800static void
2801bbr_log_exit_gain(struct tcp_bbr *bbr, uint32_t cts, int32_t entry_method)
2802{
2803 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2804 union tcp_log_stackspecific log;
2805
2806 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2807 log.u_bbr.flex1 = bbr->r_ctl.rc_target_at_state;
2808 log.u_bbr.flex2 = (bbr->rc_tp->t_maxseg - bbr->rc_last_options);
2809 log.u_bbr.flex3 = bbr->r_ctl.gain_epoch;
2810 log.u_bbr.flex4 = bbr->r_ctl.rc_pace_max_segs;
2811 log.u_bbr.flex5 = bbr->r_ctl.rc_pace_min_segs;
2812 log.u_bbr.flex6 = bbr->r_ctl.rc_bbr_state_atflight;
2813 log.u_bbr.flex7 = 0;
2814 log.u_bbr.flex8 = entry_method;
2815 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2816 &bbr->rc_inp->inp_socket->so_rcv,
2817 &bbr->rc_inp->inp_socket->so_snd,
2818 BBR_LOG_EXIT_GAIN, 0,
2819 0, &log, false, &bbr->rc_tv);
2820 }
2821}
2822
2823static void
2824bbr_log_settings_change(struct tcp_bbr *bbr, int settings_desired)
2825{
2826 if (bbr_verbose_logging && (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF)) {
2827 union tcp_log_stackspecific log;
2828
2829 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);
2830 /* R-HU */
2831 log.u_bbr.flex1 = 0;
2832 log.u_bbr.flex2 = 0;
2833 log.u_bbr.flex3 = 0;
2834 log.u_bbr.flex4 = 0;
2835 log.u_bbr.flex7 = 0;
2836 log.u_bbr.flex8 = settings_desired;
2837
2838 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2839 &bbr->rc_inp->inp_socket->so_rcv,
2840 &bbr->rc_inp->inp_socket->so_snd,
2841 BBR_LOG_SETTINGS_CHG, 0,
2842 0, &log, false, &bbr->rc_tv);
2843 }
2844}
2845
2846/*
2847 * Returns the bw from the our filter.
2848 */
2849static inline uint64_t
2850bbr_get_full_bw(struct tcp_bbr *bbr)
2851{
2852 uint64_t bw;
2853
2854 bw = get_filter_value(&bbr->r_ctl.rc_delrate);
2855
2856 return (bw);
2857}
2858
2859static inline void
2860bbr_set_pktepoch(struct tcp_bbr *bbr, uint32_t cts, int32_t line)
2861{
2862 uint64_t calclr;
2863 uint32_t lost, del;
2864
2865 if (bbr->r_ctl.rc_lost > bbr->r_ctl.rc_lost_at_pktepoch)
2866 lost = bbr->r_ctl.rc_lost - bbr->r_ctl.rc_lost_at_pktepoch;
2867 else
2868 lost = 0;
2869 del = bbr->r_ctl.rc_delivered - bbr->r_ctl.rc_pkt_epoch_del;
2870 if (lost == 0) {
2871 calclr = 0;
2872 } else if (del) {
2873 calclr = lost;
2874 calclr *= (uint64_t)1000;
2875 calclr /= (uint64_t)del;
2876 } else {
2877 /* Nothing delivered? 100.0% loss */
2878 calclr = 1000;
2879 }
2880 bbr->r_ctl.rc_pkt_epoch_loss_rate = (uint32_t)calclr;
2881 if (IN_RECOVERY(bbr->rc_tp->t_flags))
2882 bbr->r_ctl.recovery_lr += (uint32_t)calclr;
2883 bbr->r_ctl.rc_pkt_epoch++;
2884 if (bbr->rc_no_pacing &&
2885 (bbr->r_ctl.rc_pkt_epoch >= bbr->no_pacing_until)) {
2886 bbr->rc_no_pacing = 0;
2887 tcp_bbr_tso_size_check(bbr, cts);
2888 }
2889 bbr->r_ctl.rc_pkt_epoch_rtt = bbr_calc_time(cts, bbr->r_ctl.rc_pkt_epoch_time);
2890 bbr->r_ctl.rc_pkt_epoch_time = cts;
2891 /* What was our loss rate */
2892 bbr_log_pkt_epoch(bbr, cts, line, lost, del);
2893 bbr->r_ctl.rc_pkt_epoch_del = bbr->r_ctl.rc_delivered;
2894 bbr->r_ctl.rc_lost_at_pktepoch = bbr->r_ctl.rc_lost;
2895}
2896
2897static inline void
2898bbr_set_epoch(struct tcp_bbr *bbr, uint32_t cts, int32_t line)
2899{
2900 uint32_t epoch_time;
2901
2902 /* Tick the RTT clock */
2903 bbr->r_ctl.rc_rtt_epoch++;
2904 epoch_time = cts - bbr->r_ctl.rc_rcv_epoch_start;
2905 bbr_log_time_epoch(bbr, cts, line, epoch_time);
2906 bbr->r_ctl.rc_rcv_epoch_start = cts;
2907}
2908
2909static inline void
2910bbr_isit_a_pkt_epoch(struct tcp_bbr *bbr, uint32_t cts, struct bbr_sendmap *rsm, int32_t line, int32_t cum_acked)
2911{
2912 if (SEQ_GEQ(rsm->r_delivered, bbr->r_ctl.rc_pkt_epoch_del)) {
2913 bbr->rc_is_pkt_epoch_now = 1;
2914 }
2915}
2916
2917/*
2918 * Returns the bw from either the b/w filter
2919 * or from the lt_bw (if the connection is being
2920 * policed).
2921 */
2922static inline uint64_t
2923__bbr_get_bw(struct tcp_bbr *bbr)
2924{
2925 uint64_t bw, min_bw;
2926 uint64_t rtt;
2927 int gm_measure_cnt = 1;
2928
2929 /*
2930 * For startup we make, like google, a
2931 * minimum b/w. This is generated from the
2932 * IW and the rttProp. We do fall back to srtt
2933 * if for some reason (initial handshake) we don't
2934 * have a rttProp. We, in the worst case, fall back
2935 * to the configured min_bw (rc_initial_hptsi_bw).
2936 */
2937 if (bbr->rc_bbr_state == BBR_STATE_STARTUP) {
2938 /* Attempt first to use rttProp */
2939 rtt = (uint64_t)get_filter_value_small(&bbr->r_ctl.rc_rttprop);
2940 if (rtt && (rtt < 0xffffffff)) {
2941measure:
2942 min_bw = (uint64_t)(bbr_initial_cwnd(bbr, bbr->rc_tp)) *
2943 ((uint64_t)1000000);
2944 min_bw /= rtt;
2945 if (min_bw < bbr->r_ctl.rc_initial_hptsi_bw) {
2946 min_bw = bbr->r_ctl.rc_initial_hptsi_bw;
2947 }
2948
2949 } else if (bbr->rc_tp->t_srtt != 0) {
2950 /* No rttProp, use srtt? */
2951 rtt = bbr_get_rtt(bbr, BBR_SRTT);
2952 goto measure;
2953 } else {
2954 min_bw = bbr->r_ctl.rc_initial_hptsi_bw;
2955 }
2956 } else
2957 min_bw = 0;
2958
2959 if ((bbr->rc_past_init_win == 0) &&
2960 (bbr->r_ctl.rc_delivered > bbr_initial_cwnd(bbr, bbr->rc_tp)))
2961 bbr->rc_past_init_win = 1;
2962 if ((bbr->rc_use_google) && (bbr->r_ctl.r_measurement_count >= 1))
2963 gm_measure_cnt = 0;
2964 if (gm_measure_cnt &&
2965 ((bbr->r_ctl.r_measurement_count < bbr_min_measurements_req) ||
2966 (bbr->rc_past_init_win == 0))) {
2967 /* For google we use our guess rate until we get 1 measurement */
2968
2969use_initial_window:
2970 rtt = (uint64_t)get_filter_value_small(&bbr->r_ctl.rc_rttprop);
2971 if (rtt && (rtt < 0xffffffff)) {
2972 /*
2973 * We have an RTT measurement. Use that in
2974 * combination with our initial window to calculate
2975 * a b/w.
2976 */
2977 bw = (uint64_t)(bbr_initial_cwnd(bbr, bbr->rc_tp)) *
2978 ((uint64_t)1000000);
2979 bw /= rtt;
2980 if (bw < bbr->r_ctl.rc_initial_hptsi_bw) {
2981 bw = bbr->r_ctl.rc_initial_hptsi_bw;
2982 }
2983 } else {
2984 /* Drop back to the 40 and punt to a default */
2985 bw = bbr->r_ctl.rc_initial_hptsi_bw;
2986 }
2987 if (bw < 1)
2988 /* Probably should panic */
2989 bw = 1;
2990 if (bw > min_bw)
2991 return (bw);
2992 else
2993 return (min_bw);
2994 }
2995 if (bbr->rc_lt_use_bw)
2996 bw = bbr->r_ctl.rc_lt_bw;
2997 else if (bbr->r_recovery_bw && (bbr->rc_use_google == 0))
2998 bw = bbr->r_ctl.red_bw;
2999 else
3000 bw = get_filter_value(&bbr->r_ctl.rc_delrate);
3001 if (bbr->rc_tp->t_peakrate_thr && (bbr->rc_use_google == 0)) {
3002 /*
3003 * Enforce user set rate limit, keep in mind that
3004 * t_peakrate_thr is in B/s already
3005 */
3006 bw = uqmin((uint64_t)bbr->rc_tp->t_peakrate_thr, bw);
3007 }
3008 if (bw == 0) {
3009 /* We should not be at 0, go to the initial window then */
3010 goto use_initial_window;
3011 }
3012 if (bw < 1)
3013 /* Probably should panic */
3014 bw = 1;
3015 if (bw < min_bw)
3016 bw = min_bw;
3017 return (bw);
3018}
3019
3020static inline uint64_t
3021bbr_get_bw(struct tcp_bbr *bbr)
3022{
3023 uint64_t bw;
3024
3025 bw = __bbr_get_bw(bbr);
3026 return (bw);
3027}
3028
3029static inline void
3030bbr_reset_lt_bw_interval(struct tcp_bbr *bbr, uint32_t cts)
3031{
3032 bbr->r_ctl.rc_lt_epoch = bbr->r_ctl.rc_pkt_epoch;
3033 bbr->r_ctl.rc_lt_time = bbr->r_ctl.rc_del_time;
3034 bbr->r_ctl.rc_lt_del = bbr->r_ctl.rc_delivered;
3035 bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost;
3036}
3037
3038static inline void
3039bbr_reset_lt_bw_sampling(struct tcp_bbr *bbr, uint32_t cts)
3040{
3041 bbr->rc_lt_is_sampling = 0;
3042 bbr->rc_lt_use_bw = 0;
3043 bbr->r_ctl.rc_lt_bw = 0;
3044 bbr_reset_lt_bw_interval(bbr, cts);
3045}
3046
3047static inline void
3048bbr_lt_bw_samp_done(struct tcp_bbr *bbr, uint64_t bw, uint32_t cts, uint32_t timin)
3049{
3050 uint64_t diff;
3051
3052 /* Do we have a previous sample? */
3053 if (bbr->r_ctl.rc_lt_bw) {
3054 /* Get the diff in bytes per second */
3055 if (bbr->r_ctl.rc_lt_bw > bw)
3056 diff = bbr->r_ctl.rc_lt_bw - bw;
3057 else
3058 diff = bw - bbr->r_ctl.rc_lt_bw;
3059 if ((diff <= bbr_lt_bw_diff) ||
3060 (diff <= (bbr->r_ctl.rc_lt_bw / bbr_lt_bw_ratio))) {
3061 /* Consider us policed */
3062 uint32_t saved_bw;
3063
3064 saved_bw = (uint32_t)bbr->r_ctl.rc_lt_bw;
3065 bbr->r_ctl.rc_lt_bw = (bw + bbr->r_ctl.rc_lt_bw) / 2; /* average of two */
3066 bbr->rc_lt_use_bw = 1;
3067 bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT;
3068 /*
3069 * Use pkt based epoch for measuring length of
3070 * policer up
3071 */
3072 bbr->r_ctl.rc_lt_epoch_use = bbr->r_ctl.rc_pkt_epoch;
3073 /*
3074 * reason 4 is we need to start consider being
3075 * policed
3076 */
3077 bbr_log_type_ltbw(bbr, cts, 4, (uint32_t)bw, saved_bw, (uint32_t)diff, timin);
3078 return;
3079 }
3080 }
3081 bbr->r_ctl.rc_lt_bw = bw;
3082 bbr_reset_lt_bw_interval(bbr, cts);
3083 bbr_log_type_ltbw(bbr, cts, 5, 0, (uint32_t)bw, 0, timin);
3084}
3085
3086static void
3087bbr_randomize_extra_state_time(struct tcp_bbr *bbr)
3088{
3089 uint32_t ran, deduct;
3090
3091 ran = arc4random_uniform(bbr_rand_ot);
3092 if (ran) {
3093 deduct = bbr->r_ctl.rc_level_state_extra / ran;
3094 bbr->r_ctl.rc_level_state_extra -= deduct;
3095 }
3096}
3097/*
3098 * Return randomly the starting state
3099 * to use in probebw.
3100 */
3101static uint8_t
3102bbr_pick_probebw_substate(struct tcp_bbr *bbr, uint32_t cts)
3103{
3104 uint32_t ran;
3105 uint8_t ret_val;
3106
3107 /* Initialize the offset to 0 */
3108 bbr->r_ctl.rc_exta_time_gd = 0;
3109 bbr->rc_hit_state_1 = 0;
3110 bbr->r_ctl.rc_level_state_extra = 0;
3111 ran = arc4random_uniform((BBR_SUBSTATE_COUNT-1));
3112 /*
3113 * The math works funny here :) the return value is used to set the
3114 * substate and then the state change is called which increments by
3115 * one. So if we return 1 (DRAIN) we will increment to 2 (LEVEL1) when
3116 * we fully enter the state. Note that the (8 - 1 - ran) assures that
3117 * we return 1 - 7, so we dont return 0 and end up starting in
3118 * state 1 (DRAIN).
3119 */
3120 ret_val = BBR_SUBSTATE_COUNT - 1 - ran;
3121 /* Set an epoch */
3122 if ((cts - bbr->r_ctl.rc_rcv_epoch_start) >= bbr_get_rtt(bbr, BBR_RTT_PROP))
3123 bbr_set_epoch(bbr, cts, __LINE__);
3124
3125 bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost;
3126 return (ret_val);
3127}
3128
3129static void
3130bbr_lt_bw_sampling(struct tcp_bbr *bbr, uint32_t cts, int32_t loss_detected)
3131{
3132 uint32_t diff, d_time;
3133 uint64_t del_time, bw, lost, delivered;
3134
3135 if (bbr->r_use_policer == 0)
3136 return;
3137 if (bbr->rc_lt_use_bw) {
3138 /* We are using lt bw do we stop yet? */
3139 diff = bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_lt_epoch_use;
3140 if (diff > bbr_lt_bw_max_rtts) {
3141 /* Reset it all */
3142reset_all:
3143 bbr_reset_lt_bw_sampling(bbr, cts);
3144 if (bbr->rc_filled_pipe) {
3145 bbr_set_epoch(bbr, cts, __LINE__);
3146 bbr->rc_bbr_substate = bbr_pick_probebw_substate(bbr, cts);
3147 bbr_substate_change(bbr, cts, __LINE__, 0);
3148 bbr->rc_bbr_state = BBR_STATE_PROBE_BW;
3149 bbr_log_type_statechange(bbr, cts, __LINE__);
3150 } else {
3151 /*
3152 * This should not happen really
3153 * unless we remove the startup/drain
3154 * restrictions above.
3155 */
3156 bbr->rc_bbr_state = BBR_STATE_STARTUP;
3157 bbr_set_epoch(bbr, cts, __LINE__);
3158 bbr->r_ctl.rc_bbr_state_time = cts;
3159 bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost;
3160 bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.rc_startup_pg;
3161 bbr->r_ctl.rc_bbr_cwnd_gain = bbr->r_ctl.rc_startup_pg;
3162 bbr_set_state_target(bbr, __LINE__);
3163 bbr_log_type_statechange(bbr, cts, __LINE__);
3164 }
3165 /* reason 0 is to stop using lt-bw */
3166 bbr_log_type_ltbw(bbr, cts, 0, 0, 0, 0, 0);
3167 return;
3168 }
3169 if (bbr_lt_intvl_fp == 0) {
3170 /* Not doing false-postive detection */
3171 return;
3172 }
3173 /* False positive detection */
3174 if (diff == bbr_lt_intvl_fp) {
3175 /* At bbr_lt_intvl_fp we record the lost */
3176 bbr->r_ctl.rc_lt_del = bbr->r_ctl.rc_delivered;
3177 bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost;
3178 } else if (diff > (bbr_lt_intvl_min_rtts + bbr_lt_intvl_fp)) {
3179 /* Now is our loss rate still high? */
3180 lost = bbr->r_ctl.rc_lost - bbr->r_ctl.rc_lt_lost;
3181 delivered = bbr->r_ctl.rc_delivered - bbr->r_ctl.rc_lt_del;
3182 if ((delivered == 0) ||
3183 (((lost * 1000)/delivered) < bbr_lt_fd_thresh)) {
3184 /* No still below our threshold */
3185 bbr_log_type_ltbw(bbr, cts, 7, lost, delivered, 0, 0);
3186 } else {
3187 /* Yikes its still high, it must be a false positive */
3188 bbr_log_type_ltbw(bbr, cts, 8, lost, delivered, 0, 0);
3189 goto reset_all;
3190 }
3191 }
3192 return;
3193 }
3194 /*
3195 * Wait for the first loss before sampling, to let the policer
3196 * exhaust its tokens and estimate the steady-state rate allowed by
3197 * the policer. Starting samples earlier includes bursts that
3198 * over-estimate the bw.
3199 */
3200 if (bbr->rc_lt_is_sampling == 0) {
3201 /* reason 1 is to begin doing the sampling */
3202 if (loss_detected == 0)
3203 return;
3204 bbr_reset_lt_bw_interval(bbr, cts);
3205 bbr->rc_lt_is_sampling = 1;
3206 bbr_log_type_ltbw(bbr, cts, 1, 0, 0, 0, 0);
3207 return;
3208 }
3209 /* Now how long were we delivering long term last> */
3210 if (TSTMP_GEQ(bbr->r_ctl.rc_del_time, bbr->r_ctl.rc_lt_time))
3211 d_time = bbr->r_ctl.rc_del_time - bbr->r_ctl.rc_lt_time;
3212 else
3213 d_time = 0;
3214
3215 /* To avoid underestimates, reset sampling if we run out of data. */
3216 if (bbr->r_ctl.r_app_limited_until) {
3217 /* Can not measure in app-limited state */
3218 bbr_reset_lt_bw_sampling(bbr, cts);
3219 /* reason 2 is to reset sampling due to app limits */
3220 bbr_log_type_ltbw(bbr, cts, 2, 0, 0, 0, d_time);
3221 return;
3222 }
3223 diff = bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_lt_epoch;
3224 if (diff < bbr_lt_intvl_min_rtts) {
3225 /*
3226 * need more samples (we don't
3227 * start on a round like linux so
3228 * we need 1 more).
3229 */
3230 /* 6 is not_enough time or no-loss */
3231 bbr_log_type_ltbw(bbr, cts, 6, 0, 0, 0, d_time);
3232 return;
3233 }
3234 if (diff > (4 * bbr_lt_intvl_min_rtts)) {
3235 /*
3236 * For now if we wait too long, reset all sampling. We need
3237 * to do some research here, its possible that we should
3238 * base this on how much loss as occurred.. something like
3239 * if its under 10% (or some thresh) reset all otherwise
3240 * don't. Thats for phase II I guess.
3241 */
3242 bbr_reset_lt_bw_sampling(bbr, cts);
3243 /* reason 3 is to reset sampling due too long of sampling */
3244 bbr_log_type_ltbw(bbr, cts, 3, 0, 0, 0, d_time);
3245 return;
3246 }
3247 /*
3248 * End sampling interval when a packet is lost, so we estimate the
3249 * policer tokens were exhausted. Stopping the sampling before the
3250 * tokens are exhausted under-estimates the policed rate.
3251 */
3252 if (loss_detected == 0) {
3253 /* 6 is not_enough time or no-loss */
3254 bbr_log_type_ltbw(bbr, cts, 6, 0, 0, 0, d_time);
3255 return;
3256 }
3257 /* Calculate packets lost and delivered in sampling interval. */
3258 lost = bbr->r_ctl.rc_lost - bbr->r_ctl.rc_lt_lost;
3259 delivered = bbr->r_ctl.rc_delivered - bbr->r_ctl.rc_lt_del;
3260 if ((delivered == 0) ||
3261 (((lost * 1000)/delivered) < bbr_lt_loss_thresh)) {
3262 bbr_log_type_ltbw(bbr, cts, 6, lost, delivered, 0, d_time);
3263 return;
3264 }
3265 if (d_time < 1000) {
3266 /* Not enough time. wait */
3267 /* 6 is not_enough time or no-loss */
3268 bbr_log_type_ltbw(bbr, cts, 6, 0, 0, 0, d_time);
3269 return;
3270 }
3271 if (d_time >= (0xffffffff / USECS_IN_MSEC)) {
3272 /* Too long */
3273 bbr_reset_lt_bw_sampling(bbr, cts);
3274 /* reason 3 is to reset sampling due too long of sampling */
3275 bbr_log_type_ltbw(bbr, cts, 3, 0, 0, 0, d_time);
3276 return;
3277 }
3278 del_time = d_time;
3279 bw = delivered;
3280 bw *= (uint64_t)USECS_IN_SECOND;
3281 bw /= del_time;
3282 bbr_lt_bw_samp_done(bbr, bw, cts, d_time);
3283}
3284
3285/*
3286 * Allocate a sendmap from our zone.
3287 */
3288static struct bbr_sendmap *
3289bbr_alloc(struct tcp_bbr *bbr)
3290{
3291 struct bbr_sendmap *rsm;
3292
3293 BBR_STAT_INC(bbr_to_alloc);
3294 rsm = uma_zalloc(bbr_zone, (M_NOWAIT | M_ZERO));
3295 if (rsm) {
3296 bbr->r_ctl.rc_num_maps_alloced++;
3297 return (rsm);
3298 }
3299 if (bbr->r_ctl.rc_free_cnt) {
3300 BBR_STAT_INC(bbr_to_alloc_emerg);
3301 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_free);
3302 TAILQ_REMOVE(&bbr->r_ctl.rc_free, rsm, r_next);
3303 bbr->r_ctl.rc_free_cnt--;
3304 return (rsm);
3305 }
3306 BBR_STAT_INC(bbr_to_alloc_failed);
3307 return (NULL);
3308}
3309
3310static struct bbr_sendmap *
3311bbr_alloc_full_limit(struct tcp_bbr *bbr)
3312{
3313 if ((V_tcp_map_entries_limit > 0) &&
3314 (bbr->r_ctl.rc_num_maps_alloced >= V_tcp_map_entries_limit)) {
3315 BBR_STAT_INC(bbr_alloc_limited);
3316 if (!bbr->alloc_limit_reported) {
3317 bbr->alloc_limit_reported = 1;
3318 BBR_STAT_INC(bbr_alloc_limited_conns);
3319 }
3320 return (NULL);
3321 }
3322 return (bbr_alloc(bbr));
3323}
3324
3325/* wrapper to allocate a sendmap entry, subject to a specific limit */
3326static struct bbr_sendmap *
3327bbr_alloc_limit(struct tcp_bbr *bbr, uint8_t limit_type)
3328{
3329 struct bbr_sendmap *rsm;
3330
3331 if (limit_type) {
3332 /* currently there is only one limit type */
3333 if (V_tcp_map_split_limit > 0 &&
3334 bbr->r_ctl.rc_num_split_allocs >= V_tcp_map_split_limit) {
3335 BBR_STAT_INC(bbr_split_limited);
3336 if (!bbr->alloc_limit_reported) {
3337 bbr->alloc_limit_reported = 1;
3338 BBR_STAT_INC(bbr_alloc_limited_conns);
3339 }
3340 return (NULL);
3341 }
3342 }
3343
3344 /* allocate and mark in the limit type, if set */
3345 rsm = bbr_alloc(bbr);
3346 if (rsm != NULL && limit_type) {
3347 rsm->r_limit_type = limit_type;
3348 bbr->r_ctl.rc_num_split_allocs++;
3349 }
3350 return (rsm);
3351}
3352
3353static void
3354bbr_free(struct tcp_bbr *bbr, struct bbr_sendmap *rsm)
3355{
3356 if (rsm->r_limit_type) {
3357 /* currently there is only one limit type */
3358 bbr->r_ctl.rc_num_split_allocs--;
3359 }
3360 if (rsm->r_is_smallmap)
3361 bbr->r_ctl.rc_num_small_maps_alloced--;
3362 if (bbr->r_ctl.rc_tlp_send == rsm)
3363 bbr->r_ctl.rc_tlp_send = NULL;
3364 if (bbr->r_ctl.rc_resend == rsm) {
3365 bbr->r_ctl.rc_resend = NULL;
3366 }
3367 if (bbr->r_ctl.rc_next == rsm)
3368 bbr->r_ctl.rc_next = NULL;
3369 if (bbr->r_ctl.rc_sacklast == rsm)
3370 bbr->r_ctl.rc_sacklast = NULL;
3371 if (bbr->r_ctl.rc_free_cnt < bbr_min_req_free) {
3372 memset(rsm, 0, sizeof(struct bbr_sendmap));
3373 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_free, rsm, r_next);
3374 rsm->r_limit_type = 0;
3375 bbr->r_ctl.rc_free_cnt++;
3376 return;
3377 }
3378 bbr->r_ctl.rc_num_maps_alloced--;
3379 uma_zfree(bbr_zone, rsm);
3380}
3381
3382/*
3383 * Returns the BDP.
3384 */
3385static uint64_t
3386bbr_get_bw_delay_prod(uint64_t rtt, uint64_t bw) {
3387 /*
3388 * Calculate the bytes in flight needed given the bw (in bytes per
3389 * second) and the specifyed rtt in useconds. We need to put out the
3390 * returned value per RTT to match that rate. Gain will normally
3391 * raise it up from there.
3392 *
3393 * This should not overflow as long as the bandwidth is below 1
3394 * TByte per second (bw < 10**12 = 2**40) and the rtt is smaller
3395 * than 1000 seconds (rtt < 10**3 * 10**6 = 10**9 = 2**30).
3396 */
3397 uint64_t usec_per_sec;
3398
3399 usec_per_sec = USECS_IN_SECOND;
3400 return ((rtt * bw) / usec_per_sec);
3401}
3402
3403/*
3404 * Return the initial cwnd.
3405 */
3406static uint32_t
3407bbr_initial_cwnd(struct tcp_bbr *bbr, struct tcpcb *tp)
3408{
3409 uint32_t i_cwnd;
3410
3411 if (bbr->rc_init_win) {
3412 i_cwnd = bbr->rc_init_win * tp->t_maxseg;
3413 } else if (V_tcp_initcwnd_segments)
3414 i_cwnd = min((V_tcp_initcwnd_segments * tp->t_maxseg),
3415 max(2 * tp->t_maxseg, 14600));
3416 else if (V_tcp_do_rfc3390)
3417 i_cwnd = min(4 * tp->t_maxseg,
3418 max(2 * tp->t_maxseg, 4380));
3419 else {
3420 /* Per RFC5681 Section 3.1 */
3421 if (tp->t_maxseg > 2190)
3422 i_cwnd = 2 * tp->t_maxseg;
3423 else if (tp->t_maxseg > 1095)
3424 i_cwnd = 3 * tp->t_maxseg;
3425 else
3426 i_cwnd = 4 * tp->t_maxseg;
3427 }
3428 return (i_cwnd);
3429}
3430
3431/*
3432 * Given a specified gain, return the target
3433 * cwnd based on that gain.
3434 */
3435static uint32_t
3436bbr_get_raw_target_cwnd(struct tcp_bbr *bbr, uint32_t gain, uint64_t bw)
3437{
3438 uint64_t bdp, rtt;
3439 uint32_t cwnd;
3440
3441 if ((get_filter_value_small(&bbr->r_ctl.rc_rttprop) == 0xffffffff) ||
3442 (bbr_get_full_bw(bbr) == 0)) {
3443 /* No measurements yet */
3444 return (bbr_initial_cwnd(bbr, bbr->rc_tp));
3445 }
3446 /*
3447 * Get bytes per RTT needed (rttProp is normally in
3448 * bbr_cwndtarget_rtt_touse)
3449 */
3450 rtt = bbr_get_rtt(bbr, bbr_cwndtarget_rtt_touse);
3451 /* Get the bdp from the two values */
3452 bdp = bbr_get_bw_delay_prod(rtt, bw);
3453 /* Now apply the gain */
3454 cwnd = (uint32_t)(((bdp * ((uint64_t)gain)) + (uint64_t)(BBR_UNIT - 1)) / ((uint64_t)BBR_UNIT));
3455
3456 return (cwnd);
3457}
3458
3459static uint32_t
3460bbr_get_target_cwnd(struct tcp_bbr *bbr, uint64_t bw, uint32_t gain)
3461{
3462 uint32_t cwnd, mss;
3463
3464 mss = min((bbr->rc_tp->t_maxseg - bbr->rc_last_options), bbr->r_ctl.rc_pace_max_segs);
3465 /* Get the base cwnd with gain rounded to a mss */
3466 cwnd = roundup(bbr_get_raw_target_cwnd(bbr, bw, gain), mss);
3467 /*
3468 * Add in N (2 default since we do not have a
3469 * fq layer to trap packets in) quanta's per the I-D
3470 * section 4.2.3.2 quanta adjust.
3471 */
3472 cwnd += (bbr_quanta * bbr->r_ctl.rc_pace_max_segs);
3473 if (bbr->rc_use_google) {
3474 if((bbr->rc_bbr_state == BBR_STATE_PROBE_BW) &&
3475 (bbr_state_val(bbr) == BBR_SUB_GAIN)) {
3476 /*
3477 * The linux implementation adds
3478 * an extra 2 x mss in gain cycle which
3479 * is documented no-where except in the code.
3480 * so we add more for Neal undocumented feature
3481 */
3482 cwnd += 2 * mss;
3483 }
3484 if ((cwnd / mss) & 0x1) {
3485 /* Round up for odd num mss */
3486 cwnd += mss;
3487 }
3488 }
3489 /* Are we below the min cwnd? */
3490 if (cwnd < get_min_cwnd(bbr))
3491 return (get_min_cwnd(bbr));
3492 return (cwnd);
3493}
3494
3495static uint16_t
3496bbr_gain_adjust(struct tcp_bbr *bbr, uint16_t gain)
3497{
3498 if (gain < 1)
3499 gain = 1;
3500 return (gain);
3501}
3502
3503static uint32_t
3504bbr_get_header_oh(struct tcp_bbr *bbr)
3505{
3506 int seg_oh;
3507
3508 seg_oh = 0;
3509 if (bbr->r_ctl.rc_inc_tcp_oh) {
3510 /* Do we include TCP overhead? */
3511 seg_oh = (bbr->rc_last_options + sizeof(struct tcphdr));
3512 }
3513 if (bbr->r_ctl.rc_inc_ip_oh) {
3514 /* Do we include IP overhead? */
3515#ifdef INET6
3516 if (bbr->r_is_v6) {
3517 seg_oh += sizeof(struct ip6_hdr);
3518 } else
3519#endif
3520 {
3521
3522#ifdef INET
3523 seg_oh += sizeof(struct ip);
3524#endif
3525 }
3526 }
3527 if (bbr->r_ctl.rc_inc_enet_oh) {
3528 /* Do we include the ethernet overhead? */
3529 seg_oh += sizeof(struct ether_header);
3530 }
3531 return(seg_oh);
3532}
3533
3534static uint32_t
3535bbr_get_pacing_length(struct tcp_bbr *bbr, uint16_t gain, uint32_t useconds_time, uint64_t bw)
3536{
3537 uint64_t divor, res, tim;
3538
3539 if (useconds_time == 0)
3540 return (0);
3541 gain = bbr_gain_adjust(bbr, gain);
3542 divor = (uint64_t)USECS_IN_SECOND * (uint64_t)BBR_UNIT;
3543 tim = useconds_time;
3544 res = (tim * bw * gain) / divor;
3545 if (res == 0)
3546 res = 1;
3547 return ((uint32_t)res);
3548}
3549
3550/*
3551 * Given a gain and a length return the delay in useconds that
3552 * should be used to evenly space out packets
3553 * on the connection (based on the gain factor).
3554 */
3555static uint32_t
3556bbr_get_pacing_delay(struct tcp_bbr *bbr, uint16_t gain, int32_t len, uint32_t cts, int nolog)
3557{
3558 uint64_t bw, lentim, res;
3559 uint32_t usecs, srtt, over = 0;
3560 uint32_t seg_oh, num_segs, maxseg;
3561
3562 if (len == 0)
3563 return (0);
3564
3565 maxseg = bbr->rc_tp->t_maxseg - bbr->rc_last_options;
3566 num_segs = (len + maxseg - 1) / maxseg;
3567 if (bbr->rc_use_google == 0) {
3568 seg_oh = bbr_get_header_oh(bbr);
3569 len += (num_segs * seg_oh);
3570 }
3571 gain = bbr_gain_adjust(bbr, gain);
3572 bw = bbr_get_bw(bbr);
3573 if (bbr->rc_use_google) {
3574 uint64_t cbw;
3575
3576 /*
3577 * Reduce the b/w by the google discount
3578 * factor 10 = 1%.
3579 */
3580 cbw = bw * (uint64_t)(1000 - bbr->r_ctl.bbr_google_discount);
3581 cbw /= (uint64_t)1000;
3582 /* We don't apply a discount if it results in 0 */
3583 if (cbw > 0)
3584 bw = cbw;
3585 }
3586 lentim = ((uint64_t)len *
3587 (uint64_t)USECS_IN_SECOND *
3588 (uint64_t)BBR_UNIT);
3589 res = lentim / ((uint64_t)gain * bw);
3590 if (res == 0)
3591 res = 1;
3592 usecs = (uint32_t)res;
3593 srtt = bbr_get_rtt(bbr, BBR_SRTT);
3594 if (bbr_hptsi_max_mul && bbr_hptsi_max_div &&
3595 (bbr->rc_use_google == 0) &&
3596 (usecs > ((srtt * bbr_hptsi_max_mul) / bbr_hptsi_max_div))) {
3597 /*
3598 * We cannot let the delay be more than 1/2 the srtt time.
3599 * Otherwise we cannot pace out or send properly.
3600 */
3601 over = usecs = (srtt * bbr_hptsi_max_mul) / bbr_hptsi_max_div;
3602 BBR_STAT_INC(bbr_hpts_min_time);
3603 }
3604 if (!nolog)
3605 bbr_log_pacing_delay_calc(bbr, gain, len, cts, usecs, bw, over, 1);
3606 return (usecs);
3607}
3608
3609static void
3610bbr_ack_received(struct tcpcb *tp, struct tcp_bbr *bbr, struct tcphdr *th, uint32_t bytes_this_ack,
3611 uint32_t sack_changed, uint32_t prev_acked, int32_t line, uint32_t losses)
3612{
3613 INP_WLOCK_ASSERT(tp->t_inpcb);
3614 uint64_t bw;
3615 uint32_t cwnd, target_cwnd, saved_bytes, maxseg;
3616 int32_t meth;
3617
3618#ifdef STATS
3619 if ((tp->t_flags & TF_GPUTINPROG) &&
3620 SEQ_GEQ(th->th_ack, tp->gput_ack)) {
3621 /*
3622 * Strech acks and compressed acks will cause this to
3623 * oscillate but we are doing it the same way as the main
3624 * stack so it will be compariable (though possibly not
3625 * ideal).
3626 */
3627 int32_t cgput;
3628 int64_t gput, time_stamp;
3629
3630 gput = (int64_t) (th->th_ack - tp->gput_seq) * 8;
3631 time_stamp = max(1, ((bbr->r_ctl.rc_rcvtime - tp->gput_ts) / 1000));
3632 cgput = gput / time_stamp;
3633 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_GPUT,
3634 cgput);
3635 if (tp->t_stats_gput_prev > 0)
3636 stats_voi_update_abs_s32(tp->t_stats,
3637 VOI_TCP_GPUT_ND,
3638 ((gput - tp->t_stats_gput_prev) * 100) /
3639 tp->t_stats_gput_prev);
3640 tp->t_flags &= ~TF_GPUTINPROG;
3641 tp->t_stats_gput_prev = cgput;
3642 }
3643#endif
3644 if ((bbr->rc_bbr_state == BBR_STATE_PROBE_RTT) &&
3645 ((bbr->r_ctl.bbr_rttprobe_gain_val == 0) || bbr->rc_use_google)) {
3646 /* We don't change anything in probe-rtt */
3647 return;
3648 }
3649 maxseg = tp->t_maxseg - bbr->rc_last_options;
3650 saved_bytes = bytes_this_ack;
3651 bytes_this_ack += sack_changed;
3652 if (bytes_this_ack > prev_acked) {
3653 bytes_this_ack -= prev_acked;
3654 /*
3655 * A byte ack'd gives us a full mss
3656 * to be like linux i.e. they count packets.
3657 */
3658 if ((bytes_this_ack < maxseg) && bbr->rc_use_google)
3659 bytes_this_ack = maxseg;
3660 } else {
3661 /* Unlikely */
3662 bytes_this_ack = 0;
3663 }
3664 cwnd = tp->snd_cwnd;
3665 bw = get_filter_value(&bbr->r_ctl.rc_delrate);
3666 if (bw)
3667 target_cwnd = bbr_get_target_cwnd(bbr,
3668 bw,
3669 (uint32_t)bbr->r_ctl.rc_bbr_cwnd_gain);
3670 else
3671 target_cwnd = bbr_initial_cwnd(bbr, bbr->rc_tp);
3672 if (IN_RECOVERY(tp->t_flags) &&
3673 (bbr->bbr_prev_in_rec == 0)) {
3674 /*
3675 * We are entering recovery and
3676 * thus packet conservation.
3677 */
3678 bbr->pkt_conservation = 1;
3679 bbr->r_ctl.rc_recovery_start = bbr->r_ctl.rc_rcvtime;
3680 cwnd = ctf_flight_size(tp,
3681 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) +
3682 bytes_this_ack;
3683 }
3684 if (IN_RECOVERY(tp->t_flags)) {
3685 uint32_t flight;
3686
3687 bbr->bbr_prev_in_rec = 1;
3688 if (cwnd > losses) {
3689 cwnd -= losses;
3690 if (cwnd < maxseg)
3691 cwnd = maxseg;
3692 } else
3693 cwnd = maxseg;
3694 flight = ctf_flight_size(tp,
3695 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
3696 bbr_log_type_cwndupd(bbr, flight, 0,
3697 losses, 10, 0, 0, line);
3698 if (bbr->pkt_conservation) {
3699 uint32_t time_in;
3700
3701 if (TSTMP_GEQ(bbr->r_ctl.rc_rcvtime, bbr->r_ctl.rc_recovery_start))
3702 time_in = bbr->r_ctl.rc_rcvtime - bbr->r_ctl.rc_recovery_start;
3703 else
3704 time_in = 0;
3705
3706 if (time_in >= bbr_get_rtt(bbr, BBR_RTT_PROP)) {
3707 /* Clear packet conservation after an rttProp */
3708 bbr->pkt_conservation = 0;
3709 } else {
3710 if ((flight + bytes_this_ack) > cwnd)
3711 cwnd = flight + bytes_this_ack;
3712 if (cwnd < get_min_cwnd(bbr))
3713 cwnd = get_min_cwnd(bbr);
3714 tp->snd_cwnd = cwnd;
3715 bbr_log_type_cwndupd(bbr, saved_bytes, sack_changed,
3716 prev_acked, 1, target_cwnd, th->th_ack, line);
3717 return;
3718 }
3719 }
3720 } else
3721 bbr->bbr_prev_in_rec = 0;
3722 if ((bbr->rc_use_google == 0) && bbr->r_ctl.restrict_growth) {
3723 bbr->r_ctl.restrict_growth--;
3724 if (bytes_this_ack > maxseg)
3725 bytes_this_ack = maxseg;
3726 }
3727 if (bbr->rc_filled_pipe) {
3728 /*
3729 * Here we have exited startup and filled the pipe. We will
3730 * thus allow the cwnd to shrink to the target. We hit here
3731 * mostly.
3732 */
3733 uint32_t s_cwnd;
3734
3735 meth = 2;
3736 s_cwnd = min((cwnd + bytes_this_ack), target_cwnd);
3737 if (s_cwnd > cwnd)
3738 cwnd = s_cwnd;
3739 else if (bbr_cwnd_may_shrink || bbr->rc_use_google || bbr->rc_no_pacing)
3740 cwnd = s_cwnd;
3741 } else {
3742 /*
3743 * Here we are still in startup, we increase cwnd by what
3744 * has been acked.
3745 */
3746 if ((cwnd < target_cwnd) ||
3747 (bbr->rc_past_init_win == 0)) {
3748 meth = 3;
3749 cwnd += bytes_this_ack;
3750 } else {
3751 /*
3752 * Method 4 means we are at target so no gain in
3753 * startup and past the initial window.
3754 */
3755 meth = 4;
3756 }
3757 }
3758 tp->snd_cwnd = max(cwnd, get_min_cwnd(bbr));
3759 bbr_log_type_cwndupd(bbr, saved_bytes, sack_changed, prev_acked, meth, target_cwnd, th->th_ack, line);
3760}
3761
3762static void
3763tcp_bbr_partialack(struct tcpcb *tp)
3764{
3765 struct tcp_bbr *bbr;
3766
3767 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
3768 INP_WLOCK_ASSERT(tp->t_inpcb);
3769 if (ctf_flight_size(tp,
3770 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) <=
3771 tp->snd_cwnd) {
3772 bbr->r_wanted_output = 1;
3773 }
3774}
3775
3776static void
3777bbr_post_recovery(struct tcpcb *tp)
3778{
3779 struct tcp_bbr *bbr;
3780 uint32_t flight;
3781
3782 INP_WLOCK_ASSERT(tp->t_inpcb);
3783 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
3784 /*
3785 * Here we just exit recovery.
3786 */
3787 EXIT_RECOVERY(tp->t_flags);
3788 /* Lock in our b/w reduction for the specified number of pkt-epochs */
3789 bbr->r_recovery_bw = 0;
3790 tp->snd_recover = tp->snd_una;
3791 tcp_bbr_tso_size_check(bbr, bbr->r_ctl.rc_rcvtime);
3792 bbr->pkt_conservation = 0;
3793 if (bbr->rc_use_google == 0) {
3794 /*
3795 * For non-google mode lets
3796 * go ahead and make sure we clear
3797 * the recovery state so if we
3798 * bounce back in to recovery we
3799 * will do PC.
3800 */
3801 bbr->bbr_prev_in_rec = 0;
3802 }
3803 bbr_log_type_exit_rec(bbr);
3804 if (bbr->rc_bbr_state != BBR_STATE_PROBE_RTT) {
3805 tp->snd_cwnd = max(tp->snd_cwnd, bbr->r_ctl.rc_cwnd_on_ent);
3806 bbr_log_type_cwndupd(bbr, 0, 0, 0, 15, 0, 0, __LINE__);
3807 } else {
3808 /* For probe-rtt case lets fix up its saved_cwnd */
3809 if (bbr->r_ctl.rc_saved_cwnd < bbr->r_ctl.rc_cwnd_on_ent) {
3810 bbr->r_ctl.rc_saved_cwnd = bbr->r_ctl.rc_cwnd_on_ent;
3811 bbr_log_type_cwndupd(bbr, 0, 0, 0, 16, 0, 0, __LINE__);
3812 }
3813 }
3814 flight = ctf_flight_size(tp,
3815 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
3816 if ((bbr->rc_use_google == 0) &&
3817 bbr_do_red) {
3818 uint64_t val, lr2use;
3819 uint32_t maxseg, newcwnd, acks_inflight, ratio, cwnd;
3820 uint32_t *cwnd_p;
3821
3822 if (bbr_get_rtt(bbr, BBR_SRTT)) {
3823 val = ((uint64_t)bbr_get_rtt(bbr, BBR_RTT_PROP) * (uint64_t)1000);
3824 val /= bbr_get_rtt(bbr, BBR_SRTT);
3825 ratio = (uint32_t)val;
3826 } else
3827 ratio = 1000;
3828
3829 bbr_log_type_cwndupd(bbr, bbr_red_mul, bbr_red_div,
3830 bbr->r_ctl.recovery_lr, 21,
3831 ratio,
3832 bbr->r_ctl.rc_red_cwnd_pe,
3833 __LINE__);
3834 if ((ratio < bbr_do_red) || (bbr_do_red == 0))
3835 goto done;
3836 if (((bbr->rc_bbr_state == BBR_STATE_PROBE_RTT) &&
3837 bbr_prtt_slam_cwnd) ||
3838 (bbr_sub_drain_slam_cwnd &&
3839 (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) &&
3840 bbr->rc_hit_state_1 &&
3841 (bbr_state_val(bbr) == BBR_SUB_DRAIN)) ||
3842 ((bbr->rc_bbr_state == BBR_STATE_DRAIN) &&
3843 bbr_slam_cwnd_in_main_drain)) {
3844 /*
3845 * Here we must poke at the saved cwnd
3846 * as well as the cwnd.
3847 */
3848 cwnd = bbr->r_ctl.rc_saved_cwnd;
3849 cwnd_p = &bbr->r_ctl.rc_saved_cwnd;
3850 } else {
3851 cwnd = tp->snd_cwnd;
3852 cwnd_p = &tp->snd_cwnd;
3853 }
3854 maxseg = tp->t_maxseg - bbr->rc_last_options;
3855 /* Add the overall lr with the recovery lr */
3856 if (bbr->r_ctl.rc_lost == 0)
3857 lr2use = 0;
3858 else if (bbr->r_ctl.rc_delivered == 0)
3859 lr2use = 1000;
3860 else {
3861 lr2use = bbr->r_ctl.rc_lost * 1000;
3862 lr2use /= bbr->r_ctl.rc_delivered;
3863 }
3864 lr2use += bbr->r_ctl.recovery_lr;
3865 acks_inflight = (flight / (maxseg * 2));
3866 if (bbr_red_scale) {
3867 lr2use *= bbr_get_rtt(bbr, BBR_SRTT);
3868 lr2use /= bbr_red_scale;
3869 if ((bbr_red_growth_restrict) &&
3870 ((bbr_get_rtt(bbr, BBR_SRTT)/bbr_red_scale) > 1))
3871 bbr->r_ctl.restrict_growth += acks_inflight;
3872 }
3873 if (lr2use) {
3874 val = (uint64_t)cwnd * lr2use;
3875 val /= 1000;
3876 if (cwnd > val)
3877 newcwnd = roundup((cwnd - val), maxseg);
3878 else
3879 newcwnd = maxseg;
3880 } else {
3881 val = (uint64_t)cwnd * (uint64_t)bbr_red_mul;
3882 val /= (uint64_t)bbr_red_div;
3883 newcwnd = roundup((uint32_t)val, maxseg);
3884 }
3885 /* with standard delayed acks how many acks can I expect? */
3886 if (bbr_drop_limit == 0) {
3887 /*
3888 * Anticpate how much we will
3889 * raise the cwnd based on the acks.
3890 */
3891 if ((newcwnd + (acks_inflight * maxseg)) < get_min_cwnd(bbr)) {
3892 /* We do enforce the min (with the acks) */
3893 newcwnd = (get_min_cwnd(bbr) - acks_inflight);
3894 }
3895 } else {
3896 /*
3897 * A strict drop limit of N is is inplace
3898 */
3899 if (newcwnd < (bbr_drop_limit * maxseg)) {
3900 newcwnd = bbr_drop_limit * maxseg;
3901 }
3902 }
3903 /* For the next N acks do we restrict the growth */
3904 *cwnd_p = newcwnd;
3905 if (tp->snd_cwnd > newcwnd)
3906 tp->snd_cwnd = newcwnd;
3907 bbr_log_type_cwndupd(bbr, bbr_red_mul, bbr_red_div, val, 22,
3908 (uint32_t)lr2use,
3909 bbr_get_rtt(bbr, BBR_SRTT), __LINE__);
3910 bbr->r_ctl.rc_red_cwnd_pe = bbr->r_ctl.rc_pkt_epoch;
3911 }
3912done:
3913 bbr->r_ctl.recovery_lr = 0;
3914 if (flight <= tp->snd_cwnd) {
3915 bbr->r_wanted_output = 1;
3916 }
3917 tcp_bbr_tso_size_check(bbr, bbr->r_ctl.rc_rcvtime);
3918}
3919
3920static void
3921bbr_setup_red_bw(struct tcp_bbr *bbr, uint32_t cts)
3922{
3923 bbr->r_ctl.red_bw = get_filter_value(&bbr->r_ctl.rc_delrate);
3924 /* Limit the drop in b/w to 1/2 our current filter. */
3925 if (bbr->r_ctl.red_bw > bbr->r_ctl.rc_bbr_cur_del_rate)
3926 bbr->r_ctl.red_bw = bbr->r_ctl.rc_bbr_cur_del_rate;
3927 if (bbr->r_ctl.red_bw < (get_filter_value(&bbr->r_ctl.rc_delrate) / 2))
3928 bbr->r_ctl.red_bw = get_filter_value(&bbr->r_ctl.rc_delrate) / 2;
3929 tcp_bbr_tso_size_check(bbr, cts);
3930}
3931
3932static void
3933bbr_cong_signal(struct tcpcb *tp, struct tcphdr *th, uint32_t type, struct bbr_sendmap *rsm)
3934{
3935 struct tcp_bbr *bbr;
3936
3937 INP_WLOCK_ASSERT(tp->t_inpcb);
3938#ifdef STATS
3939 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_CSIG, type);
3940#endif
3941 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
3942 switch (type) {
3943 case CC_NDUPACK:
3944 if (!IN_RECOVERY(tp->t_flags)) {
3945 tp->snd_recover = tp->snd_max;
3946 /* Start a new epoch */
3947 bbr_set_pktepoch(bbr, bbr->r_ctl.rc_rcvtime, __LINE__);
3948 if (bbr->rc_lt_is_sampling || bbr->rc_lt_use_bw) {
3949 /*
3950 * Move forward the lt epoch
3951 * so it won't count the truncated
3952 * epoch.
3953 */
3954 bbr->r_ctl.rc_lt_epoch++;
3955 }
3956 if (bbr->rc_bbr_state == BBR_STATE_STARTUP) {
3957 /*
3958 * Just like the policer detection code
3959 * if we are in startup we must push
3960 * forward the last startup epoch
3961 * to hide the truncated PE.
3962 */
3963 bbr->r_ctl.rc_bbr_last_startup_epoch++;
3964 }
3965 bbr->r_ctl.rc_cwnd_on_ent = tp->snd_cwnd;
3966 ENTER_RECOVERY(tp->t_flags);
3967 bbr->rc_tlp_rtx_out = 0;
3968 bbr->r_ctl.recovery_lr = bbr->r_ctl.rc_pkt_epoch_loss_rate;
3969 tcp_bbr_tso_size_check(bbr, bbr->r_ctl.rc_rcvtime);
3970 if (tcp_in_hpts(bbr->rc_inp) &&
3971 ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_RACK) == 0)) {
3972 /*
3973 * When we enter recovery, we need to restart
3974 * any timers. This may mean we gain an agg
3975 * early, which will be made up for at the last
3976 * rxt out.
3977 */
3978 bbr->rc_timer_first = 1;
3979 bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime);
3980 }
3981 /*
3982 * Calculate a new cwnd based on to the current
3983 * delivery rate with no gain. We get the bdp
3984 * without gaining it up like we normally would and
3985 * we use the last cur_del_rate.
3986 */
3987 if ((bbr->rc_use_google == 0) &&
3988 (bbr->r_ctl.bbr_rttprobe_gain_val ||
3989 (bbr->rc_bbr_state != BBR_STATE_PROBE_RTT))) {
3990 tp->snd_cwnd = ctf_flight_size(tp,
3991 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) +
3992 (tp->t_maxseg - bbr->rc_last_options);
3993 if (tp->snd_cwnd < get_min_cwnd(bbr)) {
3994 /* We always gate to min cwnd */
3995 tp->snd_cwnd = get_min_cwnd(bbr);
3996 }
3997 bbr_log_type_cwndupd(bbr, 0, 0, 0, 14, 0, 0, __LINE__);
3998 }
3999 bbr_log_type_enter_rec(bbr, rsm->r_start);
4000 }
4001 break;
4002 case CC_RTO_ERR:
4003 KMOD_TCPSTAT_INC(tcps_sndrexmitbad);
4004 /* RTO was unnecessary, so reset everything. */
4005 bbr_reset_lt_bw_sampling(bbr, bbr->r_ctl.rc_rcvtime);
4006 if (bbr->rc_bbr_state != BBR_STATE_PROBE_RTT) {
4007 tp->snd_cwnd = tp->snd_cwnd_prev;
4008 tp->snd_ssthresh = tp->snd_ssthresh_prev;
4009 tp->snd_recover = tp->snd_recover_prev;
4010 tp->snd_cwnd = max(tp->snd_cwnd, bbr->r_ctl.rc_cwnd_on_ent);
4011 bbr_log_type_cwndupd(bbr, 0, 0, 0, 13, 0, 0, __LINE__);
4012 }
4013 tp->t_badrxtwin = 0;
4014 break;
4015 }
4016}
4017
4018/*
4019 * Indicate whether this ack should be delayed. We can delay the ack if
4020 * following conditions are met:
4021 * - There is no delayed ack timer in progress.
4022 * - Our last ack wasn't a 0-sized window. We never want to delay
4023 * the ack that opens up a 0-sized window.
4024 * - LRO wasn't used for this segment. We make sure by checking that the
4025 * segment size is not larger than the MSS.
4026 * - Delayed acks are enabled or this is a half-synchronized T/TCP
4027 * connection.
4028 * - The data being acked is less than a full segment (a stretch ack
4029 * of more than a segment we should ack.
4030 * - nsegs is 1 (if its more than that we received more than 1 ack).
4031 */
4032#define DELAY_ACK(tp, bbr, nsegs) \
4033 (((tp->t_flags & TF_RXWIN0SENT) == 0) && \
4034 ((tp->t_flags & TF_DELACK) == 0) && \
4035 ((bbr->bbr_segs_rcvd + nsegs) < tp->t_delayed_ack) && \
4036 (tp->t_delayed_ack || (tp->t_flags & TF_NEEDSYN)))
4037
4038/*
4039 * Return the lowest RSM in the map of
4040 * packets still in flight that is not acked.
4041 * This should normally find on the first one
4042 * since we remove packets from the send
4043 * map after they are marked ACKED.
4044 */
4045static struct bbr_sendmap *
4046bbr_find_lowest_rsm(struct tcp_bbr *bbr)
4047{
4048 struct bbr_sendmap *rsm;
4049
4050 /*
4051 * Walk the time-order transmitted list looking for an rsm that is
4052 * not acked. This will be the one that was sent the longest time
4053 * ago that is still outstanding.
4054 */
4055 TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_tmap, r_tnext) {
4056 if (rsm->r_flags & BBR_ACKED) {
4057 continue;
4058 }
4059 goto finish;
4060 }
4061finish:
4062 return (rsm);
4063}
4064
4065static struct bbr_sendmap *
4066bbr_find_high_nonack(struct tcp_bbr *bbr, struct bbr_sendmap *rsm)
4067{
4068 struct bbr_sendmap *prsm;
4069
4070 /*
4071 * Walk the sequence order list backward until we hit and arrive at
4072 * the highest seq not acked. In theory when this is called it
4073 * should be the last segment (which it was not).
4074 */
4075 prsm = rsm;
4076 TAILQ_FOREACH_REVERSE_FROM(prsm, &bbr->r_ctl.rc_map, bbr_head, r_next) {
4077 if (prsm->r_flags & (BBR_ACKED | BBR_HAS_FIN)) {
4078 continue;
4079 }
4080 return (prsm);
4081 }
4082 return (NULL);
4083}
4084
4085/*
4086 * Returns to the caller the number of microseconds that
4087 * the packet can be outstanding before we think we
4088 * should have had an ack returned.
4089 */
4090static uint32_t
4091bbr_calc_thresh_rack(struct tcp_bbr *bbr, uint32_t srtt, uint32_t cts, struct bbr_sendmap *rsm)
4092{
4093 /*
4094 * lro is the flag we use to determine if we have seen reordering.
4095 * If it gets set we have seen reordering. The reorder logic either
4096 * works in one of two ways:
4097 *
4098 * If reorder-fade is configured, then we track the last time we saw
4099 * re-ordering occur. If we reach the point where enough time as
4100 * passed we no longer consider reordering has occuring.
4101 *
4102 * Or if reorder-face is 0, then once we see reordering we consider
4103 * the connection to alway be subject to reordering and just set lro
4104 * to 1.
4105 *
4106 * In the end if lro is non-zero we add the extra time for
4107 * reordering in.
4108 */
4109 int32_t lro;
4110 uint32_t thresh, t_rxtcur;
4111
4112 if (srtt == 0)
4113 srtt = 1;
4114 if (bbr->r_ctl.rc_reorder_ts) {
4115 if (bbr->r_ctl.rc_reorder_fade) {
4116 if (SEQ_GEQ(cts, bbr->r_ctl.rc_reorder_ts)) {
4117 lro = cts - bbr->r_ctl.rc_reorder_ts;
4118 if (lro == 0) {
4119 /*
4120 * No time as passed since the last
4121 * reorder, mark it as reordering.
4122 */
4123 lro = 1;
4124 }
4125 } else {
4126 /* Negative time? */
4127 lro = 0;
4128 }
4129 if (lro > bbr->r_ctl.rc_reorder_fade) {
4130 /* Turn off reordering seen too */
4131 bbr->r_ctl.rc_reorder_ts = 0;
4132 lro = 0;
4133 }
4134 } else {
4135 /* Reodering does not fade */
4136 lro = 1;
4137 }
4138 } else {
4139 lro = 0;
4140 }
4141 thresh = srtt + bbr->r_ctl.rc_pkt_delay;
4142 if (lro) {
4143 /* It must be set, if not you get 1/4 rtt */
4144 if (bbr->r_ctl.rc_reorder_shift)
4145 thresh += (srtt >> bbr->r_ctl.rc_reorder_shift);
4146 else
4147 thresh += (srtt >> 2);
4148 } else {
4149 thresh += 1000;
4150 }
4151 /* We don't let the rack timeout be above a RTO */
4152 if ((bbr->rc_tp)->t_srtt == 0)
4153 t_rxtcur = BBR_INITIAL_RTO;
4154 else
4155 t_rxtcur = TICKS_2_USEC(bbr->rc_tp->t_rxtcur);
4156 if (thresh > t_rxtcur) {
4157 thresh = t_rxtcur;
4158 }
4159 /* And we don't want it above the RTO max either */
4160 if (thresh > (((uint32_t)bbr->rc_max_rto_sec) * USECS_IN_SECOND)) {
4161 thresh = (((uint32_t)bbr->rc_max_rto_sec) * USECS_IN_SECOND);
4162 }
4163 bbr_log_thresh_choice(bbr, cts, thresh, lro, srtt, rsm, BBR_TO_FRM_RACK);
4164 return (thresh);
4165}
4166
4167/*
4168 * Return to the caller the amount of time in mico-seconds
4169 * that should be used for the TLP timer from the last
4170 * send time of this packet.
4171 */
4172static uint32_t
4173bbr_calc_thresh_tlp(struct tcpcb *tp, struct tcp_bbr *bbr,
4174 struct bbr_sendmap *rsm, uint32_t srtt,
4175 uint32_t cts)
4176{
4177 uint32_t thresh, len, maxseg, t_rxtcur;
4178 struct bbr_sendmap *prsm;
4179
4180 if (srtt == 0)
4181 srtt = 1;
4182 if (bbr->rc_tlp_threshold)
4183 thresh = srtt + (srtt / bbr->rc_tlp_threshold);
4184 else
4185 thresh = (srtt * 2);
4186 maxseg = tp->t_maxseg - bbr->rc_last_options;
4187 /* Get the previous sent packet, if any */
4188 len = rsm->r_end - rsm->r_start;
4189
4190 /* 2.1 behavior */
4191 prsm = TAILQ_PREV(rsm, bbr_head, r_tnext);
4192 if (prsm && (len <= maxseg)) {
4193 /*
4194 * Two packets outstanding, thresh should be (2*srtt) +
4195 * possible inter-packet delay (if any).
4196 */
4197 uint32_t inter_gap = 0;
4198 int idx, nidx;
4199
4200 idx = rsm->r_rtr_cnt - 1;
4201 nidx = prsm->r_rtr_cnt - 1;
4202 if (TSTMP_GEQ(rsm->r_tim_lastsent[nidx], prsm->r_tim_lastsent[idx])) {
4203 /* Yes it was sent later (or at the same time) */
4204 inter_gap = rsm->r_tim_lastsent[idx] - prsm->r_tim_lastsent[nidx];
4205 }
4206 thresh += inter_gap;
4207 } else if (len <= maxseg) {
4208 /*
4209 * Possibly compensate for delayed-ack.
4210 */
4211 uint32_t alt_thresh;
4212
4213 alt_thresh = srtt + (srtt / 2) + bbr_delayed_ack_time;
4214 if (alt_thresh > thresh)
4215 thresh = alt_thresh;
4216 }
4217 /* Not above the current RTO */
4218 if (tp->t_srtt == 0)
4219 t_rxtcur = BBR_INITIAL_RTO;
4220 else
4221 t_rxtcur = TICKS_2_USEC(tp->t_rxtcur);
4222
4223 bbr_log_thresh_choice(bbr, cts, thresh, t_rxtcur, srtt, rsm, BBR_TO_FRM_TLP);
4224 /* Not above an RTO */
4225 if (thresh > t_rxtcur) {
4226 thresh = t_rxtcur;
4227 }
4228 /* Not above a RTO max */
4229 if (thresh > (((uint32_t)bbr->rc_max_rto_sec) * USECS_IN_SECOND)) {
4230 thresh = (((uint32_t)bbr->rc_max_rto_sec) * USECS_IN_SECOND);
4231 }
4232 /* And now apply the user TLP min */
4233 if (thresh < bbr_tlp_min) {
4234 thresh = bbr_tlp_min;
4235 }
4236 return (thresh);
4237}
4238
4239/*
4240 * Return one of three RTTs to use (in microseconds).
4241 */
4242static __inline uint32_t
4243bbr_get_rtt(struct tcp_bbr *bbr, int32_t rtt_type)
4244{
4245 uint32_t f_rtt;
4246 uint32_t srtt;
4247
4248 f_rtt = get_filter_value_small(&bbr->r_ctl.rc_rttprop);
4249 if (get_filter_value_small(&bbr->r_ctl.rc_rttprop) == 0xffffffff) {
4250 /* We have no rtt at all */
4251 if (bbr->rc_tp->t_srtt == 0)
4252 f_rtt = BBR_INITIAL_RTO;
4253 else
4254 f_rtt = (TICKS_2_USEC(bbr->rc_tp->t_srtt) >> TCP_RTT_SHIFT);
4255 /*
4256 * Since we don't know how good the rtt is apply a
4257 * delayed-ack min
4258 */
4259 if (f_rtt < bbr_delayed_ack_time) {
4260 f_rtt = bbr_delayed_ack_time;
4261 }
4262 }
4263 /* Take the filter version or last measured pkt-rtt */
4264 if (rtt_type == BBR_RTT_PROP) {
4265 srtt = f_rtt;
4266 } else if (rtt_type == BBR_RTT_PKTRTT) {
4267 if (bbr->r_ctl.rc_pkt_epoch_rtt) {
4268 srtt = bbr->r_ctl.rc_pkt_epoch_rtt;
4269 } else {
4270 /* No pkt rtt yet */
4271 srtt = f_rtt;
4272 }
4273 } else if (rtt_type == BBR_RTT_RACK) {
4274 srtt = bbr->r_ctl.rc_last_rtt;
4275 /* We need to add in any internal delay for our timer */
4276 if (bbr->rc_ack_was_delayed)
4277 srtt += bbr->r_ctl.rc_ack_hdwr_delay;
4278 } else if (rtt_type == BBR_SRTT) {
4279 srtt = (TICKS_2_USEC(bbr->rc_tp->t_srtt) >> TCP_RTT_SHIFT);
4280 } else {
4281 /* TSNH */
4282 srtt = f_rtt;
4283#ifdef BBR_INVARIANTS
4284 panic("Unknown rtt request type %d", rtt_type);
4285#endif
4286 }
4287 return (srtt);
4288}
4289
4290static int
4291bbr_is_lost(struct tcp_bbr *bbr, struct bbr_sendmap *rsm, uint32_t cts)
4292{
4293 uint32_t thresh;
4294
4295 thresh = bbr_calc_thresh_rack(bbr, bbr_get_rtt(bbr, BBR_RTT_RACK),
4296 cts, rsm);
4297 if ((cts - rsm->r_tim_lastsent[(rsm->r_rtr_cnt - 1)]) >= thresh) {
4298 /* It is lost (past time) */
4299 return (1);
4300 }
4301 return (0);
4302}
4303
4304/*
4305 * Return a sendmap if we need to retransmit something.
4306 */
4307static struct bbr_sendmap *
4308bbr_check_recovery_mode(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts)
4309{
4310 /*
4311 * Check to see that we don't need to fall into recovery. We will
4312 * need to do so if our oldest transmit is past the time we should
4313 * have had an ack.
4314 */
4315
4316 struct bbr_sendmap *rsm;
4317 int32_t idx;
4318
4319 if (TAILQ_EMPTY(&bbr->r_ctl.rc_map)) {
4320 /* Nothing outstanding that we know of */
4321 return (NULL);
4322 }
4323 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap);
4324 if (rsm == NULL) {
4325 /* Nothing in the transmit map */
4326 return (NULL);
4327 }
4328 if (tp->t_flags & TF_SENTFIN) {
4329 /* Fin restricted, don't find anything once a fin is sent */
4330 return (NULL);
4331 }
4332 if (rsm->r_flags & BBR_ACKED) {
4333 /*
4334 * Ok the first one is acked (this really should not happen
4335 * since we remove the from the tmap once they are acked)
4336 */
4337 rsm = bbr_find_lowest_rsm(bbr);
4338 if (rsm == NULL)
4339 return (NULL);
4340 }
4341 idx = rsm->r_rtr_cnt - 1;
4342 if (SEQ_LEQ(cts, rsm->r_tim_lastsent[idx])) {
4343 /* Send timestamp is the same or less? can't be ready */
4344 return (NULL);
4345 }
4346 /* Get our RTT time */
4347 if (bbr_is_lost(bbr, rsm, cts) &&
4348 ((rsm->r_dupack >= DUP_ACK_THRESHOLD) ||
4349 (rsm->r_flags & BBR_SACK_PASSED))) {
4350 if ((rsm->r_flags & BBR_MARKED_LOST) == 0) {
4351 rsm->r_flags |= BBR_MARKED_LOST;
4352 bbr->r_ctl.rc_lost += rsm->r_end - rsm->r_start;
4353 bbr->r_ctl.rc_lost_bytes += rsm->r_end - rsm->r_start;
4354 }
4355 bbr_cong_signal(tp, NULL, CC_NDUPACK, rsm);
4356#ifdef BBR_INVARIANTS
4357 if ((rsm->r_end - rsm->r_start) == 0)
4358 panic("tp:%p bbr:%p rsm:%p length is 0?", tp, bbr, rsm);
4359#endif
4360 return (rsm);
4361 }
4362 return (NULL);
4363}
4364
4365/*
4366 * RACK Timer, here we simply do logging and house keeping.
4367 * the normal bbr_output_wtime() function will call the
4368 * appropriate thing to check if we need to do a RACK retransmit.
4369 * We return 1, saying don't proceed with bbr_output_wtime only
4370 * when all timers have been stopped (destroyed PCB?).
4371 */
4372static int
4373bbr_timeout_rack(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts)
4374{
4375 /*
4376 * This timer simply provides an internal trigger to send out data.
4377 * The check_recovery_mode call will see if there are needed
4378 * retransmissions, if so we will enter fast-recovery. The output
4379 * call may or may not do the same thing depending on sysctl
4380 * settings.
4381 */
4382 uint32_t lost;
4383
4384 if (bbr->rc_all_timers_stopped) {
4385 return (1);
4386 }
4387 if (TSTMP_LT(cts, bbr->r_ctl.rc_timer_exp)) {
4388 /* Its not time yet */
4389 return (0);
4390 }
4391 BBR_STAT_INC(bbr_to_tot);
4392 lost = bbr->r_ctl.rc_lost;
4393 if (bbr->r_state && (bbr->r_state != tp->t_state))
4394 bbr_set_state(tp, bbr, 0);
4395 bbr_log_to_event(bbr, cts, BBR_TO_FRM_RACK);
4396 if (bbr->r_ctl.rc_resend == NULL) {
4397 /* Lets do the check here */
4398 bbr->r_ctl.rc_resend = bbr_check_recovery_mode(tp, bbr, cts);
4399 }
4400 if (bbr_policer_call_from_rack_to)
4401 bbr_lt_bw_sampling(bbr, cts, (bbr->r_ctl.rc_lost > lost));
4402 bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_RACK;
4403 return (0);
4404}
4405
4406static __inline void
4407bbr_clone_rsm(struct tcp_bbr *bbr, struct bbr_sendmap *nrsm, struct bbr_sendmap *rsm, uint32_t start)
4408{
4409 int idx;
4410
4411 nrsm->r_start = start;
4412 nrsm->r_end = rsm->r_end;
4413 nrsm->r_rtr_cnt = rsm->r_rtr_cnt;
4414 nrsm-> r_rtt_not_allowed = rsm->r_rtt_not_allowed;
4415 nrsm->r_flags = rsm->r_flags;
4416 /* We don't transfer forward the SYN flag */
4417 nrsm->r_flags &= ~BBR_HAS_SYN;
4418 /* We move forward the FIN flag, not that this should happen */
4419 rsm->r_flags &= ~BBR_HAS_FIN;
4420 nrsm->r_dupack = rsm->r_dupack;
4421 nrsm->r_rtr_bytes = 0;
4422 nrsm->r_is_gain = rsm->r_is_gain;
4423 nrsm->r_is_drain = rsm->r_is_drain;
4424 nrsm->r_delivered = rsm->r_delivered;
4425 nrsm->r_ts_valid = rsm->r_ts_valid;
4426 nrsm->r_del_ack_ts = rsm->r_del_ack_ts;
4427 nrsm->r_del_time = rsm->r_del_time;
4428 nrsm->r_app_limited = rsm->r_app_limited;
4429 nrsm->r_first_sent_time = rsm->r_first_sent_time;
4430 nrsm->r_flight_at_send = rsm->r_flight_at_send;
4431 /* We split a piece the lower section looses any just_ret flag. */
4432 nrsm->r_bbr_state = rsm->r_bbr_state;
4433 for (idx = 0; idx < nrsm->r_rtr_cnt; idx++) {
4434 nrsm->r_tim_lastsent[idx] = rsm->r_tim_lastsent[idx];
4435 }
4436 rsm->r_end = nrsm->r_start;
4437 idx = min((bbr->rc_tp->t_maxseg - bbr->rc_last_options), bbr->r_ctl.rc_pace_max_segs);
4438 idx /= 8;
4439 /* Check if we got too small */
4440 if ((rsm->r_is_smallmap == 0) &&
4441 ((rsm->r_end - rsm->r_start) <= idx)) {
4442 bbr->r_ctl.rc_num_small_maps_alloced++;
4443 rsm->r_is_smallmap = 1;
4444 }
4445 /* Check the new one as well */
4446 if ((nrsm->r_end - nrsm->r_start) <= idx) {
4447 bbr->r_ctl.rc_num_small_maps_alloced++;
4448 nrsm->r_is_smallmap = 1;
4449 }
4450}
4451
4452static int
4453bbr_sack_mergable(struct bbr_sendmap *at,
4454 uint32_t start, uint32_t end)
4455{
4456 /*
4457 * Given a sack block defined by
4458 * start and end, and a current postion
4459 * at. Return 1 if either side of at
4460 * would show that the block is mergable
4461 * to that side. A block to be mergable
4462 * must have overlap with the start/end
4463 * and be in the SACK'd state.
4464 */
4465 struct bbr_sendmap *l_rsm;
4466 struct bbr_sendmap *r_rsm;
4467
4468 /* first get the either side blocks */
4469 l_rsm = TAILQ_PREV(at, bbr_head, r_next);
4470 r_rsm = TAILQ_NEXT(at, r_next);
4471 if (l_rsm && (l_rsm->r_flags & BBR_ACKED)) {
4472 /* Potentially mergeable */
4473 if ((l_rsm->r_end == start) ||
4474 (SEQ_LT(start, l_rsm->r_end) &&
4475 SEQ_GT(end, l_rsm->r_end))) {
4476 /*
4477 * map blk |------|
4478 * sack blk |------|
4479 * <or>
4480 * map blk |------|
4481 * sack blk |------|
4482 */
4483 return (1);
4484 }
4485 }
4486 if (r_rsm && (r_rsm->r_flags & BBR_ACKED)) {
4487 /* Potentially mergeable */
4488 if ((r_rsm->r_start == end) ||
4489 (SEQ_LT(start, r_rsm->r_start) &&
4490 SEQ_GT(end, r_rsm->r_start))) {
4491 /*
4492 * map blk |---------|
4493 * sack blk |----|
4494 * <or>
4495 * map blk |---------|
4496 * sack blk |-------|
4497 */
4498 return (1);
4499 }
4500 }
4501 return (0);
4502}
4503
4504static struct bbr_sendmap *
4505bbr_merge_rsm(struct tcp_bbr *bbr,
4506 struct bbr_sendmap *l_rsm,
4507 struct bbr_sendmap *r_rsm)
4508{
4509 /*
4510 * We are merging two ack'd RSM's,
4511 * the l_rsm is on the left (lower seq
4512 * values) and the r_rsm is on the right
4513 * (higher seq value). The simplest way
4514 * to merge these is to move the right
4515 * one into the left. I don't think there
4516 * is any reason we need to try to find
4517 * the oldest (or last oldest retransmitted).
4518 */
4519 l_rsm->r_end = r_rsm->r_end;
4520 if (l_rsm->r_dupack < r_rsm->r_dupack)
4521 l_rsm->r_dupack = r_rsm->r_dupack;
4522 if (r_rsm->r_rtr_bytes)
4523 l_rsm->r_rtr_bytes += r_rsm->r_rtr_bytes;
4524 if (r_rsm->r_in_tmap) {
4525 /* This really should not happen */
4526 TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, r_rsm, r_tnext);
4527 }
4528 if (r_rsm->r_app_limited)
4529 l_rsm->r_app_limited = r_rsm->r_app_limited;
4530 /* Now the flags */
4531 if (r_rsm->r_flags & BBR_HAS_FIN)
4532 l_rsm->r_flags |= BBR_HAS_FIN;
4533 if (r_rsm->r_flags & BBR_TLP)
4534 l_rsm->r_flags |= BBR_TLP;
4535 if (r_rsm->r_flags & BBR_RWND_COLLAPSED)
4536 l_rsm->r_flags |= BBR_RWND_COLLAPSED;
4537 if (r_rsm->r_flags & BBR_MARKED_LOST) {
4538 /* This really should not happen */
4539 bbr->r_ctl.rc_lost_bytes -= r_rsm->r_end - r_rsm->r_start;
4540 }
4541 TAILQ_REMOVE(&bbr->r_ctl.rc_map, r_rsm, r_next);
4542 if ((r_rsm->r_limit_type == 0) && (l_rsm->r_limit_type != 0)) {
4543 /* Transfer the split limit to the map we free */
4544 r_rsm->r_limit_type = l_rsm->r_limit_type;
4545 l_rsm->r_limit_type = 0;
4546 }
4547 bbr_free(bbr, r_rsm);
4548 return(l_rsm);
4549}
4550
4551/*
4552 * TLP Timer, here we simply setup what segment we want to
4553 * have the TLP expire on, the normal bbr_output_wtime() will then
4554 * send it out.
4555 *
4556 * We return 1, saying don't proceed with bbr_output_wtime only
4557 * when all timers have been stopped (destroyed PCB?).
4558 */
4559static int
4560bbr_timeout_tlp(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts)
4561{
4562 /*
4563 * Tail Loss Probe.
4564 */
4565 struct bbr_sendmap *rsm = NULL;
4566 struct socket *so;
4567 uint32_t amm;
4568 uint32_t out, avail;
4569 uint32_t maxseg;
4570 int collapsed_win = 0;
4571
4572 if (bbr->rc_all_timers_stopped) {
4573 return (1);
4574 }
4575 if (TSTMP_LT(cts, bbr->r_ctl.rc_timer_exp)) {
4576 /* Its not time yet */
4577 return (0);
4578 }
4579 if (ctf_progress_timeout_check(tp, true)) {
4580 bbr_log_progress_event(bbr, tp, tick, PROGRESS_DROP, __LINE__);
4581 return (-ETIMEDOUT); /* tcp_drop() */
4582 }
4583 /* Did we somehow get into persists? */
4584 if (bbr->rc_in_persist) {
4585 return (0);
4586 }
4587 if (bbr->r_state && (bbr->r_state != tp->t_state))
4588 bbr_set_state(tp, bbr, 0);
4589 BBR_STAT_INC(bbr_tlp_tot);
4590 maxseg = tp->t_maxseg - bbr->rc_last_options;
4591 /*
4592 * A TLP timer has expired. We have been idle for 2 rtts. So we now
4593 * need to figure out how to force a full MSS segment out.
4594 */
4595 so = tp->t_inpcb->inp_socket;
4596 avail = sbavail(&so->so_snd);
4597 out = ctf_outstanding(tp);
4598 if (out > tp->snd_wnd) {
4599 /* special case, we need a retransmission */
4600 collapsed_win = 1;
4601 goto need_retran;
4602 }
4603 if (avail > out) {
4604 /* New data is available */
4605 amm = avail - out;
4606 if (amm > maxseg) {
4607 amm = maxseg;
4608 } else if ((amm < maxseg) && ((tp->t_flags & TF_NODELAY) == 0)) {
4609 /* not enough to fill a MTU and no-delay is off */
4610 goto need_retran;
4611 }
4612 /* Set the send-new override */
4613 if ((out + amm) <= tp->snd_wnd) {
4614 bbr->rc_tlp_new_data = 1;
4615 } else {
4616 goto need_retran;
4617 }
4618 bbr->r_ctl.rc_tlp_seg_send_cnt = 0;
4619 bbr->r_ctl.rc_last_tlp_seq = tp->snd_max;
4620 bbr->r_ctl.rc_tlp_send = NULL;
4621 /* cap any slots */
4622 BBR_STAT_INC(bbr_tlp_newdata);
4623 goto send;
4624 }
4625need_retran:
4626 /*
4627 * Ok we need to arrange the last un-acked segment to be re-sent, or
4628 * optionally the first un-acked segment.
4629 */
4630 if (collapsed_win == 0) {
4631 rsm = TAILQ_LAST_FAST(&bbr->r_ctl.rc_map, bbr_sendmap, r_next);
4632 if (rsm && (BBR_ACKED | BBR_HAS_FIN)) {
4633 rsm = bbr_find_high_nonack(bbr, rsm);
4634 }
4635 if (rsm == NULL) {
4636 goto restore;
4637 }
4638 } else {
4639 /*
4640 * We must find the last segment
4641 * that was acceptable by the client.
4642 */
4643 TAILQ_FOREACH_REVERSE(rsm, &bbr->r_ctl.rc_map, bbr_head, r_next) {
4644 if ((rsm->r_flags & BBR_RWND_COLLAPSED) == 0) {
4645 /* Found one */
4646 break;
4647 }
4648 }
4649 if (rsm == NULL) {
4650 /* None? if so send the first */
4651 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
4652 if (rsm == NULL)
4653 goto restore;
4654 }
4655 }
4656 if ((rsm->r_end - rsm->r_start) > maxseg) {
4657 /*
4658 * We need to split this the last segment in two.
4659 */
4660 struct bbr_sendmap *nrsm;
4661
4662 nrsm = bbr_alloc_full_limit(bbr);
4663 if (nrsm == NULL) {
4664 /*
4665 * We can't get memory to split, we can either just
4666 * not split it. Or retransmit the whole piece, lets
4667 * do the large send (BTLP :-) ).
4668 */
4669 goto go_for_it;
4670 }
4671 bbr_clone_rsm(bbr, nrsm, rsm, (rsm->r_end - maxseg));
4672 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next);
4673 if (rsm->r_in_tmap) {
4674 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext);
4675 nrsm->r_in_tmap = 1;
4676 }
4677 rsm->r_flags &= (~BBR_HAS_FIN);
4678 rsm = nrsm;
4679 }
4680go_for_it:
4681 bbr->r_ctl.rc_tlp_send = rsm;
4682 bbr->rc_tlp_rtx_out = 1;
4683 if (rsm->r_start == bbr->r_ctl.rc_last_tlp_seq) {
4684 bbr->r_ctl.rc_tlp_seg_send_cnt++;
4685 tp->t_rxtshift++;
4686 } else {
4687 bbr->r_ctl.rc_last_tlp_seq = rsm->r_start;
4688 bbr->r_ctl.rc_tlp_seg_send_cnt = 1;
4689 }
4690send:
4691 if (bbr->r_ctl.rc_tlp_seg_send_cnt > bbr_tlp_max_resend) {
4692 /*
4693 * Can't [re]/transmit a segment we have retranmitted the
4694 * max times. We need the retransmit timer to take over.
4695 */
4696restore:
4697 bbr->rc_tlp_new_data = 0;
4698 bbr->r_ctl.rc_tlp_send = NULL;
4699 if (rsm)
4700 rsm->r_flags &= ~BBR_TLP;
4701 BBR_STAT_INC(bbr_tlp_retran_fail);
4702 return (0);
4703 } else if (rsm) {
4704 rsm->r_flags |= BBR_TLP;
4705 }
4706 if (rsm && (rsm->r_start == bbr->r_ctl.rc_last_tlp_seq) &&
4707 (bbr->r_ctl.rc_tlp_seg_send_cnt > bbr_tlp_max_resend)) {
4708 /*
4709 * We have retransmitted to many times for TLP. Switch to
4710 * the regular RTO timer
4711 */
4712 goto restore;
4713 }
4714 bbr_log_to_event(bbr, cts, BBR_TO_FRM_TLP);
4715 bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_TLP;
4716 return (0);
4717}
4718
4719/*
4720 * Delayed ack Timer, here we simply need to setup the
4721 * ACK_NOW flag and remove the DELACK flag. From there
4722 * the output routine will send the ack out.
4723 *
4724 * We only return 1, saying don't proceed, if all timers
4725 * are stopped (destroyed PCB?).
4726 */
4727static int
4728bbr_timeout_delack(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts)
4729{
4730 if (bbr->rc_all_timers_stopped) {
4731 return (1);
4732 }
4733 bbr_log_to_event(bbr, cts, BBR_TO_FRM_DELACK);
4734 tp->t_flags &= ~TF_DELACK;
4735 tp->t_flags |= TF_ACKNOW;
4736 KMOD_TCPSTAT_INC(tcps_delack);
4737 bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_DELACK;
4738 return (0);
4739}
4740
4741/*
4742 * Here we send a KEEP-ALIVE like probe to the
4743 * peer, we do not send data.
4744 *
4745 * We only return 1, saying don't proceed, if all timers
4746 * are stopped (destroyed PCB?).
4747 */
4748static int
4749bbr_timeout_persist(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts)
4750{
4751 struct tcptemp *t_template;
4752 int32_t retval = 1;
4753
4754 if (bbr->rc_all_timers_stopped) {
4755 return (1);
4756 }
4757 if (bbr->rc_in_persist == 0)
4758 return (0);
4759 KASSERT(tp->t_inpcb != NULL,
4760 ("%s: tp %p tp->t_inpcb == NULL", __func__, tp));
4761 /*
4762 * Persistence timer into zero window. Force a byte to be output, if
4763 * possible.
4764 */
4765 bbr_log_to_event(bbr, cts, BBR_TO_FRM_PERSIST);
4766 bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_PERSIT;
4767 KMOD_TCPSTAT_INC(tcps_persisttimeo);
4768 /*
4769 * Have we exceeded the user specified progress time?
4770 */
4771 if (ctf_progress_timeout_check(tp, true)) {
4772 bbr_log_progress_event(bbr, tp, tick, PROGRESS_DROP, __LINE__);
4773 return (-ETIMEDOUT); /* tcp_drop() */
4774 }
4775 /*
4776 * Hack: if the peer is dead/unreachable, we do not time out if the
4777 * window is closed. After a full backoff, drop the connection if
4778 * the idle time (no responses to probes) reaches the maximum
4779 * backoff that we would use if retransmitting.
4780 */
4781 if (tp->t_rxtshift == TCP_MAXRXTSHIFT &&
4782 (ticks - tp->t_rcvtime >= tcp_maxpersistidle ||
4783 ticks - tp->t_rcvtime >= TCP_REXMTVAL(tp) * tcp_totbackoff)) {
4784 KMOD_TCPSTAT_INC(tcps_persistdrop);
4785 tcp_log_end_status(tp, TCP_EI_STATUS_PERSIST_MAX);
4786 return (-ETIMEDOUT); /* tcp_drop() */
4787 }
4788 if ((sbavail(&bbr->rc_inp->inp_socket->so_snd) == 0) &&
4789 tp->snd_una == tp->snd_max) {
4790 bbr_exit_persist(tp, bbr, cts, __LINE__);
4791 retval = 0;
4792 goto out;
4793 }
4794 /*
4795 * If the user has closed the socket then drop a persisting
4796 * connection after a much reduced timeout.
4797 */
4798 if (tp->t_state > TCPS_CLOSE_WAIT &&
4799 (ticks - tp->t_rcvtime) >= TCPTV_PERSMAX) {
4800 KMOD_TCPSTAT_INC(tcps_persistdrop);
4801 tcp_log_end_status(tp, TCP_EI_STATUS_PERSIST_MAX);
4802 return (-ETIMEDOUT); /* tcp_drop() */
4803 }
4804 t_template = tcpip_maketemplate(bbr->rc_inp);
4805 if (t_template) {
4806 tcp_respond(tp, t_template->tt_ipgen,
4807 &t_template->tt_t, (struct mbuf *)NULL,
4808 tp->rcv_nxt, tp->snd_una - 1, 0);
4809 /* This sends an ack */
4810 if (tp->t_flags & TF_DELACK)
4811 tp->t_flags &= ~TF_DELACK;
4812 free(t_template, M_TEMP);
4813 }
4814 if (tp->t_rxtshift < TCP_MAXRXTSHIFT)
4815 tp->t_rxtshift++;
4816 bbr_start_hpts_timer(bbr, tp, cts, 3, 0, 0);
4817out:
4818 return (retval);
4819}
4820
4821/*
4822 * If a keepalive goes off, we had no other timers
4823 * happening. We always return 1 here since this
4824 * routine either drops the connection or sends
4825 * out a segment with respond.
4826 */
4827static int
4828bbr_timeout_keepalive(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts)
4829{
4830 struct tcptemp *t_template;
4831 struct inpcb *inp;
4832
4833 if (bbr->rc_all_timers_stopped) {
4834 return (1);
4835 }
4836 bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_KEEP;
4837 inp = tp->t_inpcb;
4838 bbr_log_to_event(bbr, cts, BBR_TO_FRM_KEEP);
4839 /*
4840 * Keep-alive timer went off; send something or drop connection if
4841 * idle for too long.
4842 */
4843 KMOD_TCPSTAT_INC(tcps_keeptimeo);
4844 if (tp->t_state < TCPS_ESTABLISHED)
4845 goto dropit;
4846 if ((V_tcp_always_keepalive || inp->inp_socket->so_options & SO_KEEPALIVE) &&
4847 tp->t_state <= TCPS_CLOSING) {
4848 if (ticks - tp->t_rcvtime >= TP_KEEPIDLE(tp) + TP_MAXIDLE(tp))
4849 goto dropit;
4850 /*
4851 * Send a packet designed to force a response if the peer is
4852 * up and reachable: either an ACK if the connection is
4853 * still alive, or an RST if the peer has closed the
4854 * connection due to timeout or reboot. Using sequence
4855 * number tp->snd_una-1 causes the transmitted zero-length
4856 * segment to lie outside the receive window; by the
4857 * protocol spec, this requires the correspondent TCP to
4858 * respond.
4859 */
4860 KMOD_TCPSTAT_INC(tcps_keepprobe);
4861 t_template = tcpip_maketemplate(inp);
4862 if (t_template) {
4863 tcp_respond(tp, t_template->tt_ipgen,
4864 &t_template->tt_t, (struct mbuf *)NULL,
4865 tp->rcv_nxt, tp->snd_una - 1, 0);
4866 free(t_template, M_TEMP);
4867 }
4868 }
4869 bbr_start_hpts_timer(bbr, tp, cts, 4, 0, 0);
4870 return (1);
4871dropit:
4872 KMOD_TCPSTAT_INC(tcps_keepdrops);
4873 tcp_log_end_status(tp, TCP_EI_STATUS_KEEP_MAX);
4874 return (-ETIMEDOUT); /* tcp_drop() */
4875}
4876
4877/*
4878 * Retransmit helper function, clear up all the ack
4879 * flags and take care of important book keeping.
4880 */
4881static void
4882bbr_remxt_tmr(struct tcpcb *tp)
4883{
4884 /*
4885 * The retransmit timer went off, all sack'd blocks must be
4886 * un-acked.
4887 */
4888 struct bbr_sendmap *rsm, *trsm = NULL;
4889 struct tcp_bbr *bbr;
4890 uint32_t cts, lost;
4891
4892 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
4893 cts = tcp_get_usecs(&bbr->rc_tv);
4894 lost = bbr->r_ctl.rc_lost;
4895 if (bbr->r_state && (bbr->r_state != tp->t_state))
4896 bbr_set_state(tp, bbr, 0);
4897
4898 TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) {
4899 if (rsm->r_flags & BBR_ACKED) {
4900 uint32_t old_flags;
4901
4902 rsm->r_dupack = 0;
4903 if (rsm->r_in_tmap == 0) {
4904 /* We must re-add it back to the tlist */
4905 if (trsm == NULL) {
4906 TAILQ_INSERT_HEAD(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
4907 } else {
4908 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, trsm, rsm, r_tnext);
4909 }
4910 rsm->r_in_tmap = 1;
4911 }
4912 old_flags = rsm->r_flags;
4913 rsm->r_flags |= BBR_RXT_CLEARED;
4914 rsm->r_flags &= ~(BBR_ACKED | BBR_SACK_PASSED | BBR_WAS_SACKPASS);
4915 bbr_log_type_rsmclear(bbr, cts, rsm, old_flags, __LINE__);
4916 } else {
4917 if ((tp->t_state < TCPS_ESTABLISHED) &&
4918 (rsm->r_start == tp->snd_una)) {
4919 /*
4920 * Special case for TCP FO. Where
4921 * we sent more data beyond the snd_max.
4922 * We don't mark that as lost and stop here.
4923 */
4924 break;
4925 }
4926 if ((rsm->r_flags & BBR_MARKED_LOST) == 0) {
4927 bbr->r_ctl.rc_lost += rsm->r_end - rsm->r_start;
4928 bbr->r_ctl.rc_lost_bytes += rsm->r_end - rsm->r_start;
4929 }
4930 if (bbr_marks_rxt_sack_passed) {
4931 /*
4932 * With this option, we will rack out
4933 * in 1ms increments the rest of the packets.
4934 */
4935 rsm->r_flags |= BBR_SACK_PASSED | BBR_MARKED_LOST;
4936 rsm->r_flags &= ~BBR_WAS_SACKPASS;
4937 } else {
4938 /*
4939 * With this option we only mark them lost
4940 * and remove all sack'd markings. We will run
4941 * another RXT or a TLP. This will cause
4942 * us to eventually send more based on what
4943 * ack's come in.
4944 */
4945 rsm->r_flags |= BBR_MARKED_LOST;
4946 rsm->r_flags &= ~BBR_WAS_SACKPASS;
4947 rsm->r_flags &= ~BBR_SACK_PASSED;
4948 }
4949 }
4950 trsm = rsm;
4951 }
4952 bbr->r_ctl.rc_resend = TAILQ_FIRST(&bbr->r_ctl.rc_map);
4953 /* Clear the count (we just un-acked them) */
4954 bbr_log_to_event(bbr, cts, BBR_TO_FRM_TMR);
4955 bbr->rc_tlp_new_data = 0;
4956 bbr->r_ctl.rc_tlp_seg_send_cnt = 0;
4957 /* zap the behindness on a rxt */
4958 bbr->r_ctl.rc_hptsi_agg_delay = 0;
4959 bbr->r_agg_early_set = 0;
4960 bbr->r_ctl.rc_agg_early = 0;
4961 bbr->rc_tlp_rtx_out = 0;
4962 bbr->r_ctl.rc_sacked = 0;
4963 bbr->r_ctl.rc_sacklast = NULL;
4964 bbr->r_timer_override = 1;
4965 bbr_lt_bw_sampling(bbr, cts, (bbr->r_ctl.rc_lost > lost));
4966}
4967
4968/*
4969 * Re-transmit timeout! If we drop the PCB we will return 1, otherwise
4970 * we will setup to retransmit the lowest seq number outstanding.
4971 */
4972static int
4973bbr_timeout_rxt(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts)
4974{
4975 int32_t rexmt;
4976 int32_t retval = 0;
4977 bool isipv6;
4978
4979 bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_RXT;
4980 if (bbr->rc_all_timers_stopped) {
4981 return (1);
4982 }
4983 if (TCPS_HAVEESTABLISHED(tp->t_state) &&
4984 (tp->snd_una == tp->snd_max)) {
4985 /* Nothing outstanding .. nothing to do */
4986 return (0);
4987 }
4988 /*
4989 * Retransmission timer went off. Message has not been acked within
4990 * retransmit interval. Back off to a longer retransmit interval
4991 * and retransmit one segment.
4992 */
4993 if (ctf_progress_timeout_check(tp, true)) {
4994 bbr_log_progress_event(bbr, tp, tick, PROGRESS_DROP, __LINE__);
4995 return (-ETIMEDOUT); /* tcp_drop() */
4996 }
4997 bbr_remxt_tmr(tp);
4998 if ((bbr->r_ctl.rc_resend == NULL) ||
4999 ((bbr->r_ctl.rc_resend->r_flags & BBR_RWND_COLLAPSED) == 0)) {
5000 /*
5001 * If the rwnd collapsed on
5002 * the one we are retransmitting
5003 * it does not count against the
5004 * rxt count.
5005 */
5006 tp->t_rxtshift++;
5007 }
5008 if (tp->t_rxtshift > TCP_MAXRXTSHIFT) {
5009 tp->t_rxtshift = TCP_MAXRXTSHIFT;
5010 KMOD_TCPSTAT_INC(tcps_timeoutdrop);
5011 tcp_log_end_status(tp, TCP_EI_STATUS_RETRAN);
5012 /* XXXGL: previously t_softerror was casted to uint16_t */
5013 MPASS(tp->t_softerror >= 0);
5014 retval = tp->t_softerror ? -tp->t_softerror : -ETIMEDOUT;
5015 return (retval); /* tcp_drop() */
5016 }
5017 if (tp->t_state == TCPS_SYN_SENT) {
5018 /*
5019 * If the SYN was retransmitted, indicate CWND to be limited
5020 * to 1 segment in cc_conn_init().
5021 */
5022 tp->snd_cwnd = 1;
5023 } else if (tp->t_rxtshift == 1) {
5024 /*
5025 * first retransmit; record ssthresh and cwnd so they can be
5026 * recovered if this turns out to be a "bad" retransmit. A
5027 * retransmit is considered "bad" if an ACK for this segment
5028 * is received within RTT/2 interval; the assumption here is
5029 * that the ACK was already in flight. See "On Estimating
5030 * End-to-End Network Path Properties" by Allman and Paxson
5031 * for more details.
5032 */
5033 tp->snd_cwnd = tp->t_maxseg - bbr->rc_last_options;
5034 if (!IN_RECOVERY(tp->t_flags)) {
5035 tp->snd_cwnd_prev = tp->snd_cwnd;
5036 tp->snd_ssthresh_prev = tp->snd_ssthresh;
5037 tp->snd_recover_prev = tp->snd_recover;
5038 tp->t_badrxtwin = ticks + (tp->t_srtt >> (TCP_RTT_SHIFT + 1));
5039 tp->t_flags |= TF_PREVVALID;
5040 } else {
5041 tp->t_flags &= ~TF_PREVVALID;
5042 }
5043 tp->snd_cwnd = tp->t_maxseg - bbr->rc_last_options;
5044 } else {
5045 tp->snd_cwnd = tp->t_maxseg - bbr->rc_last_options;
5046 tp->t_flags &= ~TF_PREVVALID;
5047 }
5048 KMOD_TCPSTAT_INC(tcps_rexmttimeo);
5049 if ((tp->t_state == TCPS_SYN_SENT) ||
5050 (tp->t_state == TCPS_SYN_RECEIVED))
5051 rexmt = USEC_2_TICKS(BBR_INITIAL_RTO) * tcp_backoff[tp->t_rxtshift];
5052 else
5053 rexmt = TCP_REXMTVAL(tp) * tcp_backoff[tp->t_rxtshift];
5054 TCPT_RANGESET(tp->t_rxtcur, rexmt,
5055 MSEC_2_TICKS(bbr->r_ctl.rc_min_rto_ms),
5056 MSEC_2_TICKS(((uint32_t)bbr->rc_max_rto_sec) * 1000));
5057 /*
5058 * We enter the path for PLMTUD if connection is established or, if
5059 * connection is FIN_WAIT_1 status, reason for the last is that if
5060 * amount of data we send is very small, we could send it in couple
5061 * of packets and process straight to FIN. In that case we won't
5062 * catch ESTABLISHED state.
5063 */
5064#ifdef INET6
5065 isipv6 = (tp->t_inpcb->inp_vflag & INP_IPV6) ? true : false;
5066#else
5067 isipv6 = false;
5068#endif
5069 if (((V_tcp_pmtud_blackhole_detect == 1) ||
5070 (V_tcp_pmtud_blackhole_detect == 2 && !isipv6) ||
5071 (V_tcp_pmtud_blackhole_detect == 3 && isipv6)) &&
5072 ((tp->t_state == TCPS_ESTABLISHED) ||
5073 (tp->t_state == TCPS_FIN_WAIT_1))) {
5074 /*
5075 * Idea here is that at each stage of mtu probe (usually,
5076 * 1448 -> 1188 -> 524) should be given 2 chances to recover
5077 * before further clamping down. 'tp->t_rxtshift % 2 == 0'
5078 * should take care of that.
5079 */
5080 if (((tp->t_flags2 & (TF2_PLPMTU_PMTUD | TF2_PLPMTU_MAXSEGSNT)) ==
5081 (TF2_PLPMTU_PMTUD | TF2_PLPMTU_MAXSEGSNT)) &&
5082 (tp->t_rxtshift >= 2 && tp->t_rxtshift < 6 &&
5083 tp->t_rxtshift % 2 == 0)) {
5084 /*
5085 * Enter Path MTU Black-hole Detection mechanism: -
5086 * Disable Path MTU Discovery (IP "DF" bit). -
5087 * Reduce MTU to lower value than what we negotiated
5088 * with peer.
5089 */
5090 if ((tp->t_flags2 & TF2_PLPMTU_BLACKHOLE) == 0) {
5091 /*
5092 * Record that we may have found a black
5093 * hole.
5094 */
5095 tp->t_flags2 |= TF2_PLPMTU_BLACKHOLE;
5096 /* Keep track of previous MSS. */
5097 tp->t_pmtud_saved_maxseg = tp->t_maxseg;
5098 }
5099 /*
5100 * Reduce the MSS to blackhole value or to the
5101 * default in an attempt to retransmit.
5102 */
5103#ifdef INET6
5104 isipv6 = bbr->r_is_v6;
5105 if (isipv6 &&
5106 tp->t_maxseg > V_tcp_v6pmtud_blackhole_mss) {
5107 /* Use the sysctl tuneable blackhole MSS. */
5108 tp->t_maxseg = V_tcp_v6pmtud_blackhole_mss;
5109 KMOD_TCPSTAT_INC(tcps_pmtud_blackhole_activated);
5110 } else if (isipv6) {
5111 /* Use the default MSS. */
5112 tp->t_maxseg = V_tcp_v6mssdflt;
5113 /*
5114 * Disable Path MTU Discovery when we switch
5115 * to minmss.
5116 */
5117 tp->t_flags2 &= ~TF2_PLPMTU_PMTUD;
5118 KMOD_TCPSTAT_INC(tcps_pmtud_blackhole_activated_min_mss);
5119 }
5120#endif
5121#if defined(INET6) && defined(INET)
5122 else
5123#endif
5124#ifdef INET
5125 if (tp->t_maxseg > V_tcp_pmtud_blackhole_mss) {
5126 /* Use the sysctl tuneable blackhole MSS. */
5127 tp->t_maxseg = V_tcp_pmtud_blackhole_mss;
5128 KMOD_TCPSTAT_INC(tcps_pmtud_blackhole_activated);
5129 } else {
5130 /* Use the default MSS. */
5131 tp->t_maxseg = V_tcp_mssdflt;
5132 /*
5133 * Disable Path MTU Discovery when we switch
5134 * to minmss.
5135 */
5136 tp->t_flags2 &= ~TF2_PLPMTU_PMTUD;
5137 KMOD_TCPSTAT_INC(tcps_pmtud_blackhole_activated_min_mss);
5138 }
5139#endif
5140 } else {
5141 /*
5142 * If further retransmissions are still unsuccessful
5143 * with a lowered MTU, maybe this isn't a blackhole
5144 * and we restore the previous MSS and blackhole
5145 * detection flags. The limit '6' is determined by
5146 * giving each probe stage (1448, 1188, 524) 2
5147 * chances to recover.
5148 */
5149 if ((tp->t_flags2 & TF2_PLPMTU_BLACKHOLE) &&
5150 (tp->t_rxtshift >= 6)) {
5151 tp->t_flags2 |= TF2_PLPMTU_PMTUD;
5152 tp->t_flags2 &= ~TF2_PLPMTU_BLACKHOLE;
5153 tp->t_maxseg = tp->t_pmtud_saved_maxseg;
5154 KMOD_TCPSTAT_INC(tcps_pmtud_blackhole_failed);
5155 }
5156 }
5157 }
5158 /*
5159 * Disable RFC1323 and SACK if we haven't got any response to our
5160 * third SYN to work-around some broken terminal servers (most of
5161 * which have hopefully been retired) that have bad VJ header
5162 * compression code which trashes TCP segments containing
5163 * unknown-to-them TCP options.
5164 */
5165 if (tcp_rexmit_drop_options && (tp->t_state == TCPS_SYN_SENT) &&
5166 (tp->t_rxtshift == 3))
5167 tp->t_flags &= ~(TF_REQ_SCALE | TF_REQ_TSTMP | TF_SACK_PERMIT);
5168 /*
5169 * If we backed off this far, our srtt estimate is probably bogus.
5170 * Clobber it so we'll take the next rtt measurement as our srtt;
5171 * move the current srtt into rttvar to keep the current retransmit
5172 * times until then.
5173 */
5174 if (tp->t_rxtshift > TCP_MAXRXTSHIFT / 4) {
5175#ifdef INET6
5176 if (bbr->r_is_v6)
5177 in6_losing(tp->t_inpcb);
5178 else
5179#endif
5180 in_losing(tp->t_inpcb);
5181 tp->t_rttvar += (tp->t_srtt >> TCP_RTT_SHIFT);
5182 tp->t_srtt = 0;
5183 }
5184 sack_filter_clear(&bbr->r_ctl.bbr_sf, tp->snd_una);
5185 tp->snd_recover = tp->snd_max;
5186 tp->t_flags |= TF_ACKNOW;
5187 tp->t_rtttime = 0;
5188
5189 return (retval);
5190}
5191
5192static int
5193bbr_process_timers(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts, uint8_t hpts_calling)
5194{
5195 int32_t ret = 0;
5196 int32_t timers = (bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK);
5197
5198 if (timers == 0) {
5199 return (0);
5200 }
5201 if (tp->t_state == TCPS_LISTEN) {
5202 /* no timers on listen sockets */
5203 if (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)
5204 return (0);
5205 return (1);
5206 }
5207 if (TSTMP_LT(cts, bbr->r_ctl.rc_timer_exp)) {
5208 uint32_t left;
5209
5210 if (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) {
5211 ret = -1;
5212 bbr_log_to_processing(bbr, cts, ret, 0, hpts_calling);
5213 return (0);
5214 }
5215 if (hpts_calling == 0) {
5216 ret = -2;
5217 bbr_log_to_processing(bbr, cts, ret, 0, hpts_calling);
5218 return (0);
5219 }
5220 /*
5221 * Ok our timer went off early and we are not paced false
5222 * alarm, go back to sleep.
5223 */
5224 left = bbr->r_ctl.rc_timer_exp - cts;
5225 ret = -3;
5226 bbr_log_to_processing(bbr, cts, ret, left, hpts_calling);
5227 tcp_hpts_insert(tp->t_inpcb, HPTS_USEC_TO_SLOTS(left));
5228 return (1);
5229 }
5230 bbr->rc_tmr_stopped = 0;
5231 bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_MASK;
5232 if (timers & PACE_TMR_DELACK) {
5233 ret = bbr_timeout_delack(tp, bbr, cts);
5234 } else if (timers & PACE_TMR_PERSIT) {
5235 ret = bbr_timeout_persist(tp, bbr, cts);
5236 } else if (timers & PACE_TMR_RACK) {
5237 bbr->r_ctl.rc_tlp_rxt_last_time = cts;
5238 ret = bbr_timeout_rack(tp, bbr, cts);
5239 } else if (timers & PACE_TMR_TLP) {
5240 bbr->r_ctl.rc_tlp_rxt_last_time = cts;
5241 ret = bbr_timeout_tlp(tp, bbr, cts);
5242 } else if (timers & PACE_TMR_RXT) {
5243 bbr->r_ctl.rc_tlp_rxt_last_time = cts;
5244 ret = bbr_timeout_rxt(tp, bbr, cts);
5245 } else if (timers & PACE_TMR_KEEP) {
5246 ret = bbr_timeout_keepalive(tp, bbr, cts);
5247 }
5248 bbr_log_to_processing(bbr, cts, ret, timers, hpts_calling);
5249 return (ret);
5250}
5251
5252static void
5253bbr_timer_cancel(struct tcp_bbr *bbr, int32_t line, uint32_t cts)
5254{
5255 if (bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK) {
5256 uint8_t hpts_removed = 0;
5257
5258 if (tcp_in_hpts(bbr->rc_inp) &&
5259 (bbr->rc_timer_first == 1)) {
5260 /*
5261 * If we are canceling timer's when we have the
5262 * timer ahead of the output being paced. We also
5263 * must remove ourselves from the hpts.
5264 */
5265 hpts_removed = 1;
5266 tcp_hpts_remove(bbr->rc_inp);
5267 if (bbr->r_ctl.rc_last_delay_val) {
5268 /* Update the last hptsi delay too */
5269 uint32_t time_since_send;
5270
5271 if (TSTMP_GT(cts, bbr->rc_pacer_started))
5272 time_since_send = cts - bbr->rc_pacer_started;
5273 else
5274 time_since_send = 0;
5275 if (bbr->r_ctl.rc_last_delay_val > time_since_send) {
5276 /* Cut down our slot time */
5277 bbr->r_ctl.rc_last_delay_val -= time_since_send;
5278 } else {
5279 bbr->r_ctl.rc_last_delay_val = 0;
5280 }
5281 bbr->rc_pacer_started = cts;
5282 }
5283 }
5284 bbr->rc_timer_first = 0;
5285 bbr_log_to_cancel(bbr, line, cts, hpts_removed);
5286 bbr->rc_tmr_stopped = bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK;
5287 bbr->r_ctl.rc_hpts_flags &= ~(PACE_TMR_MASK);
5288 }
5289}
5290
5291static void
5292bbr_timer_stop(struct tcpcb *tp, uint32_t timer_type)
5293{
5294 struct tcp_bbr *bbr;
5295
5296 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
5297 bbr->rc_all_timers_stopped = 1;
5298 return;
5299}
5300
5301/*
5302 * stop all timers always returning 0.
5303 */
5304static int
5305bbr_stopall(struct tcpcb *tp)
5306{
5307 return (0);
5308}
5309
5310static void
5311bbr_timer_activate(struct tcpcb *tp, uint32_t timer_type, uint32_t delta)
5312{
5313 return;
5314}
5315
5316/*
5317 * return true if a bbr timer (rack or tlp) is active.
5318 */
5319static int
5320bbr_timer_active(struct tcpcb *tp, uint32_t timer_type)
5321{
5322 return (0);
5323}
5324
5325static uint32_t
5326bbr_get_earliest_send_outstanding(struct tcp_bbr *bbr, struct bbr_sendmap *u_rsm, uint32_t cts)
5327{
5328 struct bbr_sendmap *rsm;
5329
5330 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap);
5331 if ((rsm == NULL) || (u_rsm == rsm))
5332 return (cts);
5333 return(rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)]);
5334}
5335
5336static void
5337bbr_update_rsm(struct tcpcb *tp, struct tcp_bbr *bbr,
5338 struct bbr_sendmap *rsm, uint32_t cts, uint32_t pacing_time)
5339{
5340 int32_t idx;
5341
5342 rsm->r_rtr_cnt++;
5343 rsm->r_dupack = 0;
5344 if (rsm->r_rtr_cnt > BBR_NUM_OF_RETRANS) {
5345 rsm->r_rtr_cnt = BBR_NUM_OF_RETRANS;
5346 rsm->r_flags |= BBR_OVERMAX;
5347 }
5348 if (rsm->r_flags & BBR_RWND_COLLAPSED) {
5349 /* Take off the collapsed flag at rxt */
5350 rsm->r_flags &= ~BBR_RWND_COLLAPSED;
5351 }
5352 if (rsm->r_flags & BBR_MARKED_LOST) {
5353 /* We have retransmitted, its no longer lost */
5354 rsm->r_flags &= ~BBR_MARKED_LOST;
5355 bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start;
5356 }
5357 if (rsm->r_flags & BBR_RXT_CLEARED) {
5358 /*
5359 * We hit a RXT timer on it and
5360 * we cleared the "acked" flag.
5361 * We now have it going back into
5362 * flight, we can remove the cleared
5363 * flag and possibly do accounting on
5364 * this piece.
5365 */
5366 rsm->r_flags &= ~BBR_RXT_CLEARED;
5367 }
5368 if ((rsm->r_rtr_cnt > 1) && ((rsm->r_flags & BBR_TLP) == 0)) {
5369 bbr->r_ctl.rc_holes_rxt += (rsm->r_end - rsm->r_start);
5370 rsm->r_rtr_bytes += (rsm->r_end - rsm->r_start);
5371 }
5372 idx = rsm->r_rtr_cnt - 1;
5373 rsm->r_tim_lastsent[idx] = cts;
5374 rsm->r_pacing_delay = pacing_time;
5375 rsm->r_delivered = bbr->r_ctl.rc_delivered;
5376 rsm->r_ts_valid = bbr->rc_ts_valid;
5377 if (bbr->rc_ts_valid)
5378 rsm->r_del_ack_ts = bbr->r_ctl.last_inbound_ts;
5379 if (bbr->r_ctl.r_app_limited_until)
5380 rsm->r_app_limited = 1;
5381 else
5382 rsm->r_app_limited = 0;
5383 if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW)
5384 rsm->r_bbr_state = bbr_state_val(bbr);
5385 else
5386 rsm->r_bbr_state = 8;
5387 if (rsm->r_flags & BBR_ACKED) {
5388 /* Problably MTU discovery messing with us */
5389 uint32_t old_flags;
5390
5391 old_flags = rsm->r_flags;
5392 rsm->r_flags &= ~BBR_ACKED;
5393 bbr_log_type_rsmclear(bbr, cts, rsm, old_flags, __LINE__);
5394 bbr->r_ctl.rc_sacked -= (rsm->r_end - rsm->r_start);
5395 if (bbr->r_ctl.rc_sacked == 0)
5396 bbr->r_ctl.rc_sacklast = NULL;
5397 }
5398 if (rsm->r_in_tmap) {
5399 TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
5400 }
5401 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
5402 rsm->r_in_tmap = 1;
5403 if (rsm->r_flags & BBR_SACK_PASSED) {
5404 /* We have retransmitted due to the SACK pass */
5405 rsm->r_flags &= ~BBR_SACK_PASSED;
5406 rsm->r_flags |= BBR_WAS_SACKPASS;
5407 }
5408 rsm->r_first_sent_time = bbr_get_earliest_send_outstanding(bbr, rsm, cts);
5409 rsm->r_flight_at_send = ctf_flight_size(bbr->rc_tp,
5410 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
5411 bbr->r_ctl.rc_next = TAILQ_NEXT(rsm, r_next);
5412 if (bbr->r_ctl.rc_bbr_hptsi_gain > BBR_UNIT) {
5413 rsm->r_is_gain = 1;
5414 rsm->r_is_drain = 0;
5415 } else if (bbr->r_ctl.rc_bbr_hptsi_gain < BBR_UNIT) {
5416 rsm->r_is_drain = 1;
5417 rsm->r_is_gain = 0;
5418 } else {
5419 rsm->r_is_drain = 0;
5420 rsm->r_is_gain = 0;
5421 }
5422 rsm->r_del_time = bbr->r_ctl.rc_del_time; /* TEMP GOOGLE CODE */
5423}
5424
5425/*
5426 * Returns 0, or the sequence where we stopped
5427 * updating. We also update the lenp to be the amount
5428 * of data left.
5429 */
5430
5431static uint32_t
5432bbr_update_entry(struct tcpcb *tp, struct tcp_bbr *bbr,
5433 struct bbr_sendmap *rsm, uint32_t cts, int32_t *lenp, uint32_t pacing_time)
5434{
5435 /*
5436 * We (re-)transmitted starting at rsm->r_start for some length
5437 * (possibly less than r_end.
5438 */
5439 struct bbr_sendmap *nrsm;
5440 uint32_t c_end;
5441 int32_t len;
5442
5443 len = *lenp;
5444 c_end = rsm->r_start + len;
5445 if (SEQ_GEQ(c_end, rsm->r_end)) {
5446 /*
5447 * We retransmitted the whole piece or more than the whole
5448 * slopping into the next rsm.
5449 */
5450 bbr_update_rsm(tp, bbr, rsm, cts, pacing_time);
5451 if (c_end == rsm->r_end) {
5452 *lenp = 0;
5453 return (0);
5454 } else {
5455 int32_t act_len;
5456
5457 /* Hangs over the end return whats left */
5458 act_len = rsm->r_end - rsm->r_start;
5459 *lenp = (len - act_len);
5460 return (rsm->r_end);
5461 }
5462 /* We don't get out of this block. */
5463 }
5464 /*
5465 * Here we retransmitted less than the whole thing which means we
5466 * have to split this into what was transmitted and what was not.
5467 */
5468 nrsm = bbr_alloc_full_limit(bbr);
5469 if (nrsm == NULL) {
5470 *lenp = 0;
5471 return (0);
5472 }
5473 /*
5474 * So here we are going to take the original rsm and make it what we
5475 * retransmitted. nrsm will be the tail portion we did not
5476 * retransmit. For example say the chunk was 1, 11 (10 bytes). And
5477 * we retransmitted 5 bytes i.e. 1, 5. The original piece shrinks to
5478 * 1, 6 and the new piece will be 6, 11.
5479 */
5480 bbr_clone_rsm(bbr, nrsm, rsm, c_end);
5481 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next);
5482 nrsm->r_dupack = 0;
5483 if (rsm->r_in_tmap) {
5484 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext);
5485 nrsm->r_in_tmap = 1;
5486 }
5487 rsm->r_flags &= (~BBR_HAS_FIN);
5488 bbr_update_rsm(tp, bbr, rsm, cts, pacing_time);
5489 *lenp = 0;
5490 return (0);
5491}
5492
5493static uint64_t
5494bbr_get_hardware_rate(struct tcp_bbr *bbr)
5495{
5496 uint64_t bw;
5497
5498 bw = bbr_get_bw(bbr);
5499 bw *= (uint64_t)bbr_hptsi_gain[BBR_SUB_GAIN];
5500 bw /= (uint64_t)BBR_UNIT;
5501 return(bw);
5502}
5503
5504static void
5505bbr_setup_less_of_rate(struct tcp_bbr *bbr, uint32_t cts,
5506 uint64_t act_rate, uint64_t rate_wanted)
5507{
5508 /*
5509 * We could not get a full gains worth
5510 * of rate.
5511 */
5512 if (get_filter_value(&bbr->r_ctl.rc_delrate) >= act_rate) {
5513 /* we can't even get the real rate */
5514 uint64_t red;
5515
5516 bbr->skip_gain = 1;
5517 bbr->gain_is_limited = 0;
5518 red = get_filter_value(&bbr->r_ctl.rc_delrate) - act_rate;
5519 if (red)
5520 filter_reduce_by(&bbr->r_ctl.rc_delrate, red, cts);
5521 } else {
5522 /* We can use a lower gain */
5523 bbr->skip_gain = 0;
5524 bbr->gain_is_limited = 1;
5525 }
5526}
5527
5528static void
5529bbr_update_hardware_pacing_rate(struct tcp_bbr *bbr, uint32_t cts)
5530{
5531 const struct tcp_hwrate_limit_table *nrte;
5532 int error, rate = -1;
5533
5534 if (bbr->r_ctl.crte == NULL)
5535 return;
5536 if ((bbr->rc_inp->inp_route.ro_nh == NULL) ||
5537 (bbr->rc_inp->inp_route.ro_nh->nh_ifp == NULL)) {
5538 /* Lost our routes? */
5539 /* Clear the way for a re-attempt */
5540 bbr->bbr_attempt_hdwr_pace = 0;
5541lost_rate:
5542 bbr->gain_is_limited = 0;
5543 bbr->skip_gain = 0;
5544 bbr->bbr_hdrw_pacing = 0;
5545 counter_u64_add(bbr_flows_whdwr_pacing, -1);
5546 counter_u64_add(bbr_flows_nohdwr_pacing, 1);
5547 tcp_bbr_tso_size_check(bbr, cts);
5548 return;
5549 }
5550 rate = bbr_get_hardware_rate(bbr);
5551 nrte = tcp_chg_pacing_rate(bbr->r_ctl.crte,
5552 bbr->rc_tp,
5553 bbr->rc_inp->inp_route.ro_nh->nh_ifp,
5554 rate,
5555 (RS_PACING_GEQ|RS_PACING_SUB_OK),
5556 &error, NULL);
5557 if (nrte == NULL) {
5558 goto lost_rate;
5559 }
5560 if (nrte != bbr->r_ctl.crte) {
5561 bbr->r_ctl.crte = nrte;
5562 if (error == 0) {
5563 BBR_STAT_INC(bbr_hdwr_rl_mod_ok);
5564 if (bbr->r_ctl.crte->rate < rate) {
5565 /* We have a problem */
5566 bbr_setup_less_of_rate(bbr, cts,
5567 bbr->r_ctl.crte->rate, rate);
5568 } else {
5569 /* We are good */
5570 bbr->gain_is_limited = 0;
5571 bbr->skip_gain = 0;
5572 }
5573 } else {
5574 /* A failure should release the tag */
5575 BBR_STAT_INC(bbr_hdwr_rl_mod_fail);
5576 bbr->gain_is_limited = 0;
5577 bbr->skip_gain = 0;
5578 bbr->bbr_hdrw_pacing = 0;
5579 }
5580 bbr_type_log_hdwr_pacing(bbr,
5581 bbr->r_ctl.crte->ptbl->rs_ifp,
5582 rate,
5583 ((bbr->r_ctl.crte == NULL) ? 0 : bbr->r_ctl.crte->rate),
5584 __LINE__,
5585 cts,
5586 error);
5587 }
5588}
5589
5590static void
5591bbr_adjust_for_hw_pacing(struct tcp_bbr *bbr, uint32_t cts)
5592{
5593 /*
5594 * If we have hardware pacing support
5595 * we need to factor that in for our
5596 * TSO size.
5597 */
5598 const struct tcp_hwrate_limit_table *rlp;
5599 uint32_t cur_delay, seg_sz, maxseg, new_tso, delta, hdwr_delay;
5600
5601 if ((bbr->bbr_hdrw_pacing == 0) ||
5602 (IN_RECOVERY(bbr->rc_tp->t_flags)) ||
5603 (bbr->r_ctl.crte == NULL))
5604 return;
5605 if (bbr->hw_pacing_set == 0) {
5606 /* Not yet by the hdwr pacing count delay */
5607 return;
5608 }
5609 if (bbr_hdwr_pace_adjust == 0) {
5610 /* No adjustment */
5611 return;
5612 }
5613 rlp = bbr->r_ctl.crte;
5614 if (bbr->rc_tp->t_maxseg > bbr->rc_last_options)
5615 maxseg = bbr->rc_tp->t_maxseg - bbr->rc_last_options;
5616 else
5617 maxseg = BBR_MIN_SEG - bbr->rc_last_options;
5618 /*
5619 * So lets first get the
5620 * time we will take between
5621 * TSO sized sends currently without
5622 * hardware help.
5623 */
5624 cur_delay = bbr_get_pacing_delay(bbr, BBR_UNIT,
5625 bbr->r_ctl.rc_pace_max_segs, cts, 1);
5626 hdwr_delay = bbr->r_ctl.rc_pace_max_segs / maxseg;
5627 hdwr_delay *= rlp->time_between;
5628 if (cur_delay > hdwr_delay)
5629 delta = cur_delay - hdwr_delay;
5630 else
5631 delta = 0;
5632 bbr_log_type_tsosize(bbr, cts, delta, cur_delay, hdwr_delay,
5633 (bbr->r_ctl.rc_pace_max_segs / maxseg),
5634 1);
5635 if (delta &&
5636 (delta < (max(rlp->time_between,
5637 bbr->r_ctl.bbr_hptsi_segments_delay_tar)))) {
5638 /*
5639 * Now lets divide by the pacing
5640 * time between each segment the
5641 * hardware sends rounding up and
5642 * derive a bytes from that. We multiply
5643 * that by bbr_hdwr_pace_adjust to get
5644 * more bang for our buck.
5645 *
5646 * The goal is to have the software pacer
5647 * waiting no more than an additional
5648 * pacing delay if we can (without the
5649 * compensation i.e. x bbr_hdwr_pace_adjust).
5650 */
5651 seg_sz = max(((cur_delay + rlp->time_between)/rlp->time_between),
5652 (bbr->r_ctl.rc_pace_max_segs/maxseg));
5653 seg_sz *= bbr_hdwr_pace_adjust;
5654 if (bbr_hdwr_pace_floor &&
5655 (seg_sz < bbr->r_ctl.crte->ptbl->rs_min_seg)) {
5656 /* Currently hardware paces
5657 * out rs_min_seg segments at a time.
5658 * We need to make sure we always send at least
5659 * a full burst of bbr_hdwr_pace_floor down.
5660 */
5661 seg_sz = bbr->r_ctl.crte->ptbl->rs_min_seg;
5662 }
5663 seg_sz *= maxseg;
5664 } else if (delta == 0) {
5665 /*
5666 * The highest pacing rate is
5667 * above our b/w gained. This means
5668 * we probably are going quite fast at
5669 * the hardware highest rate. Lets just multiply
5670 * the calculated TSO size by the
5671 * multiplier factor (its probably
5672 * 4 segments in the default config for
5673 * mlx).
5674 */
5675 seg_sz = bbr->r_ctl.rc_pace_max_segs * bbr_hdwr_pace_adjust;
5676 if (bbr_hdwr_pace_floor &&
5677 (seg_sz < bbr->r_ctl.crte->ptbl->rs_min_seg)) {
5678 /* Currently hardware paces
5679 * out rs_min_seg segments at a time.
5680 * We need to make sure we always send at least
5681 * a full burst of bbr_hdwr_pace_floor down.
5682 */
5683 seg_sz = bbr->r_ctl.crte->ptbl->rs_min_seg;
5684 }
5685 } else {
5686 /*
5687 * The pacing time difference is so
5688 * big that the hardware will
5689 * pace out more rapidly then we
5690 * really want and then we
5691 * will have a long delay. Lets just keep
5692 * the same TSO size so its as if
5693 * we were not using hdwr pacing (we
5694 * just gain a bit of spacing from the
5695 * hardware if seg_sz > 1).
5696 */
5697 seg_sz = bbr->r_ctl.rc_pace_max_segs;
5698 }
5699 if (seg_sz > bbr->r_ctl.rc_pace_max_segs)
5700 new_tso = seg_sz;
5701 else
5702 new_tso = bbr->r_ctl.rc_pace_max_segs;
5703 if (new_tso >= (PACE_MAX_IP_BYTES-maxseg))
5704 new_tso = PACE_MAX_IP_BYTES - maxseg;
5705
5706 if (new_tso != bbr->r_ctl.rc_pace_max_segs) {
5707 bbr_log_type_tsosize(bbr, cts, new_tso, 0, bbr->r_ctl.rc_pace_max_segs, maxseg, 0);
5708 bbr->r_ctl.rc_pace_max_segs = new_tso;
5709 }
5710}
5711
5712static void
5713tcp_bbr_tso_size_check(struct tcp_bbr *bbr, uint32_t cts)
5714{
5715 uint64_t bw;
5716 uint32_t old_tso = 0, new_tso;
5717 uint32_t maxseg, bytes;
5718 uint32_t tls_seg=0;
5719 /*
5720 * Google/linux uses the following algorithm to determine
5721 * the TSO size based on the b/w of the link (from Neal Cardwell email 9/27/18):
5722 *
5723 * bytes = bw_in_bytes_per_second / 1000
5724 * bytes = min(bytes, 64k)
5725 * tso_segs = bytes / MSS
5726 * if (bw < 1.2Mbs)
5727 * min_tso_segs = 1
5728 * else
5729 * min_tso_segs = 2
5730 * tso_segs = max(tso_segs, min_tso_segs)
5731 *
5732 * * Note apply a device specific limit (we apply this in the
5733 * tcp_m_copym).
5734 * Note that before the initial measurement is made google bursts out
5735 * a full iwnd just like new-reno/cubic.
5736 *
5737 * We do not use this algorithm. Instead we
5738 * use a two phased approach:
5739 *
5740 * if ( bw <= per-tcb-cross-over)
5741 * goal_tso = calculate how much with this bw we
5742 * can send in goal-time seconds.
5743 * if (goal_tso > mss)
5744 * seg = goal_tso / mss
5745 * tso = seg * mss
5746 * else
5747 * tso = mss
5748 * if (tso > per-tcb-max)
5749 * tso = per-tcb-max
5750 * else if ( bw > 512Mbps)
5751 * tso = max-tso (64k/mss)
5752 * else
5753 * goal_tso = bw / per-tcb-divsor
5754 * seg = (goal_tso + mss-1)/mss
5755 * tso = seg * mss
5756 *
5757 * if (tso < per-tcb-floor)
5758 * tso = per-tcb-floor
5759 * if (tso > per-tcb-utter_max)
5760 * tso = per-tcb-utter_max
5761 *
5762 * Note the default per-tcb-divisor is 1000 (same as google).
5763 * the goal cross over is 30Mbps however. To recreate googles
5764 * algorithm you need to set:
5765 *
5766 * cross-over = 23,168,000 bps
5767 * goal-time = 18000
5768 * per-tcb-max = 2
5769 * per-tcb-divisor = 1000
5770 * per-tcb-floor = 1
5771 *
5772 * This will get you "google bbr" behavior with respect to tso size.
5773 *
5774 * Note we do set anything TSO size until we are past the initial
5775 * window. Before that we gnerally use either a single MSS
5776 * or we use the full IW size (so we burst a IW at a time)
5777 */
5778
5779 if (bbr->rc_tp->t_maxseg > bbr->rc_last_options) {
5780 maxseg = bbr->rc_tp->t_maxseg - bbr->rc_last_options;
5781 } else {
5782 maxseg = BBR_MIN_SEG - bbr->rc_last_options;
5783 }
5784 old_tso = bbr->r_ctl.rc_pace_max_segs;
5785 if (bbr->rc_past_init_win == 0) {
5786 /*
5787 * Not enough data has been acknowledged to make a
5788 * judgement. Set up the initial TSO based on if we
5789 * are sending a full IW at once or not.
5790 */
5791 if (bbr->rc_use_google)
5792 bbr->r_ctl.rc_pace_max_segs = ((bbr->rc_tp->t_maxseg - bbr->rc_last_options) * 2);
5793 else if (bbr->bbr_init_win_cheat)
5794 bbr->r_ctl.rc_pace_max_segs = bbr_initial_cwnd(bbr, bbr->rc_tp);
5795 else
5796 bbr->r_ctl.rc_pace_max_segs = bbr->rc_tp->t_maxseg - bbr->rc_last_options;
5797 if (bbr->r_ctl.rc_pace_min_segs != bbr->rc_tp->t_maxseg)
5798 bbr->r_ctl.rc_pace_min_segs = bbr->rc_tp->t_maxseg;
5799 if (bbr->r_ctl.rc_pace_max_segs == 0) {
5800 bbr->r_ctl.rc_pace_max_segs = maxseg;
5801 }
5802 bbr_log_type_tsosize(bbr, cts, bbr->r_ctl.rc_pace_max_segs, tls_seg, old_tso, maxseg, 0);
5803 bbr_adjust_for_hw_pacing(bbr, cts);
5804 return;
5805 }
5811 bw = bbr_get_bw(bbr);
5812 if (IN_RECOVERY(bbr->rc_tp->t_flags) &&
5813 (bbr->rc_use_google == 0)) {
5814 /* We clamp to one MSS in recovery */
5815 new_tso = maxseg;
5816 } else if (bbr->rc_use_google) {
5817 int min_tso_segs;
5818
5819 /* Google considers the gain too */
5820 if (bbr->r_ctl.rc_bbr_hptsi_gain != BBR_UNIT) {
5821 bw *= bbr->r_ctl.rc_bbr_hptsi_gain;
5822 bw /= BBR_UNIT;
5823 }
5824 bytes = bw / 1024;
5825 if (bytes > (64 * 1024))
5826 bytes = 64 * 1024;
5827 new_tso = bytes / maxseg;
5828 if (bw < ONE_POINT_TWO_MEG)
5829 min_tso_segs = 1;
5830 else
5831 min_tso_segs = 2;
5832 if (new_tso < min_tso_segs)
5833 new_tso = min_tso_segs;
5834 new_tso *= maxseg;
5835 } else if (bbr->rc_no_pacing) {
5836 new_tso = (PACE_MAX_IP_BYTES / maxseg) * maxseg;
5837 } else if (bw <= bbr->r_ctl.bbr_cross_over) {
5838 /*
5839 * Calculate the worse case b/w TSO if we are inserting no
5840 * more than a delay_target number of TSO's.
5841 */
5842 uint32_t tso_len, min_tso;
5843
5844 tso_len = bbr_get_pacing_length(bbr, BBR_UNIT, bbr->r_ctl.bbr_hptsi_segments_delay_tar, bw);
5845 if (tso_len > maxseg) {
5846 new_tso = tso_len / maxseg;
5847 if (new_tso > bbr->r_ctl.bbr_hptsi_segments_max)
5848 new_tso = bbr->r_ctl.bbr_hptsi_segments_max;
5849 new_tso *= maxseg;
5850 } else {
5851 /*
5852 * less than a full sized frame yikes.. long rtt or
5853 * low bw?
5854 */
5855 min_tso = bbr_minseg(bbr);
5856 if ((tso_len > min_tso) && (bbr_all_get_min == 0))
5857 new_tso = rounddown(tso_len, min_tso);
5858 else
5859 new_tso = min_tso;
5860 }
5861 } else if (bw > FIVETWELVE_MBPS) {
5862 /*
5863 * This guy is so fast b/w wise that we can TSO as large as
5864 * possible of segments that the NIC will allow.
5865 */
5866 new_tso = rounddown(PACE_MAX_IP_BYTES, maxseg);
5867 } else {
5868 /*
5869 * This formula is based on attempting to send a segment or
5870 * more every bbr_hptsi_per_second. The default is 1000
5871 * which means you are targeting what you can send every 1ms
5872 * based on the peers bw.
5873 *
5874 * If the number drops to say 500, then you are looking more
5875 * at 2ms and you will raise how much we send in a single
5876 * TSO thus saving CPU (less bbr_output_wtime() calls). The
5877 * trade off of course is you will send more at once and
5878 * thus tend to clump up the sends into larger "bursts"
5879 * building a queue.
5880 */
5881 bw /= bbr->r_ctl.bbr_hptsi_per_second;
5882 new_tso = roundup(bw, (uint64_t)maxseg);
5883 /*
5884 * Gate the floor to match what our lower than 48Mbps
5885 * algorithm does. The ceiling (bbr_hptsi_segments_max) thus
5886 * becomes the floor for this calculation.
5887 */
5888 if (new_tso < (bbr->r_ctl.bbr_hptsi_segments_max * maxseg))
5889 new_tso = (bbr->r_ctl.bbr_hptsi_segments_max * maxseg);
5890 }
5891 if (bbr->r_ctl.bbr_hptsi_segments_floor && (new_tso < (maxseg * bbr->r_ctl.bbr_hptsi_segments_floor)))
5892 new_tso = maxseg * bbr->r_ctl.bbr_hptsi_segments_floor;
5893 if (new_tso > PACE_MAX_IP_BYTES)
5894 new_tso = rounddown(PACE_MAX_IP_BYTES, maxseg);
5895 /* Enforce an utter maximum. */
5896 if (bbr->r_ctl.bbr_utter_max && (new_tso > (bbr->r_ctl.bbr_utter_max * maxseg))) {
5897 new_tso = bbr->r_ctl.bbr_utter_max * maxseg;
5898 }
5899 if (old_tso != new_tso) {
5900 /* Only log changes */
5901 bbr_log_type_tsosize(bbr, cts, new_tso, tls_seg, old_tso, maxseg, 0);
5902 bbr->r_ctl.rc_pace_max_segs = new_tso;
5903 }
5904 /* We have hardware pacing! */
5905 bbr_adjust_for_hw_pacing(bbr, cts);
5906}
5907
5908static void
5909bbr_log_output(struct tcp_bbr *bbr, struct tcpcb *tp, struct tcpopt *to, int32_t len,
5910 uint32_t seq_out, uint16_t th_flags, int32_t err, uint32_t cts,
5911 struct mbuf *mb, int32_t * abandon, struct bbr_sendmap *hintrsm, uint32_t delay_calc,
5912 struct sockbuf *sb)
5913{
5914
5915 struct bbr_sendmap *rsm, *nrsm;
5916 register uint32_t snd_max, snd_una;
5917 uint32_t pacing_time;
5918 /*
5919 * Add to the RACK log of packets in flight or retransmitted. If
5920 * there is a TS option we will use the TS echoed, if not we will
5921 * grab a TS.
5922 *
5923 * Retransmissions will increment the count and move the ts to its
5924 * proper place. Note that if options do not include TS's then we
5925 * won't be able to effectively use the ACK for an RTT on a retran.
5926 *
5927 * Notes about r_start and r_end. Lets consider a send starting at
5928 * sequence 1 for 10 bytes. In such an example the r_start would be
5929 * 1 (starting sequence) but the r_end would be r_start+len i.e. 11.
5930 * This means that r_end is actually the first sequence for the next
5931 * slot (11).
5932 *
5933 */
5934 INP_WLOCK_ASSERT(tp->t_inpcb);
5935 if (err) {
5936 /*
5937 * We don't log errors -- we could but snd_max does not
5938 * advance in this case either.
5939 */
5940 return;
5941 }
5942 if (th_flags & TH_RST) {
5943 /*
5944 * We don't log resets and we return immediately from
5945 * sending
5946 */
5947 *abandon = 1;
5948 return;
5949 }
5950 snd_una = tp->snd_una;
5951 if (th_flags & (TH_SYN | TH_FIN) && (hintrsm == NULL)) {
5952 /*
5953 * The call to bbr_log_output is made before bumping
5954 * snd_max. This means we can record one extra byte on a SYN
5955 * or FIN if seq_out is adding more on and a FIN is present
5956 * (and we are not resending).
5957 */
5958 if ((th_flags & TH_SYN) && (tp->iss == seq_out))
5959 len++;
5960 if (th_flags & TH_FIN)
5961 len++;
5962 }
5963 if (SEQ_LEQ((seq_out + len), snd_una)) {
5964 /* Are sending an old segment to induce an ack (keep-alive)? */
5965 return;
5966 }
5967 if (SEQ_LT(seq_out, snd_una)) {
5968 /* huh? should we panic? */
5969 uint32_t end;
5970
5971 end = seq_out + len;
5972 seq_out = snd_una;
5973 len = end - seq_out;
5974 }
5975 snd_max = tp->snd_max;
5976 if (len == 0) {
5977 /* We don't log zero window probes */
5978 return;
5979 }
5980 pacing_time = bbr_get_pacing_delay(bbr, bbr->r_ctl.rc_bbr_hptsi_gain, len, cts, 1);
5981 /* First question is it a retransmission? */
5982 if (seq_out == snd_max) {
5983again:
5984 rsm = bbr_alloc(bbr);
5985 if (rsm == NULL) {
5986 return;
5987 }
5988 rsm->r_flags = 0;
5989 if (th_flags & TH_SYN)
5990 rsm->r_flags |= BBR_HAS_SYN;
5991 if (th_flags & TH_FIN)
5992 rsm->r_flags |= BBR_HAS_FIN;
5993 rsm->r_tim_lastsent[0] = cts;
5994 rsm->r_rtr_cnt = 1;
5995 rsm->r_rtr_bytes = 0;
5996 rsm->r_start = seq_out;
5997 rsm->r_end = rsm->r_start + len;
5998 rsm->r_dupack = 0;
5999 rsm->r_delivered = bbr->r_ctl.rc_delivered;
6000 rsm->r_pacing_delay = pacing_time;
6001 rsm->r_ts_valid = bbr->rc_ts_valid;
6002 if (bbr->rc_ts_valid)
6003 rsm->r_del_ack_ts = bbr->r_ctl.last_inbound_ts;
6004 rsm->r_del_time = bbr->r_ctl.rc_del_time;
6005 if (bbr->r_ctl.r_app_limited_until)
6006 rsm->r_app_limited = 1;
6007 else
6008 rsm->r_app_limited = 0;
6009 rsm->r_first_sent_time = bbr_get_earliest_send_outstanding(bbr, rsm, cts);
6010 rsm->r_flight_at_send = ctf_flight_size(bbr->rc_tp,
6011 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
6012 /*
6013 * Here we must also add in this rsm since snd_max
6014 * is updated after we return from a new send.
6015 */
6016 rsm->r_flight_at_send += len;
6017 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_map, rsm, r_next);
6018 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
6019 rsm->r_in_tmap = 1;
6020 if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW)
6021 rsm->r_bbr_state = bbr_state_val(bbr);
6022 else
6023 rsm->r_bbr_state = 8;
6024 if (bbr->r_ctl.rc_bbr_hptsi_gain > BBR_UNIT) {
6025 rsm->r_is_gain = 1;
6026 rsm->r_is_drain = 0;
6027 } else if (bbr->r_ctl.rc_bbr_hptsi_gain < BBR_UNIT) {
6028 rsm->r_is_drain = 1;
6029 rsm->r_is_gain = 0;
6030 } else {
6031 rsm->r_is_drain = 0;
6032 rsm->r_is_gain = 0;
6033 }
6034 return;
6035 }
6036 /*
6037 * If we reach here its a retransmission and we need to find it.
6038 */
6039more:
6040 if (hintrsm && (hintrsm->r_start == seq_out)) {
6041 rsm = hintrsm;
6042 hintrsm = NULL;
6043 } else if (bbr->r_ctl.rc_next) {
6044 /* We have a hint from a previous run */
6045 rsm = bbr->r_ctl.rc_next;
6046 } else {
6047 /* No hints sorry */
6048 rsm = NULL;
6049 }
6050 if ((rsm) && (rsm->r_start == seq_out)) {
6051 /*
6052 * We used rc_next or hintrsm to retransmit, hopefully the
6053 * likely case.
6054 */
6055 seq_out = bbr_update_entry(tp, bbr, rsm, cts, &len, pacing_time);
6056 if (len == 0) {
6057 return;
6058 } else {
6059 goto more;
6060 }
6061 }
6062 /* Ok it was not the last pointer go through it the hard way. */
6063 TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) {
6064 if (rsm->r_start == seq_out) {
6065 seq_out = bbr_update_entry(tp, bbr, rsm, cts, &len, pacing_time);
6066 bbr->r_ctl.rc_next = TAILQ_NEXT(rsm, r_next);
6067 if (len == 0) {
6068 return;
6069 } else {
6070 continue;
6071 }
6072 }
6073 if (SEQ_GEQ(seq_out, rsm->r_start) && SEQ_LT(seq_out, rsm->r_end)) {
6074 /* Transmitted within this piece */
6075 /*
6076 * Ok we must split off the front and then let the
6077 * update do the rest
6078 */
6079 nrsm = bbr_alloc_full_limit(bbr);
6080 if (nrsm == NULL) {
6081 bbr_update_rsm(tp, bbr, rsm, cts, pacing_time);
6082 return;
6083 }
6084 /*
6085 * copy rsm to nrsm and then trim the front of rsm
6086 * to not include this part.
6087 */
6088 bbr_clone_rsm(bbr, nrsm, rsm, seq_out);
6089 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next);
6090 if (rsm->r_in_tmap) {
6091 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext);
6092 nrsm->r_in_tmap = 1;
6093 }
6094 rsm->r_flags &= (~BBR_HAS_FIN);
6095 seq_out = bbr_update_entry(tp, bbr, nrsm, cts, &len, pacing_time);
6096 if (len == 0) {
6097 return;
6098 }
6099 }
6100 }
6101 /*
6102 * Hmm not found in map did they retransmit both old and on into the
6103 * new?
6104 */
6105 if (seq_out == tp->snd_max) {
6106 goto again;
6107 } else if (SEQ_LT(seq_out, tp->snd_max)) {
6108#ifdef BBR_INVARIANTS
6109 printf("seq_out:%u len:%d snd_una:%u snd_max:%u -- but rsm not found?\n",
6110 seq_out, len, tp->snd_una, tp->snd_max);
6111 printf("Starting Dump of all rack entries\n");
6112 TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) {
6113 printf("rsm:%p start:%u end:%u\n",
6114 rsm, rsm->r_start, rsm->r_end);
6115 }
6116 printf("Dump complete\n");
6117 panic("seq_out not found rack:%p tp:%p",
6118 bbr, tp);
6119#endif
6120 } else {
6121#ifdef BBR_INVARIANTS
6122 /*
6123 * Hmm beyond sndmax? (only if we are using the new rtt-pack
6124 * flag)
6125 */
6126 panic("seq_out:%u(%d) is beyond snd_max:%u tp:%p",
6127 seq_out, len, tp->snd_max, tp);
6128#endif
6129 }
6130}
6131
6132static void
6133bbr_collapse_rtt(struct tcpcb *tp, struct tcp_bbr *bbr, int32_t rtt)
6134{
6135 /*
6136 * Collapse timeout back the cum-ack moved.
6137 */
6138 tp->t_rxtshift = 0;
6139 tp->t_softerror = 0;
6140}
6141
6142static void
6143tcp_bbr_xmit_timer(struct tcp_bbr *bbr, uint32_t rtt_usecs, uint32_t rsm_send_time, uint32_t r_start, uint32_t tsin)
6144{
6145 bbr->rtt_valid = 1;
6146 bbr->r_ctl.cur_rtt = rtt_usecs;
6147 bbr->r_ctl.ts_in = tsin;
6148 if (rsm_send_time)
6149 bbr->r_ctl.cur_rtt_send_time = rsm_send_time;
6150}
6151
6152static void
6153bbr_make_timestamp_determination(struct tcp_bbr *bbr)
6154{
6191 uint64_t delta, peer_delta, delta_up;
6192
6193 delta = bbr->r_ctl.cur_rtt_send_time - bbr->r_ctl.bbr_ts_check_our_cts;
6194 if (delta < bbr_min_usec_delta) {
6195 /*
6196 * Have not seen a min amount of time
6197 * between our send times so we can
6198 * make a determination of the timestamp
6199 * yet.
6200 */
6201 return;
6202 }
6203 peer_delta = bbr->r_ctl.last_inbound_ts - bbr->r_ctl.bbr_ts_check_tstmp;
6204 if (peer_delta < bbr_min_peer_delta) {
6205 /*
6206 * We may have enough in the form of
6207 * our delta but the peers number
6208 * has not changed that much. It could
6209 * be its clock ratio is such that
6210 * we need more data (10ms tick) or
6211 * there may be other compression scenarios
6212 * going on. In any event we need the
6213 * spread to be larger.
6214 */
6215 return;
6216 }
6217 /* Ok lets first see which way our delta is going */
6218 if (peer_delta > delta) {
6219 /* Very unlikely, the peer without
6220 * compensation shows that it saw
6221 * the two sends arrive further apart
6222 * then we saw then in micro-seconds.
6223 */
6224 if (peer_delta < (delta + ((delta * (uint64_t)1000)/ (uint64_t)bbr_delta_percent))) {
6225 /* well it looks like the peer is a micro-second clock. */
6226 bbr->rc_ts_clock_set = 1;
6227 bbr->r_ctl.bbr_peer_tsratio = 1;
6228 } else {
6229 bbr->rc_ts_cant_be_used = 1;
6230 bbr->rc_ts_clock_set = 1;
6231 }
6232 return;
6233 }
6234 /* Ok we know that the peer_delta is smaller than our send distance */
6235 bbr->rc_ts_clock_set = 1;
6236 /* First question is it within the percentage that they are using usec time? */
6237 delta_up = (peer_delta * 1000) / (uint64_t)bbr_delta_percent;
6238 if ((peer_delta + delta_up) >= delta) {
6239 /* Its a usec clock */
6240 bbr->r_ctl.bbr_peer_tsratio = 1;
6241 bbr_log_tstmp_validation(bbr, peer_delta, delta);
6242 return;
6243 }
6244 /* Ok if not usec, what about 10usec (though unlikely)? */
6245 delta_up = (peer_delta * 1000 * 10) / (uint64_t)bbr_delta_percent;
6246 if (((peer_delta * 10) + delta_up) >= delta) {
6247 bbr->r_ctl.bbr_peer_tsratio = 10;
6248 bbr_log_tstmp_validation(bbr, peer_delta, delta);
6249 return;
6250 }
6251 /* And what about 100usec (though again unlikely)? */
6252 delta_up = (peer_delta * 1000 * 100) / (uint64_t)bbr_delta_percent;
6253 if (((peer_delta * 100) + delta_up) >= delta) {
6254 bbr->r_ctl.bbr_peer_tsratio = 100;
6255 bbr_log_tstmp_validation(bbr, peer_delta, delta);
6256 return;
6257 }
6258 /* And how about 1 msec (the most likely one)? */
6259 delta_up = (peer_delta * 1000 * 1000) / (uint64_t)bbr_delta_percent;
6260 if (((peer_delta * 1000) + delta_up) >= delta) {
6261 bbr->r_ctl.bbr_peer_tsratio = 1000;
6262 bbr_log_tstmp_validation(bbr, peer_delta, delta);
6263 return;
6264 }
6265 /* Ok if not msec could it be 10 msec? */
6266 delta_up = (peer_delta * 1000 * 10000) / (uint64_t)bbr_delta_percent;
6267 if (((peer_delta * 10000) + delta_up) >= delta) {
6268 bbr->r_ctl.bbr_peer_tsratio = 10000;
6269 return;
6270 }
6271 /* If we fall down here the clock tick so slowly we can't use it */
6272 bbr->rc_ts_cant_be_used = 1;
6273 bbr->r_ctl.bbr_peer_tsratio = 0;
6274 bbr_log_tstmp_validation(bbr, peer_delta, delta);
6275}
6276
6277/*
6278 * Collect new round-trip time estimate
6279 * and update averages and current timeout.
6280 */
6281static void
6282tcp_bbr_xmit_timer_commit(struct tcp_bbr *bbr, struct tcpcb *tp, uint32_t cts)
6283{
6284 int32_t delta;
6285 uint32_t rtt, tsin;
6286 int32_t rtt_ticks;
6287
6288 if (bbr->rtt_valid == 0)
6289 /* No valid sample */
6290 return;
6291
6292 rtt = bbr->r_ctl.cur_rtt;
6293 tsin = bbr->r_ctl.ts_in;
6294 if (bbr->rc_prtt_set_ts) {
6295 /*
6296 * We are to force feed the rttProp filter due
6297 * to an entry into PROBE_RTT. This assures
6298 * that the times are sync'd between when we
6299 * go into PROBE_RTT and the filter expiration.
6300 *
6301 * Google does not use a true filter, so they do
6302 * this implicitly since they only keep one value
6303 * and when they enter probe-rtt they update the
6304 * value to the newest rtt.
6305 */
6306 uint32_t rtt_prop;
6307
6308 bbr->rc_prtt_set_ts = 0;
6309 rtt_prop = get_filter_value_small(&bbr->r_ctl.rc_rttprop);
6310 if (rtt > rtt_prop)
6311 filter_increase_by_small(&bbr->r_ctl.rc_rttprop, (rtt - rtt_prop), cts);
6312 else
6313 apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, cts);
6314 }
6315 if (bbr->rc_ack_was_delayed)
6316 rtt += bbr->r_ctl.rc_ack_hdwr_delay;
6317
6318 if (rtt < bbr->r_ctl.rc_lowest_rtt)
6319 bbr->r_ctl.rc_lowest_rtt = rtt;
6320 bbr_log_rtt_sample(bbr, rtt, tsin);
6321 if (bbr->r_init_rtt) {
6322 /*
6323 * The initial rtt is not-trusted, nuke it and lets get
6324 * our first valid measurement in.
6325 */
6326 bbr->r_init_rtt = 0;
6327 tp->t_srtt = 0;
6328 }
6329 if ((bbr->rc_ts_clock_set == 0) && bbr->rc_ts_valid) {
6330 /*
6331 * So we have not yet figured out
6332 * what the peers TSTMP value is
6333 * in (most likely ms). We need a
6334 * series of cum-ack's to determine
6335 * this reliably.
6336 */
6337 if (bbr->rc_ack_is_cumack) {
6338 if (bbr->rc_ts_data_set) {
6339 /* Lets attempt to determine the timestamp granularity. */
6340 bbr_make_timestamp_determination(bbr);
6341 } else {
6342 bbr->rc_ts_data_set = 1;
6343 bbr->r_ctl.bbr_ts_check_tstmp = bbr->r_ctl.last_inbound_ts;
6344 bbr->r_ctl.bbr_ts_check_our_cts = bbr->r_ctl.cur_rtt_send_time;
6345 }
6346 } else {
6347 /*
6348 * We have to have consecutive acks
6349 * reset any "filled" state to none.
6350 */
6351 bbr->rc_ts_data_set = 0;
6352 }
6353 }
6354 /* Round it up */
6355 rtt_ticks = USEC_2_TICKS((rtt + (USECS_IN_MSEC - 1)));
6356 if (rtt_ticks == 0)
6357 rtt_ticks = 1;
6358 if (tp->t_srtt != 0) {
6359 /*
6360 * srtt is stored as fixed point with 5 bits after the
6361 * binary point (i.e., scaled by 8). The following magic is
6362 * equivalent to the smoothing algorithm in rfc793 with an
6363 * alpha of .875 (srtt = rtt/8 + srtt*7/8 in fixed point).
6364 * Adjust rtt to origin 0.
6365 */
6366
6367 delta = ((rtt_ticks - 1) << TCP_DELTA_SHIFT)
6368 - (tp->t_srtt >> (TCP_RTT_SHIFT - TCP_DELTA_SHIFT));
6369
6370 tp->t_srtt += delta;
6371 if (tp->t_srtt <= 0)
6372 tp->t_srtt = 1;
6373
6374 /*
6375 * We accumulate a smoothed rtt variance (actually, a
6376 * smoothed mean difference), then set the retransmit timer
6377 * to smoothed rtt + 4 times the smoothed variance. rttvar
6378 * is stored as fixed point with 4 bits after the binary
6379 * point (scaled by 16). The following is equivalent to
6380 * rfc793 smoothing with an alpha of .75 (rttvar =
6381 * rttvar*3/4 + |delta| / 4). This replaces rfc793's
6382 * wired-in beta.
6383 */
6384 if (delta < 0)
6385 delta = -delta;
6386 delta -= tp->t_rttvar >> (TCP_RTTVAR_SHIFT - TCP_DELTA_SHIFT);
6387 tp->t_rttvar += delta;
6388 if (tp->t_rttvar <= 0)
6389 tp->t_rttvar = 1;
6390 if (tp->t_rttbest > tp->t_srtt + tp->t_rttvar)
6391 tp->t_rttbest = tp->t_srtt + tp->t_rttvar;
6392 } else {
6393 /*
6394 * No rtt measurement yet - use the unsmoothed rtt. Set the
6395 * variance to half the rtt (so our first retransmit happens
6396 * at 3*rtt).
6397 */
6398 tp->t_srtt = rtt_ticks << TCP_RTT_SHIFT;
6399 tp->t_rttvar = rtt_ticks << (TCP_RTTVAR_SHIFT - 1);
6400 tp->t_rttbest = tp->t_srtt + tp->t_rttvar;
6401 }
6402 KMOD_TCPSTAT_INC(tcps_rttupdated);
6403 tp->t_rttupdated++;
6404#ifdef STATS
6405 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_RTT, imax(0, rtt_ticks));
6406#endif
6407 /*
6408 * the retransmit should happen at rtt + 4 * rttvar. Because of the
6409 * way we do the smoothing, srtt and rttvar will each average +1/2
6410 * tick of bias. When we compute the retransmit timer, we want 1/2
6411 * tick of rounding and 1 extra tick because of +-1/2 tick
6412 * uncertainty in the firing of the timer. The bias will give us
6413 * exactly the 1.5 tick we need. But, because the bias is
6414 * statistical, we have to test that we don't drop below the minimum
6415 * feasible timer (which is 2 ticks).
6416 */
6417 TCPT_RANGESET(tp->t_rxtcur, TCP_REXMTVAL(tp),
6418 max(MSEC_2_TICKS(bbr->r_ctl.rc_min_rto_ms), rtt_ticks + 2),
6419 MSEC_2_TICKS(((uint32_t)bbr->rc_max_rto_sec) * 1000));
6420
6421 /*
6422 * We received an ack for a packet that wasn't retransmitted; it is
6423 * probably safe to discard any error indications we've received
6424 * recently. This isn't quite right, but close enough for now (a
6425 * route might have failed after we sent a segment, and the return
6426 * path might not be symmetrical).
6427 */
6428 tp->t_softerror = 0;
6429 rtt = (TICKS_2_USEC(bbr->rc_tp->t_srtt) >> TCP_RTT_SHIFT);
6430 if (bbr->r_ctl.bbr_smallest_srtt_this_state > rtt)
6431 bbr->r_ctl.bbr_smallest_srtt_this_state = rtt;
6432}
6433
6434static void
6435bbr_set_reduced_rtt(struct tcp_bbr *bbr, uint32_t cts, uint32_t line)
6436{
6437 bbr->r_ctl.rc_rtt_shrinks = cts;
6438 if (bbr_can_force_probertt &&
6439 (TSTMP_GT(cts, bbr->r_ctl.last_in_probertt)) &&
6440 ((cts - bbr->r_ctl.last_in_probertt) > bbr->r_ctl.rc_probertt_int)) {
6441 /*
6442 * We should enter probe-rtt its been too long
6443 * since we have been there.
6444 */
6445 bbr_enter_probe_rtt(bbr, cts, __LINE__);
6446 } else
6447 bbr_check_probe_rtt_limits(bbr, cts);
6448}
6449
6450static void
6451tcp_bbr_commit_bw(struct tcp_bbr *bbr, uint32_t cts)
6452{
6453 uint64_t orig_bw;
6454
6455 if (bbr->r_ctl.rc_bbr_cur_del_rate == 0) {
6456 /* We never apply a zero measurement */
6457 bbr_log_type_bbrupd(bbr, 20, cts, 0, 0,
6458 0, 0, 0, 0, 0, 0);
6459 return;
6460 }
6461 if (bbr->r_ctl.r_measurement_count < 0xffffffff)
6462 bbr->r_ctl.r_measurement_count++;
6463 orig_bw = get_filter_value(&bbr->r_ctl.rc_delrate);
6464 apply_filter_max(&bbr->r_ctl.rc_delrate, bbr->r_ctl.rc_bbr_cur_del_rate, bbr->r_ctl.rc_pkt_epoch);
6465 bbr_log_type_bbrupd(bbr, 21, cts, (uint32_t)orig_bw,
6466 (uint32_t)get_filter_value(&bbr->r_ctl.rc_delrate),
6467 0, 0, 0, 0, 0, 0);
6468 if (orig_bw &&
6469 (orig_bw != get_filter_value(&bbr->r_ctl.rc_delrate))) {
6470 if (bbr->bbr_hdrw_pacing) {
6471 /*
6472 * Apply a new rate to the hardware
6473 * possibly.
6474 */
6475 bbr_update_hardware_pacing_rate(bbr, cts);
6476 }
6477 bbr_set_state_target(bbr, __LINE__);
6478 tcp_bbr_tso_size_check(bbr, cts);
6479 if (bbr->r_recovery_bw) {
6480 bbr_setup_red_bw(bbr, cts);
6481 bbr_log_type_bw_reduce(bbr, BBR_RED_BW_USELRBW);
6482 }
6483 } else if ((orig_bw == 0) && get_filter_value(&bbr->r_ctl.rc_delrate))
6484 tcp_bbr_tso_size_check(bbr, cts);
6485}
6486
6487static void
6488bbr_nf_measurement(struct tcp_bbr *bbr, struct bbr_sendmap *rsm, uint32_t rtt, uint32_t cts)
6489{
6490 if (bbr->rc_in_persist == 0) {
6491 /* We log only when not in persist */
6492 /* Translate to a Bytes Per Second */
6493 uint64_t tim, bw, ts_diff, ts_bw;
6494 uint32_t delivered;
6495
6496 if (TSTMP_GT(bbr->r_ctl.rc_del_time, rsm->r_del_time))
6497 tim = (uint64_t)(bbr->r_ctl.rc_del_time - rsm->r_del_time);
6498 else
6499 tim = 1;
6500 /*
6501 * Now that we have processed the tim (skipping the sample
6502 * or possibly updating the time, go ahead and
6503 * calculate the cdr.
6504 */
6505 delivered = (bbr->r_ctl.rc_delivered - rsm->r_delivered);
6506 bw = (uint64_t)delivered;
6507 bw *= (uint64_t)USECS_IN_SECOND;
6508 bw /= tim;
6509 if (bw == 0) {
6510 /* We must have a calculatable amount */
6511 return;
6512 }
6513 /*
6514 * If we are using this b/w shove it in now so we
6515 * can see in the trace viewer if it gets over-ridden.
6516 */
6517 if (rsm->r_ts_valid &&
6518 bbr->rc_ts_valid &&
6519 bbr->rc_ts_clock_set &&
6520 (bbr->rc_ts_cant_be_used == 0) &&
6521 bbr->rc_use_ts_limit) {
6522 ts_diff = max((bbr->r_ctl.last_inbound_ts - rsm->r_del_ack_ts), 1);
6523 ts_diff *= bbr->r_ctl.bbr_peer_tsratio;
6524 if ((delivered == 0) ||
6525 (rtt < 1000)) {
6526 /* Can't use the ts */
6527 bbr_log_type_bbrupd(bbr, 61, cts,
6528 ts_diff,
6529 bbr->r_ctl.last_inbound_ts,
6530 rsm->r_del_ack_ts, 0,
6531 0, 0, 0, delivered);
6532 } else {
6533 ts_bw = (uint64_t)delivered;
6534 ts_bw *= (uint64_t)USECS_IN_SECOND;
6535 ts_bw /= ts_diff;
6536 bbr_log_type_bbrupd(bbr, 62, cts,
6537 (ts_bw >> 32),
6538 (ts_bw & 0xffffffff), 0, 0,
6539 0, 0, ts_diff, delivered);
6540 if ((bbr->ts_can_raise) &&
6541 (ts_bw > bw)) {
6542 bbr_log_type_bbrupd(bbr, 8, cts,
6543 delivered,
6544 ts_diff,
6545 (bw >> 32),
6546 (bw & 0x00000000ffffffff),
6547 0, 0, 0, 0);
6548 bw = ts_bw;
6549 } else if (ts_bw && (ts_bw < bw)) {
6550 bbr_log_type_bbrupd(bbr, 7, cts,
6551 delivered,
6552 ts_diff,
6553 (bw >> 32),
6554 (bw & 0x00000000ffffffff),
6555 0, 0, 0, 0);
6556 bw = ts_bw;
6557 }
6558 }
6559 }
6560 if (rsm->r_first_sent_time &&
6561 TSTMP_GT(rsm->r_tim_lastsent[(rsm->r_rtr_cnt -1)],rsm->r_first_sent_time)) {
6562 uint64_t sbw, sti;
6563 /*
6564 * We use what was in flight at the time of our
6565 * send and the size of this send to figure
6566 * out what we have been sending at (amount).
6567 * For the time we take from the time of
6568 * the send of the first send outstanding
6569 * until this send plus this sends pacing
6570 * time. This gives us a good calculation
6571 * as to the rate we have been sending at.
6572 */
6573
6574 sbw = (uint64_t)(rsm->r_flight_at_send);
6575 sbw *= (uint64_t)USECS_IN_SECOND;
6576 sti = rsm->r_tim_lastsent[(rsm->r_rtr_cnt -1)] - rsm->r_first_sent_time;
6577 sti += rsm->r_pacing_delay;
6578 sbw /= sti;
6579 if (sbw < bw) {
6580 bbr_log_type_bbrupd(bbr, 6, cts,
6581 delivered,
6582 (uint32_t)sti,
6583 (bw >> 32),
6584 (uint32_t)bw,
6585 rsm->r_first_sent_time, 0, (sbw >> 32),
6586 (uint32_t)sbw);
6587 bw = sbw;
6588 }
6589 }
6590 /* Use the google algorithm for b/w measurements */
6591 bbr->r_ctl.rc_bbr_cur_del_rate = bw;
6592 if ((rsm->r_app_limited == 0) ||
6593 (bw > get_filter_value(&bbr->r_ctl.rc_delrate))) {
6594 tcp_bbr_commit_bw(bbr, cts);
6595 bbr_log_type_bbrupd(bbr, 10, cts, (uint32_t)tim, delivered,
6596 0, 0, 0, 0, bbr->r_ctl.rc_del_time, rsm->r_del_time);
6597 }
6598 }
6599}
6600
6601static void
6602bbr_google_measurement(struct tcp_bbr *bbr, struct bbr_sendmap *rsm, uint32_t rtt, uint32_t cts)
6603{
6604 if (bbr->rc_in_persist == 0) {
6605 /* We log only when not in persist */
6606 /* Translate to a Bytes Per Second */
6607 uint64_t tim, bw;
6608 uint32_t delivered;
6609 int no_apply = 0;
6610
6611 if (TSTMP_GT(bbr->r_ctl.rc_del_time, rsm->r_del_time))
6612 tim = (uint64_t)(bbr->r_ctl.rc_del_time - rsm->r_del_time);
6613 else
6614 tim = 1;
6615 /*
6616 * Now that we have processed the tim (skipping the sample
6617 * or possibly updating the time, go ahead and
6618 * calculate the cdr.
6619 */
6620 delivered = (bbr->r_ctl.rc_delivered - rsm->r_delivered);
6621 bw = (uint64_t)delivered;
6622 bw *= (uint64_t)USECS_IN_SECOND;
6623 bw /= tim;
6624 if (tim < bbr->r_ctl.rc_lowest_rtt) {
6625 bbr_log_type_bbrupd(bbr, 99, cts, (uint32_t)tim, delivered,
6626 tim, bbr->r_ctl.rc_lowest_rtt, 0, 0, 0, 0);
6627
6628 no_apply = 1;
6629 }
6630 /*
6631 * If we are using this b/w shove it in now so we
6632 * can see in the trace viewer if it gets over-ridden.
6633 */
6634 bbr->r_ctl.rc_bbr_cur_del_rate = bw;
6635 /* Gate by the sending rate */
6636 if (rsm->r_first_sent_time &&
6637 TSTMP_GT(rsm->r_tim_lastsent[(rsm->r_rtr_cnt -1)],rsm->r_first_sent_time)) {
6638 uint64_t sbw, sti;
6639 /*
6640 * We use what was in flight at the time of our
6641 * send and the size of this send to figure
6642 * out what we have been sending at (amount).
6643 * For the time we take from the time of
6644 * the send of the first send outstanding
6645 * until this send plus this sends pacing
6646 * time. This gives us a good calculation
6647 * as to the rate we have been sending at.
6648 */
6649
6650 sbw = (uint64_t)(rsm->r_flight_at_send);
6651 sbw *= (uint64_t)USECS_IN_SECOND;
6652 sti = rsm->r_tim_lastsent[(rsm->r_rtr_cnt -1)] - rsm->r_first_sent_time;
6653 sti += rsm->r_pacing_delay;
6654 sbw /= sti;
6655 if (sbw < bw) {
6656 bbr_log_type_bbrupd(bbr, 6, cts,
6657 delivered,
6658 (uint32_t)sti,
6659 (bw >> 32),
6660 (uint32_t)bw,
6661 rsm->r_first_sent_time, 0, (sbw >> 32),
6662 (uint32_t)sbw);
6663 bw = sbw;
6664 }
6665 if ((sti > tim) &&
6666 (sti < bbr->r_ctl.rc_lowest_rtt)) {
6667 bbr_log_type_bbrupd(bbr, 99, cts, (uint32_t)tim, delivered,
6668 (uint32_t)sti, bbr->r_ctl.rc_lowest_rtt, 0, 0, 0, 0);
6669 no_apply = 1;
6670 } else
6671 no_apply = 0;
6672 }
6673 bbr->r_ctl.rc_bbr_cur_del_rate = bw;
6674 if ((no_apply == 0) &&
6675 ((rsm->r_app_limited == 0) ||
6676 (bw > get_filter_value(&bbr->r_ctl.rc_delrate)))) {
6677 tcp_bbr_commit_bw(bbr, cts);
6678 bbr_log_type_bbrupd(bbr, 10, cts, (uint32_t)tim, delivered,
6679 0, 0, 0, 0, bbr->r_ctl.rc_del_time, rsm->r_del_time);
6680 }
6681 }
6682}
6683
6684static void
6685bbr_update_bbr_info(struct tcp_bbr *bbr, struct bbr_sendmap *rsm, uint32_t rtt, uint32_t cts, uint32_t tsin,
6686 uint32_t uts, int32_t match, uint32_t rsm_send_time, int32_t ack_type, struct tcpopt *to)
6687{
6688 uint64_t old_rttprop;
6689
6690 /* Update our delivery time and amount */
6691 bbr->r_ctl.rc_delivered += (rsm->r_end - rsm->r_start);
6692 bbr->r_ctl.rc_del_time = cts;
6693 if (rtt == 0) {
6694 /*
6695 * 0 means its a retransmit, for now we don't use these for
6696 * the rest of BBR.
6697 */
6698 return;
6699 }
6700 if ((bbr->rc_use_google == 0) &&
6701 (match != BBR_RTT_BY_EXACTMATCH) &&
6702 (match != BBR_RTT_BY_TIMESTAMP)){
6703 /*
6704 * We get a lot of rtt updates, lets not pay attention to
6705 * any that are not an exact match. That way we don't have
6706 * to worry about timestamps and the whole nonsense of
6707 * unsure if its a retransmission etc (if we ever had the
6708 * timestamp fixed to always have the last thing sent this
6709 * would not be a issue).
6710 */
6711 return;
6712 }
6713 if ((bbr_no_retran && bbr->rc_use_google) &&
6714 (match != BBR_RTT_BY_EXACTMATCH) &&
6715 (match != BBR_RTT_BY_TIMESTAMP)){
6716 /*
6717 * We only do measurements in google mode
6718 * with bbr_no_retran on for sure things.
6719 */
6720 return;
6721 }
6722 /* Only update srtt if we know by exact match */
6723 tcp_bbr_xmit_timer(bbr, rtt, rsm_send_time, rsm->r_start, tsin);
6724 if (ack_type == BBR_CUM_ACKED)
6725 bbr->rc_ack_is_cumack = 1;
6726 else
6727 bbr->rc_ack_is_cumack = 0;
6728 old_rttprop = bbr_get_rtt(bbr, BBR_RTT_PROP);
6729 /*
6730 * Note the following code differs to the original
6731 * BBR spec. It calls for <= not <. However after a
6732 * long discussion in email with Neal, he acknowledged
6733 * that it should be < than so that we will have flows
6734 * going into probe-rtt (we were seeing cases where that
6735 * did not happen and caused ugly things to occur). We
6736 * have added this agreed upon fix to our code base.
6737 */
6738 if (rtt < old_rttprop) {
6739 /* Update when we last saw a rtt drop */
6740 bbr_log_rtt_shrinks(bbr, cts, 0, rtt, __LINE__, BBR_RTTS_NEWRTT, 0);
6741 bbr_set_reduced_rtt(bbr, cts, __LINE__);
6742 }
6743 bbr_log_type_bbrrttprop(bbr, rtt, (rsm ? rsm->r_end : 0), uts, cts,
6744 match, rsm->r_start, rsm->r_flags);
6745 apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, cts);
6746 if (old_rttprop != bbr_get_rtt(bbr, BBR_RTT_PROP)) {
6747 /*
6748 * The RTT-prop moved, reset the target (may be a
6749 * nop for some states).
6750 */
6751 bbr_set_state_target(bbr, __LINE__);
6752 if (bbr->rc_bbr_state == BBR_STATE_PROBE_RTT)
6753 bbr_log_rtt_shrinks(bbr, cts, 0, 0,
6754 __LINE__, BBR_RTTS_NEW_TARGET, 0);
6755 else if (old_rttprop < bbr_get_rtt(bbr, BBR_RTT_PROP))
6756 /* It went up */
6757 bbr_check_probe_rtt_limits(bbr, cts);
6758 }
6759 if ((bbr->rc_use_google == 0) &&
6760 (match == BBR_RTT_BY_TIMESTAMP)) {
6761 /*
6762 * We don't do b/w update with
6763 * these since they are not really
6764 * reliable.
6765 */
6766 return;
6767 }
6768 if (bbr->r_ctl.r_app_limited_until &&
6769 (bbr->r_ctl.rc_delivered >= bbr->r_ctl.r_app_limited_until)) {
6770 /* We are no longer app-limited */
6771 bbr->r_ctl.r_app_limited_until = 0;
6772 }
6773 if (bbr->rc_use_google) {
6774 bbr_google_measurement(bbr, rsm, rtt, cts);
6775 } else {
6776 bbr_nf_measurement(bbr, rsm, rtt, cts);
6777 }
6778}
6779
6780/*
6781 * Convert a timestamp that the main stack
6782 * uses (milliseconds) into one that bbr uses
6783 * (microseconds). Return that converted timestamp.
6784 */
6785static uint32_t
6786bbr_ts_convert(uint32_t cts) {
6787 uint32_t sec, msec;
6788
6789 sec = cts / MS_IN_USEC;
6790 msec = cts - (MS_IN_USEC * sec);
6791 return ((sec * USECS_IN_SECOND) + (msec * MS_IN_USEC));
6792}
6793
6794/*
6795 * Return 0 if we did not update the RTT time, return
6796 * 1 if we did.
6797 */
6798static int
6799bbr_update_rtt(struct tcpcb *tp, struct tcp_bbr *bbr,
6800 struct bbr_sendmap *rsm, struct tcpopt *to, uint32_t cts, int32_t ack_type, uint32_t th_ack)
6801{
6802 int32_t i;
6803 uint32_t t, uts = 0;
6804
6805 if ((rsm->r_flags & BBR_ACKED) ||
6806 (rsm->r_flags & BBR_WAS_RENEGED) ||
6807 (rsm->r_flags & BBR_RXT_CLEARED)) {
6808 /* Already done */
6809 return (0);
6810 }
6811 if (rsm->r_rtt_not_allowed) {
6812 /* Not allowed */
6813 return (0);
6814 }
6815 if (rsm->r_rtr_cnt == 1) {
6816 /*
6817 * Only one transmit. Hopefully the normal case.
6818 */
6819 if (TSTMP_GT(cts, rsm->r_tim_lastsent[0]))
6820 t = cts - rsm->r_tim_lastsent[0];
6821 else
6822 t = 1;
6823 if ((int)t <= 0)
6824 t = 1;
6825 bbr->r_ctl.rc_last_rtt = t;
6826 bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, 0,
6827 BBR_RTT_BY_EXACTMATCH, rsm->r_tim_lastsent[0], ack_type, to);
6828 return (1);
6829 }
6830 /* Convert to usecs */
6831 if ((bbr_can_use_ts_for_rtt == 1) &&
6832 (bbr->rc_use_google == 1) &&
6833 (ack_type == BBR_CUM_ACKED) &&
6834 (to->to_flags & TOF_TS) &&
6835 (to->to_tsecr != 0)) {
6836 t = tcp_tv_to_mssectick(&bbr->rc_tv) - to->to_tsecr;
6837 if (t < 1)
6838 t = 1;
6839 t *= MS_IN_USEC;
6840 bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, 0,
6841 BBR_RTT_BY_TIMESTAMP,
6842 rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)],
6843 ack_type, to);
6844 return (1);
6845 }
6846 uts = bbr_ts_convert(to->to_tsecr);
6847 if ((to->to_flags & TOF_TS) &&
6848 (to->to_tsecr != 0) &&
6849 (ack_type == BBR_CUM_ACKED) &&
6850 ((rsm->r_flags & BBR_OVERMAX) == 0)) {
6851 /*
6852 * Now which timestamp does it match? In this block the ACK
6853 * may be coming from a previous transmission.
6854 */
6855 uint32_t fudge;
6856
6857 fudge = BBR_TIMER_FUDGE;
6858 for (i = 0; i < rsm->r_rtr_cnt; i++) {
6859 if ((SEQ_GEQ(uts, (rsm->r_tim_lastsent[i] - fudge))) &&
6860 (SEQ_LEQ(uts, (rsm->r_tim_lastsent[i] + fudge)))) {
6861 if (TSTMP_GT(cts, rsm->r_tim_lastsent[i]))
6862 t = cts - rsm->r_tim_lastsent[i];
6863 else
6864 t = 1;
6865 if ((int)t <= 0)
6866 t = 1;
6867 bbr->r_ctl.rc_last_rtt = t;
6868 bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, uts, BBR_RTT_BY_TSMATCHING,
6869 rsm->r_tim_lastsent[i], ack_type, to);
6870 if ((i + 1) < rsm->r_rtr_cnt) {
6871 /* Likely */
6872 return (0);
6873 } else if (rsm->r_flags & BBR_TLP) {
6874 bbr->rc_tlp_rtx_out = 0;
6875 }
6876 return (1);
6877 }
6878 }
6879 /* Fall through if we can't find a matching timestamp */
6880 }
6881 /*
6882 * Ok its a SACK block that we retransmitted. or a windows
6883 * machine without timestamps. We can tell nothing from the
6884 * time-stamp since its not there or the time the peer last
6885 * recieved a segment that moved forward its cum-ack point.
6886 *
6887 * Lets look at the last retransmit and see what we can tell
6888 * (with BBR for space we only keep 2 note we have to keep
6889 * at least 2 so the map can not be condensed more).
6890 */
6891 i = rsm->r_rtr_cnt - 1;
6892 if (TSTMP_GT(cts, rsm->r_tim_lastsent[i]))
6893 t = cts - rsm->r_tim_lastsent[i];
6894 else
6895 goto not_sure;
6896 if (t < bbr->r_ctl.rc_lowest_rtt) {
6897 /*
6898 * We retransmitted and the ack came back in less
6899 * than the smallest rtt we have observed in the
6900 * windowed rtt. We most likey did an improper
6901 * retransmit as outlined in 4.2 Step 3 point 2 in
6902 * the rack-draft.
6903 *
6904 * Use the prior transmission to update all the
6905 * information as long as there is only one prior
6906 * transmission.
6907 */
6908 if ((rsm->r_flags & BBR_OVERMAX) == 0) {
6909#ifdef BBR_INVARIANTS
6910 if (rsm->r_rtr_cnt == 1)
6911 panic("rsm:%p bbr:%p rsm has overmax and only 1 retranmit flags:%x?", rsm, bbr, rsm->r_flags);
6912#endif
6913 i = rsm->r_rtr_cnt - 2;
6914 if (TSTMP_GT(cts, rsm->r_tim_lastsent[i]))
6915 t = cts - rsm->r_tim_lastsent[i];
6916 else
6917 t = 1;
6918 bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, uts, BBR_RTT_BY_EARLIER_RET,
6919 rsm->r_tim_lastsent[i], ack_type, to);
6920 return (0);
6921 } else {
6922 /*
6923 * Too many prior transmissions, just
6924 * updated BBR delivered
6925 */
6926not_sure:
6927 bbr_update_bbr_info(bbr, rsm, 0, cts, to->to_tsecr, uts,
6928 BBR_RTT_BY_SOME_RETRAN, 0, ack_type, to);
6929 }
6930 } else {
6931 /*
6932 * We retransmitted it and the retransmit did the
6933 * job.
6934 */
6935 if (rsm->r_flags & BBR_TLP)
6936 bbr->rc_tlp_rtx_out = 0;
6937 if ((rsm->r_flags & BBR_OVERMAX) == 0)
6938 bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, uts,
6939 BBR_RTT_BY_THIS_RETRAN, 0, ack_type, to);
6940 else
6941 bbr_update_bbr_info(bbr, rsm, 0, cts, to->to_tsecr, uts,
6942 BBR_RTT_BY_SOME_RETRAN, 0, ack_type, to);
6943 return (1);
6944 }
6945 return (0);
6946}
6947
6948/*
6949 * Mark the SACK_PASSED flag on all entries prior to rsm send wise.
6950 */
6951static void
6952bbr_log_sack_passed(struct tcpcb *tp,
6953 struct tcp_bbr *bbr, struct bbr_sendmap *rsm)
6954{
6955 struct bbr_sendmap *nrsm;
6956
6957 nrsm = rsm;
6958 TAILQ_FOREACH_REVERSE_FROM(nrsm, &bbr->r_ctl.rc_tmap,
6959 bbr_head, r_tnext) {
6960 if (nrsm == rsm) {
6961 /* Skip orginal segment he is acked */
6962 continue;
6963 }
6964 if (nrsm->r_flags & BBR_ACKED) {
6965 /* Skip ack'd segments */
6966 continue;
6967 }
6968 if (nrsm->r_flags & BBR_SACK_PASSED) {
6969 /*
6970 * We found one that is already marked
6971 * passed, we have been here before and
6972 * so all others below this are marked.
6973 */
6974 break;
6975 }
6976 BBR_STAT_INC(bbr_sack_passed);
6977 nrsm->r_flags |= BBR_SACK_PASSED;
6978 if (((nrsm->r_flags & BBR_MARKED_LOST) == 0) &&
6979 bbr_is_lost(bbr, nrsm, bbr->r_ctl.rc_rcvtime)) {
6980 bbr->r_ctl.rc_lost += nrsm->r_end - nrsm->r_start;
6981 bbr->r_ctl.rc_lost_bytes += nrsm->r_end - nrsm->r_start;
6982 nrsm->r_flags |= BBR_MARKED_LOST;
6983 }
6984 nrsm->r_flags &= ~BBR_WAS_SACKPASS;
6985 }
6986}
6987
6988/*
6989 * Returns the number of bytes that were
6990 * newly ack'd by sack blocks.
6991 */
6992static uint32_t
6993bbr_proc_sack_blk(struct tcpcb *tp, struct tcp_bbr *bbr, struct sackblk *sack,
6994 struct tcpopt *to, struct bbr_sendmap **prsm, uint32_t cts)
6995{
6996 int32_t times = 0;
6997 uint32_t start, end, changed = 0;
6998 struct bbr_sendmap *rsm, *nrsm;
6999 int32_t used_ref = 1;
7000 uint8_t went_back = 0, went_fwd = 0;
7001
7002 start = sack->start;
7003 end = sack->end;
7004 rsm = *prsm;
7005 if (rsm == NULL)
7006 used_ref = 0;
7007
7008 /* Do we locate the block behind where we last were? */
7009 if (rsm && SEQ_LT(start, rsm->r_start)) {
7010 went_back = 1;
7011 TAILQ_FOREACH_REVERSE_FROM(rsm, &bbr->r_ctl.rc_map, bbr_head, r_next) {
7012 if (SEQ_GEQ(start, rsm->r_start) &&
7013 SEQ_LT(start, rsm->r_end)) {
7014 goto do_rest_ofb;
7015 }
7016 }
7017 }
7018start_at_beginning:
7019 went_fwd = 1;
7020 /*
7021 * Ok lets locate the block where this guy is fwd from rsm (if its
7022 * set)
7023 */
7024 TAILQ_FOREACH_FROM(rsm, &bbr->r_ctl.rc_map, r_next) {
7025 if (SEQ_GEQ(start, rsm->r_start) &&
7026 SEQ_LT(start, rsm->r_end)) {
7027 break;
7028 }
7029 }
7030do_rest_ofb:
7031 if (rsm == NULL) {
7032 /*
7033 * This happens when we get duplicate sack blocks with the
7034 * same end. For example SACK 4: 100 SACK 3: 100 The sort
7035 * will not change there location so we would just start at
7036 * the end of the first one and get lost.
7037 */
7038 if (tp->t_flags & TF_SENTFIN) {
7039 /*
7040 * Check to see if we have not logged the FIN that
7041 * went out.
7042 */
7043 nrsm = TAILQ_LAST_FAST(&bbr->r_ctl.rc_map, bbr_sendmap, r_next);
7044 if (nrsm && (nrsm->r_end + 1) == tp->snd_max) {
7045 /*
7046 * Ok we did not get the FIN logged.
7047 */
7048 nrsm->r_end++;
7049 rsm = nrsm;
7050 goto do_rest_ofb;
7051 }
7052 }
7053 if (times == 1) {
7054#ifdef BBR_INVARIANTS
7055 panic("tp:%p bbr:%p sack:%p to:%p prsm:%p",
7056 tp, bbr, sack, to, prsm);
7057#else
7058 goto out;
7059#endif
7060 }
7061 times++;
7062 BBR_STAT_INC(bbr_sack_proc_restart);
7063 rsm = NULL;
7064 goto start_at_beginning;
7065 }
7066 /* Ok we have an ACK for some piece of rsm */
7067 if (rsm->r_start != start) {
7068 /*
7069 * Need to split this in two pieces the before and after.
7070 */
7071 if (bbr_sack_mergable(rsm, start, end))
7072 nrsm = bbr_alloc_full_limit(bbr);
7073 else
7074 nrsm = bbr_alloc_limit(bbr, BBR_LIMIT_TYPE_SPLIT);
7075 if (nrsm == NULL) {
7076 /* We could not allocate ignore the sack */
7077 struct sackblk blk;
7078
7079 blk.start = start;
7080 blk.end = end;
7081 sack_filter_reject(&bbr->r_ctl.bbr_sf, &blk);
7082 goto out;
7083 }
7084 bbr_clone_rsm(bbr, nrsm, rsm, start);
7085 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next);
7086 if (rsm->r_in_tmap) {
7087 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext);
7088 nrsm->r_in_tmap = 1;
7089 }
7090 rsm->r_flags &= (~BBR_HAS_FIN);
7091 rsm = nrsm;
7092 }
7093 if (SEQ_GEQ(end, rsm->r_end)) {
7094 /*
7095 * The end of this block is either beyond this guy or right
7096 * at this guy.
7097 */
7098 if ((rsm->r_flags & BBR_ACKED) == 0) {
7099 bbr_update_rtt(tp, bbr, rsm, to, cts, BBR_SACKED, 0);
7100 changed += (rsm->r_end - rsm->r_start);
7101 bbr->r_ctl.rc_sacked += (rsm->r_end - rsm->r_start);
7102 bbr_log_sack_passed(tp, bbr, rsm);
7103 if (rsm->r_flags & BBR_MARKED_LOST) {
7104 bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start;
7105 }
7106 /* Is Reordering occuring? */
7107 if (rsm->r_flags & BBR_SACK_PASSED) {
7108 BBR_STAT_INC(bbr_reorder_seen);
7109 bbr->r_ctl.rc_reorder_ts = cts;
7110 if (rsm->r_flags & BBR_MARKED_LOST) {
7111 bbr->r_ctl.rc_lost -= rsm->r_end - rsm->r_start;
7112 if (SEQ_GT(bbr->r_ctl.rc_lt_lost, bbr->r_ctl.rc_lost))
7113 /* LT sampling also needs adjustment */
7114 bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost;
7115 }
7116 }
7117 rsm->r_flags |= BBR_ACKED;
7118 rsm->r_flags &= ~(BBR_TLP|BBR_WAS_RENEGED|BBR_RXT_CLEARED|BBR_MARKED_LOST);
7119 if (rsm->r_in_tmap) {
7120 TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
7121 rsm->r_in_tmap = 0;
7122 }
7123 }
7124 bbr_isit_a_pkt_epoch(bbr, cts, rsm, __LINE__, BBR_SACKED);
7125 if (end == rsm->r_end) {
7126 /* This block only - done */
7127 goto out;
7128 }
7129 /* There is more not coverend by this rsm move on */
7130 start = rsm->r_end;
7131 nrsm = TAILQ_NEXT(rsm, r_next);
7132 rsm = nrsm;
7133 times = 0;
7134 goto do_rest_ofb;
7135 }
7136 if (rsm->r_flags & BBR_ACKED) {
7137 /* Been here done that */
7138 goto out;
7139 }
7140 /* Ok we need to split off this one at the tail */
7141 if (bbr_sack_mergable(rsm, start, end))
7142 nrsm = bbr_alloc_full_limit(bbr);
7143 else
7144 nrsm = bbr_alloc_limit(bbr, BBR_LIMIT_TYPE_SPLIT);
7145 if (nrsm == NULL) {
7146 /* failed XXXrrs what can we do but loose the sack info? */
7147 struct sackblk blk;
7148
7149 blk.start = start;
7150 blk.end = end;
7151 sack_filter_reject(&bbr->r_ctl.bbr_sf, &blk);
7152 goto out;
7153 }
7154 /* Clone it */
7155 bbr_clone_rsm(bbr, nrsm, rsm, end);
7156 /* The sack block does not cover this guy fully */
7157 rsm->r_flags &= (~BBR_HAS_FIN);
7158 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next);
7159 if (rsm->r_in_tmap) {
7160 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext);
7161 nrsm->r_in_tmap = 1;
7162 }
7163 nrsm->r_dupack = 0;
7164 bbr_update_rtt(tp, bbr, rsm, to, cts, BBR_SACKED, 0);
7165 bbr_isit_a_pkt_epoch(bbr, cts, rsm, __LINE__, BBR_SACKED);
7166 changed += (rsm->r_end - rsm->r_start);
7167 bbr->r_ctl.rc_sacked += (rsm->r_end - rsm->r_start);
7168 bbr_log_sack_passed(tp, bbr, rsm);
7169 /* Is Reordering occuring? */
7170 if (rsm->r_flags & BBR_MARKED_LOST) {
7171 bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start;
7172 }
7173 if (rsm->r_flags & BBR_SACK_PASSED) {
7174 BBR_STAT_INC(bbr_reorder_seen);
7175 bbr->r_ctl.rc_reorder_ts = cts;
7176 if (rsm->r_flags & BBR_MARKED_LOST) {
7177 bbr->r_ctl.rc_lost -= rsm->r_end - rsm->r_start;
7178 if (SEQ_GT(bbr->r_ctl.rc_lt_lost, bbr->r_ctl.rc_lost))
7179 /* LT sampling also needs adjustment */
7180 bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost;
7181 }
7182 }
7183 rsm->r_flags &= ~(BBR_TLP|BBR_WAS_RENEGED|BBR_RXT_CLEARED|BBR_MARKED_LOST);
7184 rsm->r_flags |= BBR_ACKED;
7185 if (rsm->r_in_tmap) {
7186 TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
7187 rsm->r_in_tmap = 0;
7188 }
7189out:
7190 if (rsm && (rsm->r_flags & BBR_ACKED)) {
7191 /*
7192 * Now can we merge this newly acked
7193 * block with either the previous or
7194 * next block?
7195 */
7196 nrsm = TAILQ_NEXT(rsm, r_next);
7197 if (nrsm &&
7198 (nrsm->r_flags & BBR_ACKED)) {
7199 /* yep this and next can be merged */
7200 rsm = bbr_merge_rsm(bbr, rsm, nrsm);
7201 }
7202 /* Now what about the previous? */
7203 nrsm = TAILQ_PREV(rsm, bbr_head, r_next);
7204 if (nrsm &&
7205 (nrsm->r_flags & BBR_ACKED)) {
7206 /* yep the previous and this can be merged */
7207 rsm = bbr_merge_rsm(bbr, nrsm, rsm);
7208 }
7209 }
7210 if (used_ref == 0) {
7211 BBR_STAT_INC(bbr_sack_proc_all);
7212 } else {
7213 BBR_STAT_INC(bbr_sack_proc_short);
7214 }
7215 if (went_fwd && went_back) {
7216 BBR_STAT_INC(bbr_sack_search_both);
7217 } else if (went_fwd) {
7218 BBR_STAT_INC(bbr_sack_search_fwd);
7219 } else if (went_back) {
7220 BBR_STAT_INC(bbr_sack_search_back);
7221 }
7222 /* Save off where the next seq is */
7223 if (rsm)
7224 bbr->r_ctl.rc_sacklast = TAILQ_NEXT(rsm, r_next);
7225 else
7226 bbr->r_ctl.rc_sacklast = NULL;
7227 *prsm = rsm;
7228 return (changed);
7229}
7230
7231static void inline
7232bbr_peer_reneges(struct tcp_bbr *bbr, struct bbr_sendmap *rsm, tcp_seq th_ack)
7233{
7234 struct bbr_sendmap *tmap;
7235
7236 BBR_STAT_INC(bbr_reneges_seen);
7237 tmap = NULL;
7238 while (rsm && (rsm->r_flags & BBR_ACKED)) {
7239 /* Its no longer sacked, mark it so */
7240 uint32_t oflags;
7241 bbr->r_ctl.rc_sacked -= (rsm->r_end - rsm->r_start);
7242#ifdef BBR_INVARIANTS
7243 if (rsm->r_in_tmap) {
7244 panic("bbr:%p rsm:%p flags:0x%x in tmap?",
7245 bbr, rsm, rsm->r_flags);
7246 }
7247#endif
7248 oflags = rsm->r_flags;
7249 if (rsm->r_flags & BBR_MARKED_LOST) {
7250 bbr->r_ctl.rc_lost -= rsm->r_end - rsm->r_start;
7251 bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start;
7252 if (SEQ_GT(bbr->r_ctl.rc_lt_lost, bbr->r_ctl.rc_lost))
7253 /* LT sampling also needs adjustment */
7254 bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost;
7255 }
7256 rsm->r_flags &= ~(BBR_ACKED | BBR_SACK_PASSED | BBR_WAS_SACKPASS | BBR_MARKED_LOST);
7257 rsm->r_flags |= BBR_WAS_RENEGED;
7258 rsm->r_flags |= BBR_RXT_CLEARED;
7259 bbr_log_type_rsmclear(bbr, bbr->r_ctl.rc_rcvtime, rsm, oflags, __LINE__);
7260 /* Rebuild it into our tmap */
7261 if (tmap == NULL) {
7262 TAILQ_INSERT_HEAD(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
7263 tmap = rsm;
7264 } else {
7265 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, tmap, rsm, r_tnext);
7266 tmap = rsm;
7267 }
7268 tmap->r_in_tmap = 1;
7269 /*
7270 * XXXrrs Delivered? Should we do anything here?
7271 *
7272 * Of course we don't on a rxt timeout so maybe its ok that
7273 * we don't?
7274 *
7275 * For now lets not.
7276 */
7277 rsm = TAILQ_NEXT(rsm, r_next);
7278 }
7279 /*
7280 * Now lets possibly clear the sack filter so we start recognizing
7281 * sacks that cover this area.
7282 */
7283 sack_filter_clear(&bbr->r_ctl.bbr_sf, th_ack);
7284}
7285
7286static void
7287bbr_log_syn(struct tcpcb *tp, struct tcpopt *to)
7288{
7289 struct tcp_bbr *bbr;
7290 struct bbr_sendmap *rsm;
7291 uint32_t cts;
7292
7293 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
7294 cts = bbr->r_ctl.rc_rcvtime;
7295 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
7296 if (rsm && (rsm->r_flags & BBR_HAS_SYN)) {
7297 if ((rsm->r_end - rsm->r_start) <= 1) {
7298 /* Log out the SYN completely */
7299 bbr->r_ctl.rc_holes_rxt -= rsm->r_rtr_bytes;
7300 rsm->r_rtr_bytes = 0;
7301 TAILQ_REMOVE(&bbr->r_ctl.rc_map, rsm, r_next);
7302 if (rsm->r_in_tmap) {
7303 TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
7304 rsm->r_in_tmap = 0;
7305 }
7306 if (bbr->r_ctl.rc_next == rsm) {
7307 /* scoot along the marker */
7308 bbr->r_ctl.rc_next = TAILQ_FIRST(&bbr->r_ctl.rc_map);
7309 }
7310 if (to != NULL)
7311 bbr_update_rtt(tp, bbr, rsm, to, cts, BBR_CUM_ACKED, 0);
7312 bbr_free(bbr, rsm);
7313 } else {
7314 /* There is more (Fast open)? strip out SYN. */
7315 rsm->r_flags &= ~BBR_HAS_SYN;
7316 rsm->r_start++;
7317 }
7318 }
7319}
7320
7321/*
7322 * Returns the number of bytes that were
7323 * acknowledged by SACK blocks.
7324 */
7325
7326static uint32_t
7327bbr_log_ack(struct tcpcb *tp, struct tcpopt *to, struct tcphdr *th,
7328 uint32_t *prev_acked)
7329{
7330 uint32_t changed, last_seq, entered_recovery = 0;
7331 struct tcp_bbr *bbr;
7332 struct bbr_sendmap *rsm;
7333 struct sackblk sack, sack_blocks[TCP_MAX_SACK + 1];
7334 register uint32_t th_ack;
7335 int32_t i, j, k, new_sb, num_sack_blks = 0;
7336 uint32_t cts, acked, ack_point, sack_changed = 0;
7337 uint32_t p_maxseg, maxseg, p_acked = 0;
7338
7339 INP_WLOCK_ASSERT(tp->t_inpcb);
7340 if (tcp_get_flags(th) & TH_RST) {
7341 /* We don't log resets */
7342 return (0);
7343 }
7344 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
7345 cts = bbr->r_ctl.rc_rcvtime;
7346
7347 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
7348 changed = 0;
7349 maxseg = tp->t_maxseg - bbr->rc_last_options;
7350 p_maxseg = min(bbr->r_ctl.rc_pace_max_segs, maxseg);
7351 th_ack = th->th_ack;
7352 if (SEQ_GT(th_ack, tp->snd_una)) {
7353 acked = th_ack - tp->snd_una;
7354 bbr_log_progress_event(bbr, tp, ticks, PROGRESS_UPDATE, __LINE__);
7355 bbr->rc_tp->t_acktime = ticks;
7356 } else
7357 acked = 0;
7358 if (SEQ_LEQ(th_ack, tp->snd_una)) {
7359 /* Only sent here for sack processing */
7360 goto proc_sack;
7361 }
7362 if (rsm && SEQ_GT(th_ack, rsm->r_start)) {
7363 changed = th_ack - rsm->r_start;
7364 } else if ((rsm == NULL) && ((th_ack - 1) == tp->iss)) {
7365 /*
7366 * For the SYN incoming case we will not have called
7367 * tcp_output for the sending of the SYN, so there will be
7368 * no map. All other cases should probably be a panic.
7369 */
7370 if ((to->to_flags & TOF_TS) && (to->to_tsecr != 0)) {
7371 /*
7372 * We have a timestamp that can be used to generate
7373 * an initial RTT.
7374 */
7375 uint32_t ts, now, rtt;
7376
7377 ts = bbr_ts_convert(to->to_tsecr);
7378 now = bbr_ts_convert(tcp_tv_to_mssectick(&bbr->rc_tv));
7379 rtt = now - ts;
7380 if (rtt < 1)
7381 rtt = 1;
7382 bbr_log_type_bbrrttprop(bbr, rtt,
7383 tp->iss, 0, cts,
7384 BBR_RTT_BY_TIMESTAMP, tp->iss, 0);
7385 apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, cts);
7386 changed = 1;
7387 bbr->r_wanted_output = 1;
7388 goto out;
7389 }
7390 goto proc_sack;
7391 } else if (rsm == NULL) {
7392 goto out;
7393 }
7394 if (changed) {
7395 /*
7396 * The ACK point is advancing to th_ack, we must drop off
7397 * the packets in the rack log and calculate any eligble
7398 * RTT's.
7399 */
7400 bbr->r_wanted_output = 1;
7401more:
7402 if (rsm == NULL) {
7403 if (tp->t_flags & TF_SENTFIN) {
7404 /* if we send a FIN we will not hav a map */
7405 goto proc_sack;
7406 }
7407#ifdef BBR_INVARIANTS
7408 panic("No rack map tp:%p for th:%p state:%d bbr:%p snd_una:%u snd_max:%u chg:%d\n",
7409 tp,
7410 th, tp->t_state, bbr,
7411 tp->snd_una, tp->snd_max, changed);
7412#endif
7413 goto proc_sack;
7414 }
7415 }
7416 if (SEQ_LT(th_ack, rsm->r_start)) {
7417 /* Huh map is missing this */
7418#ifdef BBR_INVARIANTS
7419 printf("Rack map starts at r_start:%u for th_ack:%u huh? ts:%d rs:%d bbr:%p\n",
7420 rsm->r_start,
7421 th_ack, tp->t_state,
7422 bbr->r_state, bbr);
7423 panic("th-ack is bad bbr:%p tp:%p", bbr, tp);
7424#endif
7425 goto proc_sack;
7426 } else if (th_ack == rsm->r_start) {
7427 /* None here to ack */
7428 goto proc_sack;
7429 }
7430 /*
7431 * Clear the dup ack counter, it will
7432 * either be freed or if there is some
7433 * remaining we need to start it at zero.
7434 */
7435 rsm->r_dupack = 0;
7436 /* Now do we consume the whole thing? */
7437 if (SEQ_GEQ(th_ack, rsm->r_end)) {
7438 /* Its all consumed. */
7439 uint32_t left;
7440
7441 if (rsm->r_flags & BBR_ACKED) {
7442 /*
7443 * It was acked on the scoreboard -- remove it from
7444 * total
7445 */
7446 p_acked += (rsm->r_end - rsm->r_start);
7447 bbr->r_ctl.rc_sacked -= (rsm->r_end - rsm->r_start);
7448 if (bbr->r_ctl.rc_sacked == 0)
7449 bbr->r_ctl.rc_sacklast = NULL;
7450 } else {
7451 bbr_update_rtt(tp, bbr, rsm, to, cts, BBR_CUM_ACKED, th_ack);
7452 if (rsm->r_flags & BBR_MARKED_LOST) {
7453 bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start;
7454 }
7455 if (rsm->r_flags & BBR_SACK_PASSED) {
7456 /*
7457 * There are acked segments ACKED on the
7458 * scoreboard further up. We are seeing
7459 * reordering.
7460 */
7461 BBR_STAT_INC(bbr_reorder_seen);
7462 bbr->r_ctl.rc_reorder_ts = cts;
7463 if (rsm->r_flags & BBR_MARKED_LOST) {
7464 bbr->r_ctl.rc_lost -= rsm->r_end - rsm->r_start;
7465 if (SEQ_GT(bbr->r_ctl.rc_lt_lost, bbr->r_ctl.rc_lost))
7466 /* LT sampling also needs adjustment */
7467 bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost;
7468 }
7469 }
7470 rsm->r_flags &= ~BBR_MARKED_LOST;
7471 }
7472 bbr->r_ctl.rc_holes_rxt -= rsm->r_rtr_bytes;
7473 rsm->r_rtr_bytes = 0;
7474 TAILQ_REMOVE(&bbr->r_ctl.rc_map, rsm, r_next);
7475 if (rsm->r_in_tmap) {
7476 TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
7477 rsm->r_in_tmap = 0;
7478 }
7479 if (bbr->r_ctl.rc_next == rsm) {
7480 /* scoot along the marker */
7481 bbr->r_ctl.rc_next = TAILQ_FIRST(&bbr->r_ctl.rc_map);
7482 }
7483 bbr_isit_a_pkt_epoch(bbr, cts, rsm, __LINE__, BBR_CUM_ACKED);
7484 /* Adjust the packet counts */
7485 left = th_ack - rsm->r_end;
7486 /* Free back to zone */
7487 bbr_free(bbr, rsm);
7488 if (left) {
7489 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
7490 goto more;
7491 }
7492 goto proc_sack;
7493 }
7494 if (rsm->r_flags & BBR_ACKED) {
7495 /*
7496 * It was acked on the scoreboard -- remove it from total
7497 * for the part being cum-acked.
7498 */
7499 p_acked += (rsm->r_end - rsm->r_start);
7500 bbr->r_ctl.rc_sacked -= (th_ack - rsm->r_start);
7501 if (bbr->r_ctl.rc_sacked == 0)
7502 bbr->r_ctl.rc_sacklast = NULL;
7503 } else {
7504 /*
7505 * It was acked up to th_ack point for the first time
7506 */
7507 struct bbr_sendmap lrsm;
7508
7509 memcpy(&lrsm, rsm, sizeof(struct bbr_sendmap));
7510 lrsm.r_end = th_ack;
7511 bbr_update_rtt(tp, bbr, &lrsm, to, cts, BBR_CUM_ACKED, th_ack);
7512 }
7513 if ((rsm->r_flags & BBR_MARKED_LOST) &&
7514 ((rsm->r_flags & BBR_ACKED) == 0)) {
7515 /*
7516 * It was marked lost and partly ack'd now
7517 * for the first time. We lower the rc_lost_bytes
7518 * and still leave it MARKED.
7519 */
7520 bbr->r_ctl.rc_lost_bytes -= th_ack - rsm->r_start;
7521 }
7522 bbr_isit_a_pkt_epoch(bbr, cts, rsm, __LINE__, BBR_CUM_ACKED);
7523 bbr->r_ctl.rc_holes_rxt -= rsm->r_rtr_bytes;
7524 rsm->r_rtr_bytes = 0;
7525 /* adjust packet count */
7526 rsm->r_start = th_ack;
7527proc_sack:
7528 /* Check for reneging */
7529 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
7530 if (rsm && (rsm->r_flags & BBR_ACKED) && (th_ack == rsm->r_start)) {
7531 /*
7532 * The peer has moved snd_una up to the edge of this send,
7533 * i.e. one that it had previously acked. The only way that
7534 * can be true if the peer threw away data (space issues)
7535 * that it had previously sacked (else it would have given
7536 * us snd_una up to (rsm->r_end). We need to undo the acked
7537 * markings here.
7538 *
7539 * Note we have to look to make sure th_ack is our
7540 * rsm->r_start in case we get an old ack where th_ack is
7541 * behind snd_una.
7542 */
7543 bbr_peer_reneges(bbr, rsm, th->th_ack);
7544 }
7545 if ((to->to_flags & TOF_SACK) == 0) {
7546 /* We are done nothing left to log */
7547 goto out;
7548 }
7549 rsm = TAILQ_LAST_FAST(&bbr->r_ctl.rc_map, bbr_sendmap, r_next);
7550 if (rsm) {
7551 last_seq = rsm->r_end;
7552 } else {
7553 last_seq = tp->snd_max;
7554 }
7555 /* Sack block processing */
7556 if (SEQ_GT(th_ack, tp->snd_una))
7557 ack_point = th_ack;
7558 else
7559 ack_point = tp->snd_una;
7560 for (i = 0; i < to->to_nsacks; i++) {
7561 bcopy((to->to_sacks + i * TCPOLEN_SACK),
7562 &sack, sizeof(sack));
7563 sack.start = ntohl(sack.start);
7564 sack.end = ntohl(sack.end);
7565 if (SEQ_GT(sack.end, sack.start) &&
7566 SEQ_GT(sack.start, ack_point) &&
7567 SEQ_LT(sack.start, tp->snd_max) &&
7568 SEQ_GT(sack.end, ack_point) &&
7569 SEQ_LEQ(sack.end, tp->snd_max)) {
7570 if ((bbr->r_ctl.rc_num_small_maps_alloced > bbr_sack_block_limit) &&
7571 (SEQ_LT(sack.end, last_seq)) &&
7572 ((sack.end - sack.start) < (p_maxseg / 8))) {
7573 /*
7574 * Not the last piece and its smaller than
7575 * 1/8th of a p_maxseg. We ignore this.
7576 */
7577 BBR_STAT_INC(bbr_runt_sacks);
7578 continue;
7579 }
7580 sack_blocks[num_sack_blks] = sack;
7581 num_sack_blks++;
7582 } else if (SEQ_LEQ(sack.start, th_ack) &&
7583 SEQ_LEQ(sack.end, th_ack)) {
7584 /*
7585 * Its a D-SACK block.
7586 */
7587 tcp_record_dsack(tp, sack.start, sack.end, 0);
7588 }
7589 }
7590 if (num_sack_blks == 0)
7591 goto out;
7592 /*
7593 * Sort the SACK blocks so we can update the rack scoreboard with
7594 * just one pass.
7595 */
7596 new_sb = sack_filter_blks(&bbr->r_ctl.bbr_sf, sack_blocks,
7597 num_sack_blks, th->th_ack);
7598 ctf_log_sack_filter(bbr->rc_tp, new_sb, sack_blocks);
7599 BBR_STAT_ADD(bbr_sack_blocks, num_sack_blks);
7600 BBR_STAT_ADD(bbr_sack_blocks_skip, (num_sack_blks - new_sb));
7601 num_sack_blks = new_sb;
7602 if (num_sack_blks < 2) {
7603 goto do_sack_work;
7604 }
7605 /* Sort the sacks */
7606 for (i = 0; i < num_sack_blks; i++) {
7607 for (j = i + 1; j < num_sack_blks; j++) {
7608 if (SEQ_GT(sack_blocks[i].end, sack_blocks[j].end)) {
7609 sack = sack_blocks[i];
7610 sack_blocks[i] = sack_blocks[j];
7611 sack_blocks[j] = sack;
7612 }
7613 }
7614 }
7615 /*
7616 * Now are any of the sack block ends the same (yes some
7617 * implememtations send these)?
7618 */
7619again:
7620 if (num_sack_blks > 1) {
7621 for (i = 0; i < num_sack_blks; i++) {
7622 for (j = i + 1; j < num_sack_blks; j++) {
7623 if (sack_blocks[i].end == sack_blocks[j].end) {
7624 /*
7625 * Ok these two have the same end we
7626 * want the smallest end and then
7627 * throw away the larger and start
7628 * again.
7629 */
7630 if (SEQ_LT(sack_blocks[j].start, sack_blocks[i].start)) {
7631 /*
7632 * The second block covers
7633 * more area use that
7634 */
7635 sack_blocks[i].start = sack_blocks[j].start;
7636 }
7637 /*
7638 * Now collapse out the dup-sack and
7639 * lower the count
7640 */
7641 for (k = (j + 1); k < num_sack_blks; k++) {
7642 sack_blocks[j].start = sack_blocks[k].start;
7643 sack_blocks[j].end = sack_blocks[k].end;
7644 j++;
7645 }
7646 num_sack_blks--;
7647 goto again;
7648 }
7649 }
7650 }
7651 }
7652do_sack_work:
7653 rsm = bbr->r_ctl.rc_sacklast;
7654 for (i = 0; i < num_sack_blks; i++) {
7655 acked = bbr_proc_sack_blk(tp, bbr, &sack_blocks[i], to, &rsm, cts);
7656 if (acked) {
7657 bbr->r_wanted_output = 1;
7658 changed += acked;
7659 sack_changed += acked;
7660 }
7661 }
7662out:
7663 *prev_acked = p_acked;
7664 if ((sack_changed) && (!IN_RECOVERY(tp->t_flags))) {
7665 /*
7666 * Ok we have a high probability that we need to go in to
7667 * recovery since we have data sack'd
7668 */
7669 struct bbr_sendmap *rsm;
7670
7671 rsm = bbr_check_recovery_mode(tp, bbr, cts);
7672 if (rsm) {
7673 /* Enter recovery */
7674 entered_recovery = 1;
7675 bbr->r_wanted_output = 1;
7676 /*
7677 * When we enter recovery we need to assure we send
7678 * one packet.
7679 */
7680 if (bbr->r_ctl.rc_resend == NULL) {
7681 bbr->r_ctl.rc_resend = rsm;
7682 }
7683 }
7684 }
7685 if (IN_RECOVERY(tp->t_flags) && (entered_recovery == 0)) {
7686 /*
7687 * See if we need to rack-retransmit anything if so set it
7688 * up as the thing to resend assuming something else is not
7689 * already in that position.
7690 */
7691 if (bbr->r_ctl.rc_resend == NULL) {
7692 bbr->r_ctl.rc_resend = bbr_check_recovery_mode(tp, bbr, cts);
7693 }
7694 }
7695 /*
7696 * We return the amount that changed via sack, this is used by the
7697 * ack-received code to augment what was changed between th_ack <->
7698 * snd_una.
7699 */
7700 return (sack_changed);
7701}
7702
7703static void
7704bbr_strike_dupack(struct tcp_bbr *bbr)
7705{
7706 struct bbr_sendmap *rsm;
7707
7708 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap);
7709 if (rsm && (rsm->r_dupack < 0xff)) {
7710 rsm->r_dupack++;
7711 if (rsm->r_dupack >= DUP_ACK_THRESHOLD)
7712 bbr->r_wanted_output = 1;
7713 }
7714}
7715
7716/*
7717 * Return value of 1, we do not need to call bbr_process_data().
7718 * return value of 0, bbr_process_data can be called.
7719 * For ret_val if its 0 the TCB is locked and valid, if its non-zero
7720 * its unlocked and probably unsafe to touch the TCB.
7721 */
7722static int
7723bbr_process_ack(struct mbuf *m, struct tcphdr *th, struct socket *so,
7724 struct tcpcb *tp, struct tcpopt *to,
7725 uint32_t tiwin, int32_t tlen,
7726 int32_t * ofia, int32_t thflags, int32_t * ret_val)
7727{
7728 int32_t ourfinisacked = 0;
7729 int32_t acked_amount;
7730 uint16_t nsegs;
7731 int32_t acked;
7732 uint32_t lost, sack_changed = 0;
7733 struct mbuf *mfree;
7734 struct tcp_bbr *bbr;
7735 uint32_t prev_acked = 0;
7736
7737 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
7738 lost = bbr->r_ctl.rc_lost;
7739 nsegs = max(1, m->m_pkthdr.lro_nsegs);
7740 if (SEQ_GT(th->th_ack, tp->snd_max)) {
7741 ctf_do_dropafterack(m, tp, th, thflags, tlen, ret_val);
7742 bbr->r_wanted_output = 1;
7743 return (1);
7744 }
7745 if (SEQ_GEQ(th->th_ack, tp->snd_una) || to->to_nsacks) {
7746 /* Process the ack */
7747 if (bbr->rc_in_persist)
7748 tp->t_rxtshift = 0;
7749 if ((th->th_ack == tp->snd_una) && (tiwin == tp->snd_wnd))
7750 bbr_strike_dupack(bbr);
7751 sack_changed = bbr_log_ack(tp, to, th, &prev_acked);
7752 }
7753 bbr_lt_bw_sampling(bbr, bbr->r_ctl.rc_rcvtime, (bbr->r_ctl.rc_lost > lost));
7754 if (__predict_false(SEQ_LEQ(th->th_ack, tp->snd_una))) {
7755 /*
7756 * Old ack, behind the last one rcv'd or a duplicate ack
7757 * with SACK info.
7758 */
7759 if (th->th_ack == tp->snd_una) {
7760 bbr_ack_received(tp, bbr, th, 0, sack_changed, prev_acked, __LINE__, 0);
7761 if (bbr->r_state == TCPS_SYN_SENT) {
7762 /*
7763 * Special case on where we sent SYN. When
7764 * the SYN-ACK is processed in syn_sent
7765 * state it bumps the snd_una. This causes
7766 * us to hit here even though we did ack 1
7767 * byte.
7768 *
7769 * Go through the nothing left case so we
7770 * send data.
7771 */
7772 goto nothing_left;
7773 }
7774 }
7775 return (0);
7776 }
7777 /*
7778 * If we reach this point, ACK is not a duplicate, i.e., it ACKs
7779 * something we sent.
7780 */
7781 if (tp->t_flags & TF_NEEDSYN) {
7782 /*
7783 * T/TCP: Connection was half-synchronized, and our SYN has
7784 * been ACK'd (so connection is now fully synchronized). Go
7785 * to non-starred state, increment snd_una for ACK of SYN,
7786 * and check if we can do window scaling.
7787 */
7788 tp->t_flags &= ~TF_NEEDSYN;
7789 tp->snd_una++;
7790 /* Do window scaling? */
7791 if ((tp->t_flags & (TF_RCVD_SCALE | TF_REQ_SCALE)) ==
7792 (TF_RCVD_SCALE | TF_REQ_SCALE)) {
7793 tp->rcv_scale = tp->request_r_scale;
7794 /* Send window already scaled. */
7795 }
7796 }
7797 INP_WLOCK_ASSERT(tp->t_inpcb);
7798
7799 acked = BYTES_THIS_ACK(tp, th);
7800 KMOD_TCPSTAT_ADD(tcps_rcvackpack, (int)nsegs);
7801 KMOD_TCPSTAT_ADD(tcps_rcvackbyte, acked);
7802
7803 /*
7804 * If we just performed our first retransmit, and the ACK arrives
7805 * within our recovery window, then it was a mistake to do the
7806 * retransmit in the first place. Recover our original cwnd and
7807 * ssthresh, and proceed to transmit where we left off.
7808 */
7809 if (tp->t_flags & TF_PREVVALID) {
7810 tp->t_flags &= ~TF_PREVVALID;
7811 if (tp->t_rxtshift == 1 &&
7812 (int)(ticks - tp->t_badrxtwin) < 0)
7813 bbr_cong_signal(tp, th, CC_RTO_ERR, NULL);
7814 }
7815 SOCKBUF_LOCK(&so->so_snd);
7816 acked_amount = min(acked, (int)sbavail(&so->so_snd));
7817 tp->snd_wnd -= acked_amount;
7818 mfree = sbcut_locked(&so->so_snd, acked_amount);
7819 /* NB: sowwakeup_locked() does an implicit unlock. */
7820 sowwakeup_locked(so);
7821 m_freem(mfree);
7822 if (SEQ_GT(th->th_ack, tp->snd_una)) {
7823 bbr_collapse_rtt(tp, bbr, TCP_REXMTVAL(tp));
7824 }
7825 tp->snd_una = th->th_ack;
7826 bbr_ack_received(tp, bbr, th, acked, sack_changed, prev_acked, __LINE__, (bbr->r_ctl.rc_lost - lost));
7827 if (IN_RECOVERY(tp->t_flags)) {
7828 if (SEQ_LT(th->th_ack, tp->snd_recover) &&
7829 (SEQ_LT(th->th_ack, tp->snd_max))) {
7830 tcp_bbr_partialack(tp);
7831 } else {
7832 bbr_post_recovery(tp);
7833 }
7834 }
7835 if (SEQ_GT(tp->snd_una, tp->snd_recover)) {
7836 tp->snd_recover = tp->snd_una;
7837 }
7838 if (SEQ_LT(tp->snd_nxt, tp->snd_max)) {
7839 tp->snd_nxt = tp->snd_max;
7840 }
7841 if (tp->snd_una == tp->snd_max) {
7842 /* Nothing left outstanding */
7843nothing_left:
7844 bbr_log_progress_event(bbr, tp, ticks, PROGRESS_CLEAR, __LINE__);
7845 if (sbavail(&tp->t_inpcb->inp_socket->so_snd) == 0)
7846 bbr->rc_tp->t_acktime = 0;
7847 if ((sbused(&so->so_snd) == 0) &&
7848 (tp->t_flags & TF_SENTFIN)) {
7849 ourfinisacked = 1;
7850 }
7851 bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime);
7852 if (bbr->rc_in_persist == 0) {
7853 bbr->r_ctl.rc_went_idle_time = bbr->r_ctl.rc_rcvtime;
7854 }
7855 sack_filter_clear(&bbr->r_ctl.bbr_sf, tp->snd_una);
7856 bbr_log_ack_clear(bbr, bbr->r_ctl.rc_rcvtime);
7857 /*
7858 * We invalidate the last ack here since we
7859 * don't want to transfer forward the time
7860 * for our sum's calculations.
7861 */
7862 if ((tp->t_state >= TCPS_FIN_WAIT_1) &&
7863 (sbavail(&so->so_snd) == 0) &&
7864 (tp->t_flags2 & TF2_DROP_AF_DATA)) {
7865 /*
7866 * The socket was gone and the peer sent data, time
7867 * to reset him.
7868 */
7869 *ret_val = 1;
7870 tcp_log_end_status(tp, TCP_EI_STATUS_DATA_A_CLOSE);
7871 /* tcp_close will kill the inp pre-log the Reset */
7872 tcp_log_end_status(tp, TCP_EI_STATUS_SERVER_RST);
7873 tp = tcp_close(tp);
7874 ctf_do_dropwithreset(m, tp, th, BANDLIM_UNLIMITED, tlen);
7875 BBR_STAT_INC(bbr_dropped_af_data);
7876 return (1);
7877 }
7878 /* Set need output so persist might get set */
7879 bbr->r_wanted_output = 1;
7880 }
7881 if (ofia)
7882 *ofia = ourfinisacked;
7883 return (0);
7884}
7885
7886static void
7887bbr_enter_persist(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts, int32_t line)
7888{
7889 if (bbr->rc_in_persist == 0) {
7890 bbr_timer_cancel(bbr, __LINE__, cts);
7891 bbr->r_ctl.rc_last_delay_val = 0;
7892 tp->t_rxtshift = 0;
7893 bbr->rc_in_persist = 1;
7894 bbr->r_ctl.rc_went_idle_time = cts;
7895 /* We should be capped when rw went to 0 but just in case */
7896 bbr_log_type_pesist(bbr, cts, 0, line, 1);
7897 /* Time freezes for the state, so do the accounting now */
7898 if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) {
7899 uint32_t time_in;
7900
7901 time_in = cts - bbr->r_ctl.rc_bbr_state_time;
7902 if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) {
7903 int32_t idx;
7904
7905 idx = bbr_state_val(bbr);
7906 counter_u64_add(bbr_state_time[(idx + 5)], time_in);
7907 } else {
7908 counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in);
7909 }
7910 }
7911 bbr->r_ctl.rc_bbr_state_time = cts;
7912 }
7913}
7914
7915static void
7916bbr_restart_after_idle(struct tcp_bbr *bbr, uint32_t cts, uint32_t idle_time)
7917{
7918 /*
7919 * Note that if idle time does not exceed our
7920 * threshold, we do nothing continuing the state
7921 * transitions we were last walking through.
7922 */
7923 if (idle_time >= bbr_idle_restart_threshold) {
7924 if (bbr->rc_use_idle_restart) {
7925 bbr->rc_bbr_state = BBR_STATE_IDLE_EXIT;
7926 /*
7927 * Set our target using BBR_UNIT, so
7928 * we increase at a dramatic rate but
7929 * we stop when we get the pipe
7930 * full again for our current b/w estimate.
7931 */
7932 bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT;
7933 bbr->r_ctl.rc_bbr_cwnd_gain = BBR_UNIT;
7934 bbr_set_state_target(bbr, __LINE__);
7935 /* Now setup our gains to ramp up */
7936 bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.rc_startup_pg;
7937 bbr->r_ctl.rc_bbr_cwnd_gain = bbr->r_ctl.rc_startup_pg;
7938 bbr_log_type_statechange(bbr, cts, __LINE__);
7939 } else if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) {
7940 bbr_substate_change(bbr, cts, __LINE__, 1);
7941 }
7942 }
7943}
7944
7945static void
7946bbr_exit_persist(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts, int32_t line)
7947{
7948 uint32_t idle_time;
7949
7950 if (bbr->rc_in_persist == 0)
7951 return;
7952 idle_time = bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time);
7953 bbr->rc_in_persist = 0;
7954 bbr->rc_hit_state_1 = 0;
7955 bbr->r_ctl.rc_del_time = cts;
7956 /*
7957 * We invalidate the last ack here since we
7958 * don't want to transfer forward the time
7959 * for our sum's calculations.
7960 */
7961 if (tcp_in_hpts(bbr->rc_inp)) {
7962 tcp_hpts_remove(bbr->rc_inp);
7963 bbr->rc_timer_first = 0;
7964 bbr->r_ctl.rc_hpts_flags = 0;
7965 bbr->r_ctl.rc_last_delay_val = 0;
7966 bbr->r_ctl.rc_hptsi_agg_delay = 0;
7967 bbr->r_agg_early_set = 0;
7968 bbr->r_ctl.rc_agg_early = 0;
7969 }
7970 bbr_log_type_pesist(bbr, cts, idle_time, line, 0);
7971 if (idle_time >= bbr_rtt_probe_time) {
7972 /*
7973 * This qualifies as a RTT_PROBE session since we drop the
7974 * data outstanding to nothing and waited more than
7975 * bbr_rtt_probe_time.
7976 */
7977 bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_PERSIST, 0);
7978 bbr->r_ctl.last_in_probertt = bbr->r_ctl.rc_rtt_shrinks = cts;
7979 }
7980 tp->t_rxtshift = 0;
7981 /*
7982 * If in probeBW and we have persisted more than an RTT lets do
7983 * special handling.
7984 */
7985 /* Force a time based epoch */
7986 bbr_set_epoch(bbr, cts, __LINE__);
7987 /*
7988 * Setup the lost so we don't count anything against the guy
7989 * we have been stuck with during persists.
7990 */
7991 bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost;
7992 /* Time un-freezes for the state */
7993 bbr->r_ctl.rc_bbr_state_time = cts;
7994 if ((bbr->rc_bbr_state == BBR_STATE_PROBE_BW) ||
7995 (bbr->rc_bbr_state == BBR_STATE_PROBE_RTT)) {
7996 /*
7997 * If we are going back to probe-bw
7998 * or probe_rtt, we may need to possibly
7999 * do a fast restart.
8000 */
8001 bbr_restart_after_idle(bbr, cts, idle_time);
8002 }
8003}
8004
8005static void
8006bbr_collapsed_window(struct tcp_bbr *bbr)
8007{
8008 /*
8009 * Now we must walk the
8010 * send map and divide the
8011 * ones left stranded. These
8012 * guys can't cause us to abort
8013 * the connection and are really
8014 * "unsent". However if a buggy
8015 * client actually did keep some
8016 * of the data i.e. collapsed the win
8017 * and refused to ack and then opened
8018 * the win and acked that data. We would
8019 * get into an ack war, the simplier
8020 * method then of just pretending we
8021 * did not send those segments something
8022 * won't work.
8023 */
8024 struct bbr_sendmap *rsm, *nrsm;
8025 tcp_seq max_seq;
8026 uint32_t maxseg;
8027 int can_split = 0;
8028 int fnd = 0;
8029
8030 maxseg = bbr->rc_tp->t_maxseg - bbr->rc_last_options;
8031 max_seq = bbr->rc_tp->snd_una + bbr->rc_tp->snd_wnd;
8032 bbr_log_type_rwnd_collapse(bbr, max_seq, 1, 0);
8033 TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) {
8034 /* Find the first seq past or at maxseq */
8035 if (rsm->r_flags & BBR_RWND_COLLAPSED)
8036 rsm->r_flags &= ~BBR_RWND_COLLAPSED;
8037 if (SEQ_GEQ(max_seq, rsm->r_start) &&
8038 SEQ_GEQ(rsm->r_end, max_seq)) {
8039 fnd = 1;
8040 break;
8041 }
8042 }
8043 bbr->rc_has_collapsed = 0;
8044 if (!fnd) {
8045 /* Nothing to do strange */
8046 return;
8047 }
8048 /*
8049 * Now can we split?
8050 *
8051 * We don't want to split if splitting
8052 * would generate too many small segments
8053 * less we let an attacker fragment our
8054 * send_map and leave us out of memory.
8055 */
8056 if ((max_seq != rsm->r_start) &&
8057 (max_seq != rsm->r_end)){
8058 /* can we split? */
8059 int res1, res2;
8060
8061 res1 = max_seq - rsm->r_start;
8062 res2 = rsm->r_end - max_seq;
8063 if ((res1 >= (maxseg/8)) &&
8064 (res2 >= (maxseg/8))) {
8065 /* No small pieces here */
8066 can_split = 1;
8067 } else if (bbr->r_ctl.rc_num_small_maps_alloced < bbr_sack_block_limit) {
8068 /* We are under the limit */
8069 can_split = 1;
8070 }
8071 }
8072 /* Ok do we need to split this rsm? */
8073 if (max_seq == rsm->r_start) {
8074 /* It's this guy no split required */
8075 nrsm = rsm;
8076 } else if (max_seq == rsm->r_end) {
8077 /* It's the next one no split required. */
8078 nrsm = TAILQ_NEXT(rsm, r_next);
8079 if (nrsm == NULL) {
8080 /* Huh? */
8081 return;
8082 }
8083 } else if (can_split && SEQ_LT(max_seq, rsm->r_end)) {
8084 /* yep we need to split it */
8085 nrsm = bbr_alloc_limit(bbr, BBR_LIMIT_TYPE_SPLIT);
8086 if (nrsm == NULL) {
8087 /* failed XXXrrs what can we do mark the whole? */
8088 nrsm = rsm;
8089 goto no_split;
8090 }
8091 /* Clone it */
8092 bbr_log_type_rwnd_collapse(bbr, max_seq, 3, 0);
8093 bbr_clone_rsm(bbr, nrsm, rsm, max_seq);
8094 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next);
8095 if (rsm->r_in_tmap) {
8096 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext);
8097 nrsm->r_in_tmap = 1;
8098 }
8099 } else {
8100 /*
8101 * Split not allowed just start here just
8102 * use this guy.
8103 */
8104 nrsm = rsm;
8105 }
8106no_split:
8107 BBR_STAT_INC(bbr_collapsed_win);
8108 /* reuse fnd as a count */
8109 fnd = 0;
8110 TAILQ_FOREACH_FROM(nrsm, &bbr->r_ctl.rc_map, r_next) {
8111 nrsm->r_flags |= BBR_RWND_COLLAPSED;
8112 fnd++;
8113 bbr->rc_has_collapsed = 1;
8114 }
8115 bbr_log_type_rwnd_collapse(bbr, max_seq, 4, fnd);
8116}
8117
8118static void
8119bbr_un_collapse_window(struct tcp_bbr *bbr)
8120{
8121 struct bbr_sendmap *rsm;
8122 int cleared = 0;
8123
8124 TAILQ_FOREACH_REVERSE(rsm, &bbr->r_ctl.rc_map, bbr_head, r_next) {
8125 if (rsm->r_flags & BBR_RWND_COLLAPSED) {
8126 /* Clear the flag */
8127 rsm->r_flags &= ~BBR_RWND_COLLAPSED;
8128 cleared++;
8129 } else
8130 break;
8131 }
8132 bbr_log_type_rwnd_collapse(bbr,
8133 (bbr->rc_tp->snd_una + bbr->rc_tp->snd_wnd), 0, cleared);
8134 bbr->rc_has_collapsed = 0;
8135}
8136
8137/*
8138 * Return value of 1, the TCB is unlocked and most
8139 * likely gone, return value of 0, the TCB is still
8140 * locked.
8141 */
8142static int
8143bbr_process_data(struct mbuf *m, struct tcphdr *th, struct socket *so,
8144 struct tcpcb *tp, int32_t drop_hdrlen, int32_t tlen,
8145 uint32_t tiwin, int32_t thflags, int32_t nxt_pkt)
8146{
8147 /*
8148 * Update window information. Don't look at window if no ACK: TAC's
8149 * send garbage on first SYN.
8150 */
8151 uint16_t nsegs;
8152 int32_t tfo_syn;
8153 struct tcp_bbr *bbr;
8154
8155 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
8156 INP_WLOCK_ASSERT(tp->t_inpcb);
8157 nsegs = max(1, m->m_pkthdr.lro_nsegs);
8158 if ((thflags & TH_ACK) &&
8159 (SEQ_LT(tp->snd_wl1, th->th_seq) ||
8160 (tp->snd_wl1 == th->th_seq && (SEQ_LT(tp->snd_wl2, th->th_ack) ||
8161 (tp->snd_wl2 == th->th_ack && tiwin > tp->snd_wnd))))) {
8162 /* keep track of pure window updates */
8163 if (tlen == 0 &&
8164 tp->snd_wl2 == th->th_ack && tiwin > tp->snd_wnd)
8165 KMOD_TCPSTAT_INC(tcps_rcvwinupd);
8166 tp->snd_wnd = tiwin;
8167 tp->snd_wl1 = th->th_seq;
8168 tp->snd_wl2 = th->th_ack;
8169 if (tp->snd_wnd > tp->max_sndwnd)
8170 tp->max_sndwnd = tp->snd_wnd;
8171 bbr->r_wanted_output = 1;
8172 } else if (thflags & TH_ACK) {
8173 if ((tp->snd_wl2 == th->th_ack) && (tiwin < tp->snd_wnd)) {
8174 tp->snd_wnd = tiwin;
8175 tp->snd_wl1 = th->th_seq;
8176 tp->snd_wl2 = th->th_ack;
8177 }
8178 }
8179 if (tp->snd_wnd < ctf_outstanding(tp))
8180 /* The peer collapsed its window on us */
8181 bbr_collapsed_window(bbr);
8182 else if (bbr->rc_has_collapsed)
8183 bbr_un_collapse_window(bbr);
8184 /* Was persist timer active and now we have window space? */
8185 if ((bbr->rc_in_persist != 0) &&
8186 (tp->snd_wnd >= min((bbr->r_ctl.rc_high_rwnd/2),
8187 bbr_minseg(bbr)))) {
8188 /*
8189 * Make the rate persist at end of persist mode if idle long
8190 * enough
8191 */
8192 bbr_exit_persist(tp, bbr, bbr->r_ctl.rc_rcvtime, __LINE__);
8193
8194 /* Make sure we output to start the timer */
8195 bbr->r_wanted_output = 1;
8196 }
8197 /* Do we need to enter persist? */
8198 if ((bbr->rc_in_persist == 0) &&
8199 (tp->snd_wnd < min((bbr->r_ctl.rc_high_rwnd/2), bbr_minseg(bbr))) &&
8200 TCPS_HAVEESTABLISHED(tp->t_state) &&
8201 (tp->snd_max == tp->snd_una) &&
8202 sbavail(&tp->t_inpcb->inp_socket->so_snd) &&
8203 (sbavail(&tp->t_inpcb->inp_socket->so_snd) > tp->snd_wnd)) {
8204 /* No send window.. we must enter persist */
8205 bbr_enter_persist(tp, bbr, bbr->r_ctl.rc_rcvtime, __LINE__);
8206 }
8207 if (tp->t_flags2 & TF2_DROP_AF_DATA) {
8208 m_freem(m);
8209 return (0);
8210 }
8211 /*
8212 * We don't support urgent data but
8213 * drag along the up just to make sure
8214 * if there is a stack switch no one
8215 * is surprised.
8216 */
8217 tp->rcv_up = tp->rcv_nxt;
8218 INP_WLOCK_ASSERT(tp->t_inpcb);
8219
8220 /*
8221 * Process the segment text, merging it into the TCP sequencing
8222 * queue, and arranging for acknowledgment of receipt if necessary.
8223 * This process logically involves adjusting tp->rcv_wnd as data is
8224 * presented to the user (this happens in tcp_usrreq.c, case
8225 * PRU_RCVD). If a FIN has already been received on this connection
8226 * then we just ignore the text.
8227 */
8228 tfo_syn = ((tp->t_state == TCPS_SYN_RECEIVED) &&
8229 IS_FASTOPEN(tp->t_flags));
8230 if ((tlen || (thflags & TH_FIN) || (tfo_syn && tlen > 0)) &&
8231 TCPS_HAVERCVDFIN(tp->t_state) == 0) {
8232 tcp_seq save_start = th->th_seq;
8233 tcp_seq save_rnxt = tp->rcv_nxt;
8234 int save_tlen = tlen;
8235
8236 m_adj(m, drop_hdrlen); /* delayed header drop */
8237 /*
8238 * Insert segment which includes th into TCP reassembly
8239 * queue with control block tp. Set thflags to whether
8240 * reassembly now includes a segment with FIN. This handles
8241 * the common case inline (segment is the next to be
8242 * received on an established connection, and the queue is
8243 * empty), avoiding linkage into and removal from the queue
8244 * and repetition of various conversions. Set DELACK for
8245 * segments received in order, but ack immediately when
8246 * segments are out of order (so fast retransmit can work).
8247 */
8248 if (th->th_seq == tp->rcv_nxt &&
8249 SEGQ_EMPTY(tp) &&
8250 (TCPS_HAVEESTABLISHED(tp->t_state) ||
8251 tfo_syn)) {
8252#ifdef NETFLIX_SB_LIMITS
8253 u_int mcnt, appended;
8254
8255 if (so->so_rcv.sb_shlim) {
8256 mcnt = m_memcnt(m);
8257 appended = 0;
8258 if (counter_fo_get(so->so_rcv.sb_shlim, mcnt,
8259 CFO_NOSLEEP, NULL) == false) {
8260 counter_u64_add(tcp_sb_shlim_fails, 1);
8261 m_freem(m);
8262 return (0);
8263 }
8264 }
8265
8266#endif
8267 if (DELAY_ACK(tp, bbr, nsegs) || tfo_syn) {
8268 bbr->bbr_segs_rcvd += max(1, nsegs);
8269 tp->t_flags |= TF_DELACK;
8270 bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime);
8271 } else {
8272 bbr->r_wanted_output = 1;
8273 tp->t_flags |= TF_ACKNOW;
8274 }
8275 tp->rcv_nxt += tlen;
8276 if (tlen &&
8277 ((tp->t_flags2 & TF2_FBYTES_COMPLETE) == 0) &&
8278 (tp->t_fbyte_in == 0)) {
8279 tp->t_fbyte_in = ticks;
8280 if (tp->t_fbyte_in == 0)
8281 tp->t_fbyte_in = 1;
8282 if (tp->t_fbyte_out && tp->t_fbyte_in)
8283 tp->t_flags2 |= TF2_FBYTES_COMPLETE;
8284 }
8285 thflags = tcp_get_flags(th) & TH_FIN;
8286 KMOD_TCPSTAT_ADD(tcps_rcvpack, (int)nsegs);
8287 KMOD_TCPSTAT_ADD(tcps_rcvbyte, tlen);
8288 SOCKBUF_LOCK(&so->so_rcv);
8289 if (so->so_rcv.sb_state & SBS_CANTRCVMORE)
8290 m_freem(m);
8291 else
8292#ifdef NETFLIX_SB_LIMITS
8293 appended =
8294#endif
8295 sbappendstream_locked(&so->so_rcv, m, 0);
8296 /* NB: sorwakeup_locked() does an implicit unlock. */
8297 sorwakeup_locked(so);
8298#ifdef NETFLIX_SB_LIMITS
8299 if (so->so_rcv.sb_shlim && appended != mcnt)
8300 counter_fo_release(so->so_rcv.sb_shlim,
8301 mcnt - appended);
8302#endif
8303
8304 } else {
8305 /*
8306 * XXX: Due to the header drop above "th" is
8307 * theoretically invalid by now. Fortunately
8308 * m_adj() doesn't actually frees any mbufs when
8309 * trimming from the head.
8310 */
8311 tcp_seq temp = save_start;
8312
8313 thflags = tcp_reass(tp, th, &temp, &tlen, m);
8314 tp->t_flags |= TF_ACKNOW;
8315 if (tp->t_flags & TF_WAKESOR) {
8316 tp->t_flags &= ~TF_WAKESOR;
8317 /* NB: sorwakeup_locked() does an implicit unlock. */
8318 sorwakeup_locked(so);
8319 }
8320 }
8321 if ((tp->t_flags & TF_SACK_PERMIT) &&
8322 (save_tlen > 0) &&
8323 TCPS_HAVEESTABLISHED(tp->t_state)) {
8324 if ((tlen == 0) && (SEQ_LT(save_start, save_rnxt))) {
8325 /*
8326 * DSACK actually handled in the fastpath
8327 * above.
8328 */
8329 tcp_update_sack_list(tp, save_start,
8330 save_start + save_tlen);
8331 } else if ((tlen > 0) && SEQ_GT(tp->rcv_nxt, save_rnxt)) {
8332 if ((tp->rcv_numsacks >= 1) &&
8333 (tp->sackblks[0].end == save_start)) {
8334 /*
8335 * Partial overlap, recorded at todrop
8336 * above.
8337 */
8338 tcp_update_sack_list(tp,
8339 tp->sackblks[0].start,
8340 tp->sackblks[0].end);
8341 } else {
8342 tcp_update_dsack_list(tp, save_start,
8343 save_start + save_tlen);
8344 }
8345 } else if (tlen >= save_tlen) {
8346 /* Update of sackblks. */
8347 tcp_update_dsack_list(tp, save_start,
8348 save_start + save_tlen);
8349 } else if (tlen > 0) {
8350 tcp_update_dsack_list(tp, save_start,
8351 save_start + tlen);
8352 }
8353 }
8354 } else {
8355 m_freem(m);
8356 thflags &= ~TH_FIN;
8357 }
8358
8359 /*
8360 * If FIN is received ACK the FIN and let the user know that the
8361 * connection is closing.
8362 */
8363 if (thflags & TH_FIN) {
8364 if (TCPS_HAVERCVDFIN(tp->t_state) == 0) {
8365 /* The socket upcall is handled by socantrcvmore. */
8366 socantrcvmore(so);
8367 /*
8368 * If connection is half-synchronized (ie NEEDSYN
8369 * flag on) then delay ACK, so it may be piggybacked
8370 * when SYN is sent. Otherwise, since we received a
8371 * FIN then no more input can be expected, send ACK
8372 * now.
8373 */
8374 if (tp->t_flags & TF_NEEDSYN) {
8375 tp->t_flags |= TF_DELACK;
8376 bbr_timer_cancel(bbr,
8377 __LINE__, bbr->r_ctl.rc_rcvtime);
8378 } else {
8379 tp->t_flags |= TF_ACKNOW;
8380 }
8381 tp->rcv_nxt++;
8382 }
8383 switch (tp->t_state) {
8384 /*
8385 * In SYN_RECEIVED and ESTABLISHED STATES enter the
8386 * CLOSE_WAIT state.
8387 */
8388 case TCPS_SYN_RECEIVED:
8389 tp->t_starttime = ticks;
8390 /* FALLTHROUGH */
8391 case TCPS_ESTABLISHED:
8392 tcp_state_change(tp, TCPS_CLOSE_WAIT);
8393 break;
8394
8395 /*
8396 * If still in FIN_WAIT_1 STATE FIN has not been
8397 * acked so enter the CLOSING state.
8398 */
8399 case TCPS_FIN_WAIT_1:
8400 tcp_state_change(tp, TCPS_CLOSING);
8401 break;
8402
8403 /*
8404 * In FIN_WAIT_2 state enter the TIME_WAIT state,
8405 * starting the time-wait timer, turning off the
8406 * other standard timers.
8407 */
8408 case TCPS_FIN_WAIT_2:
8409 bbr->rc_timer_first = 1;
8410 bbr_timer_cancel(bbr,
8411 __LINE__, bbr->r_ctl.rc_rcvtime);
8412 INP_WLOCK_ASSERT(tp->t_inpcb);
8413 tcp_twstart(tp);
8414 return (1);
8415 }
8416 }
8417 /*
8418 * Return any desired output.
8419 */
8420 if ((tp->t_flags & TF_ACKNOW) ||
8421 (sbavail(&so->so_snd) > ctf_outstanding(tp))) {
8422 bbr->r_wanted_output = 1;
8423 }
8424 INP_WLOCK_ASSERT(tp->t_inpcb);
8425 return (0);
8426}
8427
8428/*
8429 * Here nothing is really faster, its just that we
8430 * have broken out the fast-data path also just like
8431 * the fast-ack. Return 1 if we processed the packet
8432 * return 0 if you need to take the "slow-path".
8433 */
8434static int
8435bbr_do_fastnewdata(struct mbuf *m, struct tcphdr *th, struct socket *so,
8436 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
8437 uint32_t tiwin, int32_t nxt_pkt)
8438{
8439 uint16_t nsegs;
8440 int32_t newsize = 0; /* automatic sockbuf scaling */
8441 struct tcp_bbr *bbr;
8442#ifdef NETFLIX_SB_LIMITS
8443 u_int mcnt, appended;
8444#endif
8445#ifdef TCPDEBUG
8446 /*
8447 * The size of tcp_saveipgen must be the size of the max ip header,
8448 * now IPv6.
8449 */
8450 u_char tcp_saveipgen[IP6_HDR_LEN];
8451 struct tcphdr tcp_savetcp;
8452 short ostate = 0;
8453
8454#endif
8455 /* On the hpts and we would have called output */
8456 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
8457
8458 /*
8459 * If last ACK falls within this segment's sequence numbers, record
8460 * the timestamp. NOTE that the test is modified according to the
8461 * latest proposal of the tcplw@cray.com list (Braden 1993/04/26).
8462 */
8463 if (bbr->r_ctl.rc_resend != NULL) {
8464 return (0);
8465 }
8466 if (tiwin && tiwin != tp->snd_wnd) {
8467 return (0);
8468 }
8469 if (__predict_false((tp->t_flags & (TF_NEEDSYN | TF_NEEDFIN)))) {
8470 return (0);
8471 }
8472 if (__predict_false((to->to_flags & TOF_TS) &&
8473 (TSTMP_LT(to->to_tsval, tp->ts_recent)))) {
8474 return (0);
8475 }
8476 if (__predict_false((th->th_ack != tp->snd_una))) {
8477 return (0);
8478 }
8479 if (__predict_false(tlen > sbspace(&so->so_rcv))) {
8480 return (0);
8481 }
8482 if ((to->to_flags & TOF_TS) != 0 &&
8483 SEQ_LEQ(th->th_seq, tp->last_ack_sent)) {
8484 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv);
8485 tp->ts_recent = to->to_tsval;
8486 }
8487 /*
8488 * This is a pure, in-sequence data packet with nothing on the
8489 * reassembly queue and we have enough buffer space to take it.
8490 */
8491 nsegs = max(1, m->m_pkthdr.lro_nsegs);
8492
8493#ifdef NETFLIX_SB_LIMITS
8494 if (so->so_rcv.sb_shlim) {
8495 mcnt = m_memcnt(m);
8496 appended = 0;
8497 if (counter_fo_get(so->so_rcv.sb_shlim, mcnt,
8498 CFO_NOSLEEP, NULL) == false) {
8499 counter_u64_add(tcp_sb_shlim_fails, 1);
8500 m_freem(m);
8501 return (1);
8502 }
8503 }
8504#endif
8505 /* Clean receiver SACK report if present */
8506 if (tp->rcv_numsacks)
8507 tcp_clean_sackreport(tp);
8508 KMOD_TCPSTAT_INC(tcps_preddat);
8509 tp->rcv_nxt += tlen;
8510 if (tlen &&
8511 ((tp->t_flags2 & TF2_FBYTES_COMPLETE) == 0) &&
8512 (tp->t_fbyte_in == 0)) {
8513 tp->t_fbyte_in = ticks;
8514 if (tp->t_fbyte_in == 0)
8515 tp->t_fbyte_in = 1;
8516 if (tp->t_fbyte_out && tp->t_fbyte_in)
8517 tp->t_flags2 |= TF2_FBYTES_COMPLETE;
8518 }
8519 /*
8520 * Pull snd_wl1 up to prevent seq wrap relative to th_seq.
8521 */
8522 tp->snd_wl1 = th->th_seq;
8523 /*
8524 * Pull rcv_up up to prevent seq wrap relative to rcv_nxt.
8525 */
8526 tp->rcv_up = tp->rcv_nxt;
8527 KMOD_TCPSTAT_ADD(tcps_rcvpack, (int)nsegs);
8528 KMOD_TCPSTAT_ADD(tcps_rcvbyte, tlen);
8529#ifdef TCPDEBUG
8530 if (so->so_options & SO_DEBUG)
8531 tcp_trace(TA_INPUT, ostate, tp,
8532 (void *)tcp_saveipgen, &tcp_savetcp, 0);
8533#endif
8534 newsize = tcp_autorcvbuf(m, th, so, tp, tlen);
8535
8536 /* Add data to socket buffer. */
8537 SOCKBUF_LOCK(&so->so_rcv);
8538 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
8539 m_freem(m);
8540 } else {
8541 /*
8542 * Set new socket buffer size. Give up when limit is
8543 * reached.
8544 */
8545 if (newsize)
8546 if (!sbreserve_locked(&so->so_rcv,
8547 newsize, so, NULL))
8548 so->so_rcv.sb_flags &= ~SB_AUTOSIZE;
8549 m_adj(m, drop_hdrlen); /* delayed header drop */
8550
8551#ifdef NETFLIX_SB_LIMITS
8552 appended =
8553#endif
8554 sbappendstream_locked(&so->so_rcv, m, 0);
8555 ctf_calc_rwin(so, tp);
8556 }
8557 /* NB: sorwakeup_locked() does an implicit unlock. */
8558 sorwakeup_locked(so);
8559#ifdef NETFLIX_SB_LIMITS
8560 if (so->so_rcv.sb_shlim && mcnt != appended)
8561 counter_fo_release(so->so_rcv.sb_shlim, mcnt - appended);
8562#endif
8563 if (DELAY_ACK(tp, bbr, nsegs)) {
8564 bbr->bbr_segs_rcvd += max(1, nsegs);
8565 tp->t_flags |= TF_DELACK;
8566 bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime);
8567 } else {
8568 bbr->r_wanted_output = 1;
8569 tp->t_flags |= TF_ACKNOW;
8570 }
8571 return (1);
8572}
8573
8574/*
8575 * This subfunction is used to try to highly optimize the
8576 * fast path. We again allow window updates that are
8577 * in sequence to remain in the fast-path. We also add
8578 * in the __predict's to attempt to help the compiler.
8579 * Note that if we return a 0, then we can *not* process
8580 * it and the caller should push the packet into the
8581 * slow-path. If we return 1, then all is well and
8582 * the packet is fully processed.
8583 */
8584static int
8585bbr_fastack(struct mbuf *m, struct tcphdr *th, struct socket *so,
8586 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
8587 uint32_t tiwin, int32_t nxt_pkt, uint8_t iptos)
8588{
8589 int32_t acked;
8590 uint16_t nsegs;
8591 uint32_t sack_changed;
8592#ifdef TCPDEBUG
8593 /*
8594 * The size of tcp_saveipgen must be the size of the max ip header,
8595 * now IPv6.
8596 */
8597 u_char tcp_saveipgen[IP6_HDR_LEN];
8598 struct tcphdr tcp_savetcp;
8599 short ostate = 0;
8600
8601#endif
8602 uint32_t prev_acked = 0;
8603 struct tcp_bbr *bbr;
8604
8605 if (__predict_false(SEQ_LEQ(th->th_ack, tp->snd_una))) {
8606 /* Old ack, behind (or duplicate to) the last one rcv'd */
8607 return (0);
8608 }
8609 if (__predict_false(SEQ_GT(th->th_ack, tp->snd_max))) {
8610 /* Above what we have sent? */
8611 return (0);
8612 }
8613 if (__predict_false(tiwin == 0)) {
8614 /* zero window */
8615 return (0);
8616 }
8617 if (__predict_false(tp->t_flags & (TF_NEEDSYN | TF_NEEDFIN))) {
8618 /* We need a SYN or a FIN, unlikely.. */
8619 return (0);
8620 }
8621 if ((to->to_flags & TOF_TS) && __predict_false(TSTMP_LT(to->to_tsval, tp->ts_recent))) {
8622 /* Timestamp is behind .. old ack with seq wrap? */
8623 return (0);
8624 }
8625 if (__predict_false(IN_RECOVERY(tp->t_flags))) {
8626 /* Still recovering */
8627 return (0);
8628 }
8629 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
8630 if (__predict_false(bbr->r_ctl.rc_resend != NULL)) {
8631 /* We are retransmitting */
8632 return (0);
8633 }
8634 if (__predict_false(bbr->rc_in_persist != 0)) {
8635 /* In persist mode */
8636 return (0);
8637 }
8638 if (bbr->r_ctl.rc_sacked) {
8639 /* We have sack holes on our scoreboard */
8640 return (0);
8641 }
8642 /* Ok if we reach here, we can process a fast-ack */
8643 nsegs = max(1, m->m_pkthdr.lro_nsegs);
8644 sack_changed = bbr_log_ack(tp, to, th, &prev_acked);
8645 /*
8646 * We never detect loss in fast ack [we can't
8647 * have a sack and can't be in recovery so
8648 * we always pass 0 (nothing detected)].
8649 */
8650 bbr_lt_bw_sampling(bbr, bbr->r_ctl.rc_rcvtime, 0);
8651 /* Did the window get updated? */
8652 if (tiwin != tp->snd_wnd) {
8653 tp->snd_wnd = tiwin;
8654 tp->snd_wl1 = th->th_seq;
8655 if (tp->snd_wnd > tp->max_sndwnd)
8656 tp->max_sndwnd = tp->snd_wnd;
8657 }
8658 /* Do we need to exit persists? */
8659 if ((bbr->rc_in_persist != 0) &&
8660 (tp->snd_wnd >= min((bbr->r_ctl.rc_high_rwnd/2),
8661 bbr_minseg(bbr)))) {
8662 bbr_exit_persist(tp, bbr, bbr->r_ctl.rc_rcvtime, __LINE__);
8663 bbr->r_wanted_output = 1;
8664 }
8665 /* Do we need to enter persists? */
8666 if ((bbr->rc_in_persist == 0) &&
8667 (tp->snd_wnd < min((bbr->r_ctl.rc_high_rwnd/2), bbr_minseg(bbr))) &&
8668 TCPS_HAVEESTABLISHED(tp->t_state) &&
8669 (tp->snd_max == tp->snd_una) &&
8670 sbavail(&tp->t_inpcb->inp_socket->so_snd) &&
8671 (sbavail(&tp->t_inpcb->inp_socket->so_snd) > tp->snd_wnd)) {
8672 /* No send window.. we must enter persist */
8673 bbr_enter_persist(tp, bbr, bbr->r_ctl.rc_rcvtime, __LINE__);
8674 }
8675 /*
8676 * If last ACK falls within this segment's sequence numbers, record
8677 * the timestamp. NOTE that the test is modified according to the
8678 * latest proposal of the tcplw@cray.com list (Braden 1993/04/26).
8679 */
8680 if ((to->to_flags & TOF_TS) != 0 &&
8681 SEQ_LEQ(th->th_seq, tp->last_ack_sent)) {
8682 tp->ts_recent_age = bbr->r_ctl.rc_rcvtime;
8683 tp->ts_recent = to->to_tsval;
8684 }
8685 /*
8686 * This is a pure ack for outstanding data.
8687 */
8688 KMOD_TCPSTAT_INC(tcps_predack);
8689
8690 /*
8691 * "bad retransmit" recovery.
8692 */
8693 if (tp->t_flags & TF_PREVVALID) {
8694 tp->t_flags &= ~TF_PREVVALID;
8695 if (tp->t_rxtshift == 1 &&
8696 (int)(ticks - tp->t_badrxtwin) < 0)
8697 bbr_cong_signal(tp, th, CC_RTO_ERR, NULL);
8698 }
8699 /*
8700 * Recalculate the transmit timer / rtt.
8701 *
8702 * Some boxes send broken timestamp replies during the SYN+ACK
8703 * phase, ignore timestamps of 0 or we could calculate a huge RTT
8704 * and blow up the retransmit timer.
8705 */
8706 acked = BYTES_THIS_ACK(tp, th);
8707
8708#ifdef TCP_HHOOK
8709 /* Run HHOOK_TCP_ESTABLISHED_IN helper hooks. */
8710 hhook_run_tcp_est_in(tp, th, to);
8711#endif
8712
8713 KMOD_TCPSTAT_ADD(tcps_rcvackpack, (int)nsegs);
8714 KMOD_TCPSTAT_ADD(tcps_rcvackbyte, acked);
8715 sbdrop(&so->so_snd, acked);
8716
8717 if (SEQ_GT(th->th_ack, tp->snd_una))
8718 bbr_collapse_rtt(tp, bbr, TCP_REXMTVAL(tp));
8719 tp->snd_una = th->th_ack;
8720 if (tp->snd_wnd < ctf_outstanding(tp))
8721 /* The peer collapsed its window on us */
8722 bbr_collapsed_window(bbr);
8723 else if (bbr->rc_has_collapsed)
8724 bbr_un_collapse_window(bbr);
8725
8726 if (SEQ_GT(tp->snd_una, tp->snd_recover)) {
8727 tp->snd_recover = tp->snd_una;
8728 }
8729 bbr_ack_received(tp, bbr, th, acked, sack_changed, prev_acked, __LINE__, 0);
8730 /*
8731 * Pull snd_wl2 up to prevent seq wrap relative to th_ack.
8732 */
8733 tp->snd_wl2 = th->th_ack;
8734 m_freem(m);
8735 /*
8736 * If all outstanding data are acked, stop retransmit timer,
8737 * otherwise restart timer using current (possibly backed-off)
8738 * value. If process is waiting for space, wakeup/selwakeup/signal.
8739 * If data are ready to send, let tcp_output decide between more
8740 * output or persist.
8741 */
8742#ifdef TCPDEBUG
8743 if (so->so_options & SO_DEBUG)
8744 tcp_trace(TA_INPUT, ostate, tp,
8745 (void *)tcp_saveipgen,
8746 &tcp_savetcp, 0);
8747#endif
8748 /* Wake up the socket if we have room to write more */
8749 sowwakeup(so);
8750 if (tp->snd_una == tp->snd_max) {
8751 /* Nothing left outstanding */
8752 bbr_log_progress_event(bbr, tp, ticks, PROGRESS_CLEAR, __LINE__);
8753 if (sbavail(&tp->t_inpcb->inp_socket->so_snd) == 0)
8754 bbr->rc_tp->t_acktime = 0;
8755 bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime);
8756 if (bbr->rc_in_persist == 0) {
8757 bbr->r_ctl.rc_went_idle_time = bbr->r_ctl.rc_rcvtime;
8758 }
8759 sack_filter_clear(&bbr->r_ctl.bbr_sf, tp->snd_una);
8760 bbr_log_ack_clear(bbr, bbr->r_ctl.rc_rcvtime);
8761 /*
8762 * We invalidate the last ack here since we
8763 * don't want to transfer forward the time
8764 * for our sum's calculations.
8765 */
8766 bbr->r_wanted_output = 1;
8767 }
8768 if (sbavail(&so->so_snd)) {
8769 bbr->r_wanted_output = 1;
8770 }
8771 return (1);
8772}
8773
8774/*
8775 * Return value of 1, the TCB is unlocked and most
8776 * likely gone, return value of 0, the TCB is still
8777 * locked.
8778 */
8779static int
8780bbr_do_syn_sent(struct mbuf *m, struct tcphdr *th, struct socket *so,
8781 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
8782 uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos)
8783{
8784 int32_t todrop;
8785 int32_t ourfinisacked = 0;
8786 struct tcp_bbr *bbr;
8787 int32_t ret_val = 0;
8788
8789 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
8790 ctf_calc_rwin(so, tp);
8791 /*
8792 * If the state is SYN_SENT: if seg contains an ACK, but not for our
8793 * SYN, drop the input. if seg contains a RST, then drop the
8794 * connection. if seg does not contain SYN, then drop it. Otherwise
8795 * this is an acceptable SYN segment initialize tp->rcv_nxt and
8796 * tp->irs if seg contains ack then advance tp->snd_una. BRR does
8797 * not support ECN so we will not say we are capable. if SYN has
8798 * been acked change to ESTABLISHED else SYN_RCVD state arrange for
8799 * segment to be acked (eventually) continue processing rest of
8800 * data/controls, beginning with URG
8801 */
8802 if ((thflags & TH_ACK) &&
8803 (SEQ_LEQ(th->th_ack, tp->iss) ||
8804 SEQ_GT(th->th_ack, tp->snd_max))) {
8805 tcp_log_end_status(tp, TCP_EI_STATUS_RST_IN_FRONT);
8806 ctf_do_dropwithreset(m, tp, th, BANDLIM_RST_OPENPORT, tlen);
8807 return (1);
8808 }
8809 if ((thflags & (TH_ACK | TH_RST)) == (TH_ACK | TH_RST)) {
8810 TCP_PROBE5(connect__refused, NULL, tp,
8811 mtod(m, const char *), tp, th);
8812 tp = tcp_drop(tp, ECONNREFUSED);
8813 ctf_do_drop(m, tp);
8814 return (1);
8815 }
8816 if (thflags & TH_RST) {
8817 ctf_do_drop(m, tp);
8818 return (1);
8819 }
8820 if (!(thflags & TH_SYN)) {
8821 ctf_do_drop(m, tp);
8822 return (1);
8823 }
8824 tp->irs = th->th_seq;
8825 tcp_rcvseqinit(tp);
8826 if (thflags & TH_ACK) {
8827 int tfo_partial = 0;
8828
8829 KMOD_TCPSTAT_INC(tcps_connects);
8830 soisconnected(so);
8831#ifdef MAC
8832 mac_socketpeer_set_from_mbuf(m, so);
8833#endif
8834 /* Do window scaling on this connection? */
8835 if ((tp->t_flags & (TF_RCVD_SCALE | TF_REQ_SCALE)) ==
8836 (TF_RCVD_SCALE | TF_REQ_SCALE)) {
8837 tp->rcv_scale = tp->request_r_scale;
8838 }
8839 tp->rcv_adv += min(tp->rcv_wnd,
8840 TCP_MAXWIN << tp->rcv_scale);
8841 /*
8842 * If not all the data that was sent in the TFO SYN
8843 * has been acked, resend the remainder right away.
8844 */
8845 if (IS_FASTOPEN(tp->t_flags) &&
8846 (tp->snd_una != tp->snd_max)) {
8847 tp->snd_nxt = th->th_ack;
8848 tfo_partial = 1;
8849 }
8850 /*
8851 * If there's data, delay ACK; if there's also a FIN ACKNOW
8852 * will be turned on later.
8853 */
8854 if (DELAY_ACK(tp, bbr, 1) && tlen != 0 && !tfo_partial) {
8855 bbr->bbr_segs_rcvd += 1;
8856 tp->t_flags |= TF_DELACK;
8857 bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime);
8858 } else {
8859 bbr->r_wanted_output = 1;
8860 tp->t_flags |= TF_ACKNOW;
8861 }
8862 if (SEQ_GT(th->th_ack, tp->iss)) {
8863 /*
8864 * The SYN is acked
8865 * handle it specially.
8866 */
8867 bbr_log_syn(tp, to);
8868 }
8869 if (SEQ_GT(th->th_ack, tp->snd_una)) {
8870 /*
8871 * We advance snd_una for the
8872 * fast open case. If th_ack is
8873 * acknowledging data beyond
8874 * snd_una we can't just call
8875 * ack-processing since the
8876 * data stream in our send-map
8877 * will start at snd_una + 1 (one
8878 * beyond the SYN). If its just
8879 * equal we don't need to do that
8880 * and there is no send_map.
8881 */
8882 tp->snd_una++;
8883 }
8884 /*
8885 * Received <SYN,ACK> in SYN_SENT[*] state. Transitions:
8886 * SYN_SENT --> ESTABLISHED SYN_SENT* --> FIN_WAIT_1
8887 */
8888 tp->t_starttime = ticks;
8889 if (tp->t_flags & TF_NEEDFIN) {
8890 tcp_state_change(tp, TCPS_FIN_WAIT_1);
8891 tp->t_flags &= ~TF_NEEDFIN;
8892 thflags &= ~TH_SYN;
8893 } else {
8894 tcp_state_change(tp, TCPS_ESTABLISHED);
8895 TCP_PROBE5(connect__established, NULL, tp,
8896 mtod(m, const char *), tp, th);
8897 cc_conn_init(tp);
8898 }
8899 } else {
8900 /*
8901 * Received initial SYN in SYN-SENT[*] state => simultaneous
8902 * open. If segment contains CC option and there is a
8903 * cached CC, apply TAO test. If it succeeds, connection is *
8904 * half-synchronized. Otherwise, do 3-way handshake:
8905 * SYN-SENT -> SYN-RECEIVED SYN-SENT* -> SYN-RECEIVED* If
8906 * there was no CC option, clear cached CC value.
8907 */
8908 tp->t_flags |= (TF_ACKNOW | TF_NEEDSYN);
8909 tcp_state_change(tp, TCPS_SYN_RECEIVED);
8910 }
8911 INP_WLOCK_ASSERT(tp->t_inpcb);
8912 /*
8913 * Advance th->th_seq to correspond to first data byte. If data,
8914 * trim to stay within window, dropping FIN if necessary.
8915 */
8916 th->th_seq++;
8917 if (tlen > tp->rcv_wnd) {
8918 todrop = tlen - tp->rcv_wnd;
8919 m_adj(m, -todrop);
8920 tlen = tp->rcv_wnd;
8921 thflags &= ~TH_FIN;
8922 KMOD_TCPSTAT_INC(tcps_rcvpackafterwin);
8923 KMOD_TCPSTAT_ADD(tcps_rcvbyteafterwin, todrop);
8924 }
8925 tp->snd_wl1 = th->th_seq - 1;
8926 tp->rcv_up = th->th_seq;
8927 /*
8928 * Client side of transaction: already sent SYN and data. If the
8929 * remote host used T/TCP to validate the SYN, our data will be
8930 * ACK'd; if so, enter normal data segment processing in the middle
8931 * of step 5, ack processing. Otherwise, goto step 6.
8932 */
8933 if (thflags & TH_ACK) {
8934 if ((to->to_flags & TOF_TS) != 0) {
8935 uint32_t t, rtt;
8936
8937 t = tcp_tv_to_mssectick(&bbr->rc_tv);
8938 if (TSTMP_GEQ(t, to->to_tsecr)) {
8939 rtt = t - to->to_tsecr;
8940 if (rtt == 0) {
8941 rtt = 1;
8942 }
8943 rtt *= MS_IN_USEC;
8944 tcp_bbr_xmit_timer(bbr, rtt, 0, 0, 0);
8945 apply_filter_min_small(&bbr->r_ctl.rc_rttprop,
8946 rtt, bbr->r_ctl.rc_rcvtime);
8947 }
8948 }
8949 if (bbr_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val))
8950 return (ret_val);
8951 /* We may have changed to FIN_WAIT_1 above */
8952 if (tp->t_state == TCPS_FIN_WAIT_1) {
8953 /*
8954 * In FIN_WAIT_1 STATE in addition to the processing
8955 * for the ESTABLISHED state if our FIN is now
8956 * acknowledged then enter FIN_WAIT_2.
8957 */
8958 if (ourfinisacked) {
8959 /*
8960 * If we can't receive any more data, then
8961 * closing user can proceed. Starting the
8962 * timer is contrary to the specification,
8963 * but if we don't get a FIN we'll hang
8964 * forever.
8965 *
8966 * XXXjl: we should release the tp also, and
8967 * use a compressed state.
8968 */
8969 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
8970 soisdisconnected(so);
8971 tcp_timer_activate(tp, TT_2MSL,
8972 (tcp_fast_finwait2_recycle ?
8973 tcp_finwait2_timeout :
8974 TP_MAXIDLE(tp)));
8975 }
8976 tcp_state_change(tp, TCPS_FIN_WAIT_2);
8977 }
8978 }
8979 }
8980 return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen,
8981 tiwin, thflags, nxt_pkt));
8982}
8983
8984/*
8985 * Return value of 1, the TCB is unlocked and most
8986 * likely gone, return value of 0, the TCB is still
8987 * locked.
8988 */
8989static int
8990bbr_do_syn_recv(struct mbuf *m, struct tcphdr *th, struct socket *so,
8991 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
8992 uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos)
8993{
8994 int32_t ourfinisacked = 0;
8995 int32_t ret_val;
8996 struct tcp_bbr *bbr;
8997
8998 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
8999 ctf_calc_rwin(so, tp);
9000 if ((thflags & TH_ACK) &&
9001 (SEQ_LEQ(th->th_ack, tp->snd_una) ||
9002 SEQ_GT(th->th_ack, tp->snd_max))) {
9003 tcp_log_end_status(tp, TCP_EI_STATUS_RST_IN_FRONT);
9004 ctf_do_dropwithreset(m, tp, th, BANDLIM_RST_OPENPORT, tlen);
9005 return (1);
9006 }
9007 if (IS_FASTOPEN(tp->t_flags)) {
9008 /*
9009 * When a TFO connection is in SYN_RECEIVED, the only valid
9010 * packets are the initial SYN, a retransmit/copy of the
9011 * initial SYN (possibly with a subset of the original
9012 * data), a valid ACK, a FIN, or a RST.
9013 */
9014 if ((thflags & (TH_SYN | TH_ACK)) == (TH_SYN | TH_ACK)) {
9015 tcp_log_end_status(tp, TCP_EI_STATUS_RST_IN_FRONT);
9016 ctf_do_dropwithreset(m, tp, th, BANDLIM_RST_OPENPORT, tlen);
9017 return (1);
9018 } else if (thflags & TH_SYN) {
9019 /* non-initial SYN is ignored */
9020 if ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_RXT) ||
9021 (bbr->r_ctl.rc_hpts_flags & PACE_TMR_TLP) ||
9022 (bbr->r_ctl.rc_hpts_flags & PACE_TMR_RACK)) {
9023 ctf_do_drop(m, NULL);
9024 return (0);
9025 }
9026 } else if (!(thflags & (TH_ACK | TH_FIN | TH_RST))) {
9027 ctf_do_drop(m, NULL);
9028 return (0);
9029 }
9030 }
9031 if ((thflags & TH_RST) ||
9032 (tp->t_fin_is_rst && (thflags & TH_FIN)))
9033 return (ctf_process_rst(m, th, so, tp));
9034 /*
9035 * RFC 1323 PAWS: If we have a timestamp reply on this segment and
9036 * it's less than ts_recent, drop it.
9037 */
9038 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&
9039 TSTMP_LT(to->to_tsval, tp->ts_recent)) {
9040 if (ctf_ts_check(m, th, tp, tlen, thflags, &ret_val))
9041 return (ret_val);
9042 }
9043 /*
9044 * In the SYN-RECEIVED state, validate that the packet belongs to
9045 * this connection before trimming the data to fit the receive
9046 * window. Check the sequence number versus IRS since we know the
9047 * sequence numbers haven't wrapped. This is a partial fix for the
9048 * "LAND" DoS attack.
9049 */
9050 if (SEQ_LT(th->th_seq, tp->irs)) {
9051 tcp_log_end_status(tp, TCP_EI_STATUS_RST_IN_FRONT);
9052 ctf_do_dropwithreset(m, tp, th, BANDLIM_RST_OPENPORT, tlen);
9053 return (1);
9054 }
9055 INP_WLOCK_ASSERT(tp->t_inpcb);
9056 if (ctf_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val)) {
9057 return (ret_val);
9058 }
9059 /*
9060 * If last ACK falls within this segment's sequence numbers, record
9061 * its timestamp. NOTE: 1) That the test incorporates suggestions
9062 * from the latest proposal of the tcplw@cray.com list (Braden
9063 * 1993/04/26). 2) That updating only on newer timestamps interferes
9064 * with our earlier PAWS tests, so this check should be solely
9065 * predicated on the sequence space of this segment. 3) That we
9066 * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ
9067 * + SEG.Len instead of RFC1323's Last.ACK.Sent < SEG.SEQ +
9068 * SEG.Len, This modified check allows us to overcome RFC1323's
9069 * limitations as described in Stevens TCP/IP Illustrated Vol. 2
9070 * p.869. In such cases, we can still calculate the RTT correctly
9071 * when RCV.NXT == Last.ACK.Sent.
9072 */
9073 if ((to->to_flags & TOF_TS) != 0 &&
9074 SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&
9075 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +
9076 ((thflags & (TH_SYN | TH_FIN)) != 0))) {
9077 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv);
9078 tp->ts_recent = to->to_tsval;
9079 }
9080 tp->snd_wnd = tiwin;
9081 /*
9082 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag
9083 * is on (half-synchronized state), then queue data for later
9084 * processing; else drop segment and return.
9085 */
9086 if ((thflags & TH_ACK) == 0) {
9087 if (IS_FASTOPEN(tp->t_flags)) {
9088 cc_conn_init(tp);
9089 }
9090 return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen,
9091 tiwin, thflags, nxt_pkt));
9092 }
9093 KMOD_TCPSTAT_INC(tcps_connects);
9094 soisconnected(so);
9095 /* Do window scaling? */
9096 if ((tp->t_flags & (TF_RCVD_SCALE | TF_REQ_SCALE)) ==
9097 (TF_RCVD_SCALE | TF_REQ_SCALE)) {
9098 tp->rcv_scale = tp->request_r_scale;
9099 }
9100 /*
9101 * ok for the first time in lets see if we can use the ts to figure
9102 * out what the initial RTT was.
9103 */
9104 if ((to->to_flags & TOF_TS) != 0) {
9105 uint32_t t, rtt;
9106
9107 t = tcp_tv_to_mssectick(&bbr->rc_tv);
9108 if (TSTMP_GEQ(t, to->to_tsecr)) {
9109 rtt = t - to->to_tsecr;
9110 if (rtt == 0) {
9111 rtt = 1;
9112 }
9113 rtt *= MS_IN_USEC;
9114 tcp_bbr_xmit_timer(bbr, rtt, 0, 0, 0);
9115 apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, bbr->r_ctl.rc_rcvtime);
9116 }
9117 }
9118 /* Drop off any SYN in the send map (probably not there) */
9119 if (thflags & TH_ACK)
9120 bbr_log_syn(tp, to);
9121 if (IS_FASTOPEN(tp->t_flags) && tp->t_tfo_pending) {
9122 tcp_fastopen_decrement_counter(tp->t_tfo_pending);
9123 tp->t_tfo_pending = NULL;
9124 }
9125 /*
9126 * Make transitions: SYN-RECEIVED -> ESTABLISHED SYN-RECEIVED* ->
9127 * FIN-WAIT-1
9128 */
9129 tp->t_starttime = ticks;
9130 if (tp->t_flags & TF_NEEDFIN) {
9131 tcp_state_change(tp, TCPS_FIN_WAIT_1);
9132 tp->t_flags &= ~TF_NEEDFIN;
9133 } else {
9134 tcp_state_change(tp, TCPS_ESTABLISHED);
9135 TCP_PROBE5(accept__established, NULL, tp,
9136 mtod(m, const char *), tp, th);
9137 /*
9138 * TFO connections call cc_conn_init() during SYN
9139 * processing. Calling it again here for such connections
9140 * is not harmless as it would undo the snd_cwnd reduction
9141 * that occurs when a TFO SYN|ACK is retransmitted.
9142 */
9143 if (!IS_FASTOPEN(tp->t_flags))
9144 cc_conn_init(tp);
9145 }
9146 /*
9147 * Account for the ACK of our SYN prior to
9148 * regular ACK processing below, except for
9149 * simultaneous SYN, which is handled later.
9150 */
9151 if (SEQ_GT(th->th_ack, tp->snd_una) && !(tp->t_flags & TF_NEEDSYN))
9152 tp->snd_una++;
9153 /*
9154 * If segment contains data or ACK, will call tcp_reass() later; if
9155 * not, do so now to pass queued data to user.
9156 */
9157 if (tlen == 0 && (thflags & TH_FIN) == 0) {
9158 (void)tcp_reass(tp, (struct tcphdr *)0, NULL, 0,
9159 (struct mbuf *)0);
9160 if (tp->t_flags & TF_WAKESOR) {
9161 tp->t_flags &= ~TF_WAKESOR;
9162 /* NB: sorwakeup_locked() does an implicit unlock. */
9163 sorwakeup_locked(so);
9164 }
9165 }
9166 tp->snd_wl1 = th->th_seq - 1;
9167 if (bbr_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val)) {
9168 return (ret_val);
9169 }
9170 if (tp->t_state == TCPS_FIN_WAIT_1) {
9171 /* We could have went to FIN_WAIT_1 (or EST) above */
9172 /*
9173 * In FIN_WAIT_1 STATE in addition to the processing for the
9174 * ESTABLISHED state if our FIN is now acknowledged then
9175 * enter FIN_WAIT_2.
9176 */
9177 if (ourfinisacked) {
9178 /*
9179 * If we can't receive any more data, then closing
9180 * user can proceed. Starting the timer is contrary
9181 * to the specification, but if we don't get a FIN
9182 * we'll hang forever.
9183 *
9184 * XXXjl: we should release the tp also, and use a
9185 * compressed state.
9186 */
9187 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
9188 soisdisconnected(so);
9189 tcp_timer_activate(tp, TT_2MSL,
9190 (tcp_fast_finwait2_recycle ?
9191 tcp_finwait2_timeout :
9192 TP_MAXIDLE(tp)));
9193 }
9194 tcp_state_change(tp, TCPS_FIN_WAIT_2);
9195 }
9196 }
9197 return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen,
9198 tiwin, thflags, nxt_pkt));
9199}
9200
9201/*
9202 * Return value of 1, the TCB is unlocked and most
9203 * likely gone, return value of 0, the TCB is still
9204 * locked.
9205 */
9206static int
9207bbr_do_established(struct mbuf *m, struct tcphdr *th, struct socket *so,
9208 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
9209 uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos)
9210{
9211 struct tcp_bbr *bbr;
9212 int32_t ret_val;
9213
9214 /*
9215 * Header prediction: check for the two common cases of a
9216 * uni-directional data xfer. If the packet has no control flags,
9217 * is in-sequence, the window didn't change and we're not
9218 * retransmitting, it's a candidate. If the length is zero and the
9219 * ack moved forward, we're the sender side of the xfer. Just free
9220 * the data acked & wake any higher level process that was blocked
9221 * waiting for space. If the length is non-zero and the ack didn't
9222 * move, we're the receiver side. If we're getting packets in-order
9223 * (the reassembly queue is empty), add the data toc The socket
9224 * buffer and note that we need a delayed ack. Make sure that the
9225 * hidden state-flags are also off. Since we check for
9226 * TCPS_ESTABLISHED first, it can only be TH_NEEDSYN.
9227 */
9228 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
9229 if (bbr->r_ctl.rc_delivered < (4 * tp->t_maxseg)) {
9230 /*
9231 * If we have delived under 4 segments increase the initial
9232 * window if raised by the peer. We use this to determine
9233 * dynamic and static rwnd's at the end of a connection.
9234 */
9235 bbr->r_ctl.rc_init_rwnd = max(tiwin, tp->snd_wnd);
9236 }
9237 if (__predict_true(((to->to_flags & TOF_SACK) == 0)) &&
9238 __predict_true((thflags & (TH_SYN | TH_FIN | TH_RST | TH_URG | TH_ACK)) == TH_ACK) &&
9239 __predict_true(SEGQ_EMPTY(tp)) &&
9240 __predict_true(th->th_seq == tp->rcv_nxt)) {
9241 if (tlen == 0) {
9242 if (bbr_fastack(m, th, so, tp, to, drop_hdrlen, tlen,
9243 tiwin, nxt_pkt, iptos)) {
9244 return (0);
9245 }
9246 } else {
9247 if (bbr_do_fastnewdata(m, th, so, tp, to, drop_hdrlen, tlen,
9248 tiwin, nxt_pkt)) {
9249 return (0);
9250 }
9251 }
9252 }
9253 ctf_calc_rwin(so, tp);
9254
9255 if ((thflags & TH_RST) ||
9256 (tp->t_fin_is_rst && (thflags & TH_FIN)))
9257 return (ctf_process_rst(m, th, so, tp));
9258 /*
9259 * RFC5961 Section 4.2 Send challenge ACK for any SYN in
9260 * synchronized state.
9261 */
9262 if (thflags & TH_SYN) {
9263 ctf_challenge_ack(m, th, tp, &ret_val);
9264 return (ret_val);
9265 }
9266 /*
9267 * RFC 1323 PAWS: If we have a timestamp reply on this segment and
9268 * it's less than ts_recent, drop it.
9269 */
9270 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&
9271 TSTMP_LT(to->to_tsval, tp->ts_recent)) {
9272 if (ctf_ts_check(m, th, tp, tlen, thflags, &ret_val))
9273 return (ret_val);
9274 }
9275 INP_WLOCK_ASSERT(tp->t_inpcb);
9276 if (ctf_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val)) {
9277 return (ret_val);
9278 }
9279 /*
9280 * If last ACK falls within this segment's sequence numbers, record
9281 * its timestamp. NOTE: 1) That the test incorporates suggestions
9282 * from the latest proposal of the tcplw@cray.com list (Braden
9283 * 1993/04/26). 2) That updating only on newer timestamps interferes
9284 * with our earlier PAWS tests, so this check should be solely
9285 * predicated on the sequence space of this segment. 3) That we
9286 * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ
9287 * + SEG.Len instead of RFC1323's Last.ACK.Sent < SEG.SEQ +
9288 * SEG.Len, This modified check allows us to overcome RFC1323's
9289 * limitations as described in Stevens TCP/IP Illustrated Vol. 2
9290 * p.869. In such cases, we can still calculate the RTT correctly
9291 * when RCV.NXT == Last.ACK.Sent.
9292 */
9293 if ((to->to_flags & TOF_TS) != 0 &&
9294 SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&
9295 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +
9296 ((thflags & (TH_SYN | TH_FIN)) != 0))) {
9297 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv);
9298 tp->ts_recent = to->to_tsval;
9299 }
9300 /*
9301 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag
9302 * is on (half-synchronized state), then queue data for later
9303 * processing; else drop segment and return.
9304 */
9305 if ((thflags & TH_ACK) == 0) {
9306 if (tp->t_flags & TF_NEEDSYN) {
9307 return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen,
9308 tiwin, thflags, nxt_pkt));
9309 } else if (tp->t_flags & TF_ACKNOW) {
9310 ctf_do_dropafterack(m, tp, th, thflags, tlen, &ret_val);
9311 bbr->r_wanted_output = 1;
9312 return (ret_val);
9313 } else {
9314 ctf_do_drop(m, NULL);
9315 return (0);
9316 }
9317 }
9318 /*
9319 * Ack processing.
9320 */
9321 if (bbr_process_ack(m, th, so, tp, to, tiwin, tlen, NULL, thflags, &ret_val)) {
9322 return (ret_val);
9323 }
9324 if (sbavail(&so->so_snd)) {
9325 if (ctf_progress_timeout_check(tp, true)) {
9326 bbr_log_progress_event(bbr, tp, tick, PROGRESS_DROP, __LINE__);
9327 ctf_do_dropwithreset_conn(m, tp, th, BANDLIM_RST_OPENPORT, tlen);
9328 return (1);
9329 }
9330 }
9331 /* State changes only happen in bbr_process_data() */
9332 return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen,
9333 tiwin, thflags, nxt_pkt));
9334}
9335
9336/*
9337 * Return value of 1, the TCB is unlocked and most
9338 * likely gone, return value of 0, the TCB is still
9339 * locked.
9340 */
9341static int
9342bbr_do_close_wait(struct mbuf *m, struct tcphdr *th, struct socket *so,
9343 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
9344 uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos)
9345{
9346 struct tcp_bbr *bbr;
9347 int32_t ret_val;
9348
9349 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
9350 ctf_calc_rwin(so, tp);
9351 if ((thflags & TH_RST) ||
9352 (tp->t_fin_is_rst && (thflags & TH_FIN)))
9353 return (ctf_process_rst(m, th, so, tp));
9354 /*
9355 * RFC5961 Section 4.2 Send challenge ACK for any SYN in
9356 * synchronized state.
9357 */
9358 if (thflags & TH_SYN) {
9359 ctf_challenge_ack(m, th, tp, &ret_val);
9360 return (ret_val);
9361 }
9362 /*
9363 * RFC 1323 PAWS: If we have a timestamp reply on this segment and
9364 * it's less than ts_recent, drop it.
9365 */
9366 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&
9367 TSTMP_LT(to->to_tsval, tp->ts_recent)) {
9368 if (ctf_ts_check(m, th, tp, tlen, thflags, &ret_val))
9369 return (ret_val);
9370 }
9371 INP_WLOCK_ASSERT(tp->t_inpcb);
9372 if (ctf_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val)) {
9373 return (ret_val);
9374 }
9375 /*
9376 * If last ACK falls within this segment's sequence numbers, record
9377 * its timestamp. NOTE: 1) That the test incorporates suggestions
9378 * from the latest proposal of the tcplw@cray.com list (Braden
9379 * 1993/04/26). 2) That updating only on newer timestamps interferes
9380 * with our earlier PAWS tests, so this check should be solely
9381 * predicated on the sequence space of this segment. 3) That we
9382 * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ
9383 * + SEG.Len instead of RFC1323's Last.ACK.Sent < SEG.SEQ +
9384 * SEG.Len, This modified check allows us to overcome RFC1323's
9385 * limitations as described in Stevens TCP/IP Illustrated Vol. 2
9386 * p.869. In such cases, we can still calculate the RTT correctly
9387 * when RCV.NXT == Last.ACK.Sent.
9388 */
9389 if ((to->to_flags & TOF_TS) != 0 &&
9390 SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&
9391 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +
9392 ((thflags & (TH_SYN | TH_FIN)) != 0))) {
9393 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv);
9394 tp->ts_recent = to->to_tsval;
9395 }
9396 /*
9397 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag
9398 * is on (half-synchronized state), then queue data for later
9399 * processing; else drop segment and return.
9400 */
9401 if ((thflags & TH_ACK) == 0) {
9402 if (tp->t_flags & TF_NEEDSYN) {
9403 return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen,
9404 tiwin, thflags, nxt_pkt));
9405 } else if (tp->t_flags & TF_ACKNOW) {
9406 ctf_do_dropafterack(m, tp, th, thflags, tlen, &ret_val);
9407 bbr->r_wanted_output = 1;
9408 return (ret_val);
9409 } else {
9410 ctf_do_drop(m, NULL);
9411 return (0);
9412 }
9413 }
9414 /*
9415 * Ack processing.
9416 */
9417 if (bbr_process_ack(m, th, so, tp, to, tiwin, tlen, NULL, thflags, &ret_val)) {
9418 return (ret_val);
9419 }
9420 if (sbavail(&so->so_snd)) {
9421 if (ctf_progress_timeout_check(tp, true)) {
9422 bbr_log_progress_event(bbr, tp, tick, PROGRESS_DROP, __LINE__);
9423 ctf_do_dropwithreset_conn(m, tp, th, BANDLIM_RST_OPENPORT, tlen);
9424 return (1);
9425 }
9426 }
9427 return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen,
9428 tiwin, thflags, nxt_pkt));
9429}
9430
9431static int
9432bbr_check_data_after_close(struct mbuf *m, struct tcp_bbr *bbr,
9433 struct tcpcb *tp, int32_t * tlen, struct tcphdr *th, struct socket *so)
9434{
9435
9436 if (bbr->rc_allow_data_af_clo == 0) {
9437close_now:
9438 tcp_log_end_status(tp, TCP_EI_STATUS_DATA_A_CLOSE);
9439 /* tcp_close will kill the inp pre-log the Reset */
9440 tcp_log_end_status(tp, TCP_EI_STATUS_SERVER_RST);
9441 tp = tcp_close(tp);
9442 KMOD_TCPSTAT_INC(tcps_rcvafterclose);
9443 ctf_do_dropwithreset(m, tp, th, BANDLIM_UNLIMITED, (*tlen));
9444 return (1);
9445 }
9446 if (sbavail(&so->so_snd) == 0)
9447 goto close_now;
9448 /* Ok we allow data that is ignored and a followup reset */
9449 tp->rcv_nxt = th->th_seq + *tlen;
9450 tp->t_flags2 |= TF2_DROP_AF_DATA;
9451 bbr->r_wanted_output = 1;
9452 *tlen = 0;
9453 return (0);
9454}
9455
9456/*
9457 * Return value of 1, the TCB is unlocked and most
9458 * likely gone, return value of 0, the TCB is still
9459 * locked.
9460 */
9461static int
9462bbr_do_fin_wait_1(struct mbuf *m, struct tcphdr *th, struct socket *so,
9463 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
9464 uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos)
9465{
9466 int32_t ourfinisacked = 0;
9467 int32_t ret_val;
9468 struct tcp_bbr *bbr;
9469
9470 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
9471 ctf_calc_rwin(so, tp);
9472 if ((thflags & TH_RST) ||
9473 (tp->t_fin_is_rst && (thflags & TH_FIN)))
9474 return (ctf_process_rst(m, th, so, tp));
9475 /*
9476 * RFC5961 Section 4.2 Send challenge ACK for any SYN in
9477 * synchronized state.
9478 */
9479 if (thflags & TH_SYN) {
9480 ctf_challenge_ack(m, th, tp, &ret_val);
9481 return (ret_val);
9482 }
9483 /*
9484 * RFC 1323 PAWS: If we have a timestamp reply on this segment and
9485 * it's less than ts_recent, drop it.
9486 */
9487 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&
9488 TSTMP_LT(to->to_tsval, tp->ts_recent)) {
9489 if (ctf_ts_check(m, th, tp, tlen, thflags, &ret_val))
9490 return (ret_val);
9491 }
9492 INP_WLOCK_ASSERT(tp->t_inpcb);
9493 if (ctf_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val)) {
9494 return (ret_val);
9495 }
9496 /*
9497 * If new data are received on a connection after the user processes
9498 * are gone, then RST the other end.
9499 */
9500 if ((so->so_state & SS_NOFDREF) && tlen) {
9501 /*
9502 * We call a new function now so we might continue and setup
9503 * to reset at all data being ack'd.
9504 */
9505 if (bbr_check_data_after_close(m, bbr, tp, &tlen, th, so))
9506 return (1);
9507 }
9508 /*
9509 * If last ACK falls within this segment's sequence numbers, record
9510 * its timestamp. NOTE: 1) That the test incorporates suggestions
9511 * from the latest proposal of the tcplw@cray.com list (Braden
9512 * 1993/04/26). 2) That updating only on newer timestamps interferes
9513 * with our earlier PAWS tests, so this check should be solely
9514 * predicated on the sequence space of this segment. 3) That we
9515 * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ
9516 * + SEG.Len instead of RFC1323's Last.ACK.Sent < SEG.SEQ +
9517 * SEG.Len, This modified check allows us to overcome RFC1323's
9518 * limitations as described in Stevens TCP/IP Illustrated Vol. 2
9519 * p.869. In such cases, we can still calculate the RTT correctly
9520 * when RCV.NXT == Last.ACK.Sent.
9521 */
9522 if ((to->to_flags & TOF_TS) != 0 &&
9523 SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&
9524 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +
9525 ((thflags & (TH_SYN | TH_FIN)) != 0))) {
9526 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv);
9527 tp->ts_recent = to->to_tsval;
9528 }
9529 /*
9530 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag
9531 * is on (half-synchronized state), then queue data for later
9532 * processing; else drop segment and return.
9533 */
9534 if ((thflags & TH_ACK) == 0) {
9535 if (tp->t_flags & TF_NEEDSYN) {
9536 return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen,
9537 tiwin, thflags, nxt_pkt));
9538 } else if (tp->t_flags & TF_ACKNOW) {
9539 ctf_do_dropafterack(m, tp, th, thflags, tlen, &ret_val);
9540 bbr->r_wanted_output = 1;
9541 return (ret_val);
9542 } else {
9543 ctf_do_drop(m, NULL);
9544 return (0);
9545 }
9546 }
9547 /*
9548 * Ack processing.
9549 */
9550 if (bbr_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val)) {
9551 return (ret_val);
9552 }
9553 if (ourfinisacked) {
9554 /*
9555 * If we can't receive any more data, then closing user can
9556 * proceed. Starting the timer is contrary to the
9557 * specification, but if we don't get a FIN we'll hang
9558 * forever.
9559 *
9560 * XXXjl: we should release the tp also, and use a
9561 * compressed state.
9562 */
9563 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
9564 soisdisconnected(so);
9565 tcp_timer_activate(tp, TT_2MSL,
9566 (tcp_fast_finwait2_recycle ?
9567 tcp_finwait2_timeout :
9568 TP_MAXIDLE(tp)));
9569 }
9570 tcp_state_change(tp, TCPS_FIN_WAIT_2);
9571 }
9572 if (sbavail(&so->so_snd)) {
9573 if (ctf_progress_timeout_check(tp, true)) {
9574 bbr_log_progress_event(bbr, tp, tick, PROGRESS_DROP, __LINE__);
9575 ctf_do_dropwithreset_conn(m, tp, th, BANDLIM_RST_OPENPORT, tlen);
9576 return (1);
9577 }
9578 }
9579 return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen,
9580 tiwin, thflags, nxt_pkt));
9581}
9582
9583/*
9584 * Return value of 1, the TCB is unlocked and most
9585 * likely gone, return value of 0, the TCB is still
9586 * locked.
9587 */
9588static int
9589bbr_do_closing(struct mbuf *m, struct tcphdr *th, struct socket *so,
9590 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
9591 uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos)
9592{
9593 int32_t ourfinisacked = 0;
9594 int32_t ret_val;
9595 struct tcp_bbr *bbr;
9596
9597 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
9598 ctf_calc_rwin(so, tp);
9599 if ((thflags & TH_RST) ||
9600 (tp->t_fin_is_rst && (thflags & TH_FIN)))
9601 return (ctf_process_rst(m, th, so, tp));
9602 /*
9603 * RFC5961 Section 4.2 Send challenge ACK for any SYN in
9604 * synchronized state.
9605 */
9606 if (thflags & TH_SYN) {
9607 ctf_challenge_ack(m, th, tp, &ret_val);
9608 return (ret_val);
9609 }
9610 /*
9611 * RFC 1323 PAWS: If we have a timestamp reply on this segment and
9612 * it's less than ts_recent, drop it.
9613 */
9614 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&
9615 TSTMP_LT(to->to_tsval, tp->ts_recent)) {
9616 if (ctf_ts_check(m, th, tp, tlen, thflags, &ret_val))
9617 return (ret_val);
9618 }
9619 INP_WLOCK_ASSERT(tp->t_inpcb);
9620 if (ctf_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val)) {
9621 return (ret_val);
9622 }
9623 /*
9624 * If new data are received on a connection after the user processes
9625 * are gone, then RST the other end.
9626 */
9627 if ((so->so_state & SS_NOFDREF) && tlen) {
9628 /*
9629 * We call a new function now so we might continue and setup
9630 * to reset at all data being ack'd.
9631 */
9632 if (bbr_check_data_after_close(m, bbr, tp, &tlen, th, so))
9633 return (1);
9634 }
9635 /*
9636 * If last ACK falls within this segment's sequence numbers, record
9637 * its timestamp. NOTE: 1) That the test incorporates suggestions
9638 * from the latest proposal of the tcplw@cray.com list (Braden
9639 * 1993/04/26). 2) That updating only on newer timestamps interferes
9640 * with our earlier PAWS tests, so this check should be solely
9641 * predicated on the sequence space of this segment. 3) That we
9642 * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ
9643 * + SEG.Len instead of RFC1323's Last.ACK.Sent < SEG.SEQ +
9644 * SEG.Len, This modified check allows us to overcome RFC1323's
9645 * limitations as described in Stevens TCP/IP Illustrated Vol. 2
9646 * p.869. In such cases, we can still calculate the RTT correctly
9647 * when RCV.NXT == Last.ACK.Sent.
9648 */
9649 if ((to->to_flags & TOF_TS) != 0 &&
9650 SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&
9651 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +
9652 ((thflags & (TH_SYN | TH_FIN)) != 0))) {
9653 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv);
9654 tp->ts_recent = to->to_tsval;
9655 }
9656 /*
9657 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag
9658 * is on (half-synchronized state), then queue data for later
9659 * processing; else drop segment and return.
9660 */
9661 if ((thflags & TH_ACK) == 0) {
9662 if (tp->t_flags & TF_NEEDSYN) {
9663 return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen,
9664 tiwin, thflags, nxt_pkt));
9665 } else if (tp->t_flags & TF_ACKNOW) {
9666 ctf_do_dropafterack(m, tp, th, thflags, tlen, &ret_val);
9667 bbr->r_wanted_output = 1;
9668 return (ret_val);
9669 } else {
9670 ctf_do_drop(m, NULL);
9671 return (0);
9672 }
9673 }
9674 /*
9675 * Ack processing.
9676 */
9677 if (bbr_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val)) {
9678 return (ret_val);
9679 }
9680 if (ourfinisacked) {
9681 tcp_twstart(tp);
9682 m_freem(m);
9683 return (1);
9684 }
9685 if (sbavail(&so->so_snd)) {
9686 if (ctf_progress_timeout_check(tp, true)) {
9687 bbr_log_progress_event(bbr, tp, tick, PROGRESS_DROP, __LINE__);
9688 ctf_do_dropwithreset_conn(m, tp, th, BANDLIM_RST_OPENPORT, tlen);
9689 return (1);
9690 }
9691 }
9692 return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen,
9693 tiwin, thflags, nxt_pkt));
9694}
9695
9696/*
9697 * Return value of 1, the TCB is unlocked and most
9698 * likely gone, return value of 0, the TCB is still
9699 * locked.
9700 */
9701static int
9702bbr_do_lastack(struct mbuf *m, struct tcphdr *th, struct socket *so,
9703 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
9704 uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos)
9705{
9706 int32_t ourfinisacked = 0;
9707 int32_t ret_val;
9708 struct tcp_bbr *bbr;
9709
9710 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
9711 ctf_calc_rwin(so, tp);
9712 if ((thflags & TH_RST) ||
9713 (tp->t_fin_is_rst && (thflags & TH_FIN)))
9714 return (ctf_process_rst(m, th, so, tp));
9715 /*
9716 * RFC5961 Section 4.2 Send challenge ACK for any SYN in
9717 * synchronized state.
9718 */
9719 if (thflags & TH_SYN) {
9720 ctf_challenge_ack(m, th, tp, &ret_val);
9721 return (ret_val);
9722 }
9723 /*
9724 * RFC 1323 PAWS: If we have a timestamp reply on this segment and
9725 * it's less than ts_recent, drop it.
9726 */
9727 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&
9728 TSTMP_LT(to->to_tsval, tp->ts_recent)) {
9729 if (ctf_ts_check(m, th, tp, tlen, thflags, &ret_val))
9730 return (ret_val);
9731 }
9732 INP_WLOCK_ASSERT(tp->t_inpcb);
9733 if (ctf_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val)) {
9734 return (ret_val);
9735 }
9736 /*
9737 * If new data are received on a connection after the user processes
9738 * are gone, then RST the other end.
9739 */
9740 if ((so->so_state & SS_NOFDREF) && tlen) {
9741 /*
9742 * We call a new function now so we might continue and setup
9743 * to reset at all data being ack'd.
9744 */
9745 if (bbr_check_data_after_close(m, bbr, tp, &tlen, th, so))
9746 return (1);
9747 }
9748 /*
9749 * If last ACK falls within this segment's sequence numbers, record
9750 * its timestamp. NOTE: 1) That the test incorporates suggestions
9751 * from the latest proposal of the tcplw@cray.com list (Braden
9752 * 1993/04/26). 2) That updating only on newer timestamps interferes
9753 * with our earlier PAWS tests, so this check should be solely
9754 * predicated on the sequence space of this segment. 3) That we
9755 * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ
9756 * + SEG.Len instead of RFC1323's Last.ACK.Sent < SEG.SEQ +
9757 * SEG.Len, This modified check allows us to overcome RFC1323's
9758 * limitations as described in Stevens TCP/IP Illustrated Vol. 2
9759 * p.869. In such cases, we can still calculate the RTT correctly
9760 * when RCV.NXT == Last.ACK.Sent.
9761 */
9762 if ((to->to_flags & TOF_TS) != 0 &&
9763 SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&
9764 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +
9765 ((thflags & (TH_SYN | TH_FIN)) != 0))) {
9766 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv);
9767 tp->ts_recent = to->to_tsval;
9768 }
9769 /*
9770 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag
9771 * is on (half-synchronized state), then queue data for later
9772 * processing; else drop segment and return.
9773 */
9774 if ((thflags & TH_ACK) == 0) {
9775 if (tp->t_flags & TF_NEEDSYN) {
9776 return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen,
9777 tiwin, thflags, nxt_pkt));
9778 } else if (tp->t_flags & TF_ACKNOW) {
9779 ctf_do_dropafterack(m, tp, th, thflags, tlen, &ret_val);
9780 bbr->r_wanted_output = 1;
9781 return (ret_val);
9782 } else {
9783 ctf_do_drop(m, NULL);
9784 return (0);
9785 }
9786 }
9787 /*
9788 * case TCPS_LAST_ACK: Ack processing.
9789 */
9790 if (bbr_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val)) {
9791 return (ret_val);
9792 }
9793 if (ourfinisacked) {
9794 tp = tcp_close(tp);
9795 ctf_do_drop(m, tp);
9796 return (1);
9797 }
9798 if (sbavail(&so->so_snd)) {
9799 if (ctf_progress_timeout_check(tp, true)) {
9800 bbr_log_progress_event(bbr, tp, tick, PROGRESS_DROP, __LINE__);
9801 ctf_do_dropwithreset_conn(m, tp, th, BANDLIM_RST_OPENPORT, tlen);
9802 return (1);
9803 }
9804 }
9805 return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen,
9806 tiwin, thflags, nxt_pkt));
9807}
9808
9809/*
9810 * Return value of 1, the TCB is unlocked and most
9811 * likely gone, return value of 0, the TCB is still
9812 * locked.
9813 */
9814static int
9815bbr_do_fin_wait_2(struct mbuf *m, struct tcphdr *th, struct socket *so,
9816 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
9817 uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos)
9818{
9819 int32_t ourfinisacked = 0;
9820 int32_t ret_val;
9821 struct tcp_bbr *bbr;
9822
9823 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
9824 ctf_calc_rwin(so, tp);
9825 /* Reset receive buffer auto scaling when not in bulk receive mode. */
9826 if ((thflags & TH_RST) ||
9827 (tp->t_fin_is_rst && (thflags & TH_FIN)))
9828 return (ctf_process_rst(m, th, so, tp));
9829
9830 /*
9831 * RFC5961 Section 4.2 Send challenge ACK for any SYN in
9832 * synchronized state.
9833 */
9834 if (thflags & TH_SYN) {
9835 ctf_challenge_ack(m, th, tp, &ret_val);
9836 return (ret_val);
9837 }
9838 INP_WLOCK_ASSERT(tp->t_inpcb);
9839 /*
9840 * RFC 1323 PAWS: If we have a timestamp reply on this segment and
9841 * it's less than ts_recent, drop it.
9842 */
9843 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&
9844 TSTMP_LT(to->to_tsval, tp->ts_recent)) {
9845 if (ctf_ts_check(m, th, tp, tlen, thflags, &ret_val))
9846 return (ret_val);
9847 }
9848 INP_WLOCK_ASSERT(tp->t_inpcb);
9849 if (ctf_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val)) {
9850 return (ret_val);
9851 }
9852 /*
9853 * If new data are received on a connection after the user processes
9854 * are gone, then we may RST the other end depending on the outcome
9855 * of bbr_check_data_after_close.
9856 */
9857 if ((so->so_state & SS_NOFDREF) &&
9858 tlen) {
9859 /*
9860 * We call a new function now so we might continue and setup
9861 * to reset at all data being ack'd.
9862 */
9863 if (bbr_check_data_after_close(m, bbr, tp, &tlen, th, so))
9864 return (1);
9865 }
9866 INP_WLOCK_ASSERT(tp->t_inpcb);
9867 /*
9868 * If last ACK falls within this segment's sequence numbers, record
9869 * its timestamp. NOTE: 1) That the test incorporates suggestions
9870 * from the latest proposal of the tcplw@cray.com list (Braden
9871 * 1993/04/26). 2) That updating only on newer timestamps interferes
9872 * with our earlier PAWS tests, so this check should be solely
9873 * predicated on the sequence space of this segment. 3) That we
9874 * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ
9875 * + SEG.Len instead of RFC1323's Last.ACK.Sent < SEG.SEQ +
9876 * SEG.Len, This modified check allows us to overcome RFC1323's
9877 * limitations as described in Stevens TCP/IP Illustrated Vol. 2
9878 * p.869. In such cases, we can still calculate the RTT correctly
9879 * when RCV.NXT == Last.ACK.Sent.
9880 */
9881 INP_WLOCK_ASSERT(tp->t_inpcb);
9882 if ((to->to_flags & TOF_TS) != 0 &&
9883 SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&
9884 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +
9885 ((thflags & (TH_SYN | TH_FIN)) != 0))) {
9886 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv);
9887 tp->ts_recent = to->to_tsval;
9888 }
9889 /*
9890 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag
9891 * is on (half-synchronized state), then queue data for later
9892 * processing; else drop segment and return.
9893 */
9894 if ((thflags & TH_ACK) == 0) {
9895 if (tp->t_flags & TF_NEEDSYN) {
9896 return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen,
9897 tiwin, thflags, nxt_pkt));
9898 } else if (tp->t_flags & TF_ACKNOW) {
9899 ctf_do_dropafterack(m, tp, th, thflags, tlen, &ret_val);
9900 bbr->r_wanted_output = 1;
9901 return (ret_val);
9902 } else {
9903 ctf_do_drop(m, NULL);
9904 return (0);
9905 }
9906 }
9907 /*
9908 * Ack processing.
9909 */
9910 INP_WLOCK_ASSERT(tp->t_inpcb);
9911 if (bbr_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val)) {
9912 return (ret_val);
9913 }
9914 if (sbavail(&so->so_snd)) {
9915 if (ctf_progress_timeout_check(tp, true)) {
9916 bbr_log_progress_event(bbr, tp, tick, PROGRESS_DROP, __LINE__);
9917 ctf_do_dropwithreset_conn(m, tp, th, BANDLIM_RST_OPENPORT, tlen);
9918 return (1);
9919 }
9920 }
9921 INP_WLOCK_ASSERT(tp->t_inpcb);
9922 return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen,
9923 tiwin, thflags, nxt_pkt));
9924}
9925
9926static void
9927bbr_stop_all_timers(struct tcpcb *tp)
9928{
9929 struct tcp_bbr *bbr;
9930
9931 /*
9932 * Assure no timers are running.
9933 */
9934 if (tcp_timer_active(tp, TT_PERSIST)) {
9935 /* We enter in persists, set the flag appropriately */
9936 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
9937 bbr->rc_in_persist = 1;
9938 }
9939 tcp_timer_suspend(tp, TT_PERSIST);
9940 tcp_timer_suspend(tp, TT_REXMT);
9941 tcp_timer_suspend(tp, TT_KEEP);
9942 tcp_timer_suspend(tp, TT_DELACK);
9943}
9944
9945static void
9946bbr_google_mode_on(struct tcp_bbr *bbr)
9947{
9948 bbr->rc_use_google = 1;
9949 bbr->rc_no_pacing = 0;
9950 bbr->r_ctl.bbr_google_discount = bbr_google_discount;
9951 bbr->r_use_policer = bbr_policer_detection_enabled;
9952 bbr->r_ctl.rc_probertt_int = (USECS_IN_SECOND * 10);
9953 bbr->bbr_use_rack_cheat = 0;
9954 bbr->r_ctl.rc_incr_tmrs = 0;
9955 bbr->r_ctl.rc_inc_tcp_oh = 0;
9956 bbr->r_ctl.rc_inc_ip_oh = 0;
9957 bbr->r_ctl.rc_inc_enet_oh = 0;
9958 reset_time(&bbr->r_ctl.rc_delrate,
9959 BBR_NUM_RTTS_FOR_GOOG_DEL_LIMIT);
9960 reset_time_small(&bbr->r_ctl.rc_rttprop,
9961 (11 * USECS_IN_SECOND));
9962 tcp_bbr_tso_size_check(bbr, tcp_get_usecs(&bbr->rc_tv));
9963}
9964
9965static void
9966bbr_google_mode_off(struct tcp_bbr *bbr)
9967{
9968 bbr->rc_use_google = 0;
9969 bbr->r_ctl.bbr_google_discount = 0;
9970 bbr->no_pacing_until = bbr_no_pacing_until;
9971 bbr->r_use_policer = 0;
9972 if (bbr->no_pacing_until)
9973 bbr->rc_no_pacing = 1;
9974 else
9975 bbr->rc_no_pacing = 0;
9976 if (bbr_use_rack_resend_cheat)
9977 bbr->bbr_use_rack_cheat = 1;
9978 else
9979 bbr->bbr_use_rack_cheat = 0;
9980 if (bbr_incr_timers)
9981 bbr->r_ctl.rc_incr_tmrs = 1;
9982 else
9983 bbr->r_ctl.rc_incr_tmrs = 0;
9984 if (bbr_include_tcp_oh)
9985 bbr->r_ctl.rc_inc_tcp_oh = 1;
9986 else
9987 bbr->r_ctl.rc_inc_tcp_oh = 0;
9988 if (bbr_include_ip_oh)
9989 bbr->r_ctl.rc_inc_ip_oh = 1;
9990 else
9991 bbr->r_ctl.rc_inc_ip_oh = 0;
9992 if (bbr_include_enet_oh)
9993 bbr->r_ctl.rc_inc_enet_oh = 1;
9994 else
9995 bbr->r_ctl.rc_inc_enet_oh = 0;
9996 bbr->r_ctl.rc_probertt_int = bbr_rtt_probe_limit;
9997 reset_time(&bbr->r_ctl.rc_delrate,
9998 bbr_num_pktepo_for_del_limit);
9999 reset_time_small(&bbr->r_ctl.rc_rttprop,
10000 (bbr_filter_len_sec * USECS_IN_SECOND));
10001 tcp_bbr_tso_size_check(bbr, tcp_get_usecs(&bbr->rc_tv));
10002}
10003/*
10004 * Return 0 on success, non-zero on failure
10005 * which indicates the error (usually no memory).
10006 */
10007static int
10008bbr_init(struct tcpcb *tp)
10009{
10010 struct tcp_bbr *bbr = NULL;
10011 struct inpcb *inp;
10012 uint32_t cts;
10013
10014 tp->t_fb_ptr = uma_zalloc(bbr_pcb_zone, (M_NOWAIT | M_ZERO));
10015 if (tp->t_fb_ptr == NULL) {
10016 /*
10017 * We need to allocate memory but cant. The INP and INP_INFO
10018 * locks and they are recusive (happens during setup. So a
10019 * scheme to drop the locks fails :(
10020 *
10021 */
10022 return (ENOMEM);
10023 }
10024 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
10025 bbr->rtt_valid = 0;
10026 inp = tp->t_inpcb;
10027 inp->inp_flags2 |= INP_CANNOT_DO_ECN;
10028 inp->inp_flags2 |= INP_SUPPORTS_MBUFQ;
10029 TAILQ_INIT(&bbr->r_ctl.rc_map);
10030 TAILQ_INIT(&bbr->r_ctl.rc_free);
10031 TAILQ_INIT(&bbr->r_ctl.rc_tmap);
10032 bbr->rc_tp = tp;
10033 if (tp->t_inpcb) {
10034 bbr->rc_inp = tp->t_inpcb;
10035 }
10036 cts = tcp_get_usecs(&bbr->rc_tv);
10037 tp->t_acktime = 0;
10038 bbr->rc_allow_data_af_clo = bbr_ignore_data_after_close;
10039 bbr->r_ctl.rc_reorder_fade = bbr_reorder_fade;
10040 bbr->rc_tlp_threshold = bbr_tlp_thresh;
10041 bbr->r_ctl.rc_reorder_shift = bbr_reorder_thresh;
10042 bbr->r_ctl.rc_pkt_delay = bbr_pkt_delay;
10043 bbr->r_ctl.rc_min_to = bbr_min_to;
10044 bbr->rc_bbr_state = BBR_STATE_STARTUP;
10045 bbr->r_ctl.bbr_lost_at_state = 0;
10046 bbr->r_ctl.rc_lost_at_startup = 0;
10047 bbr->rc_all_timers_stopped = 0;
10048 bbr->r_ctl.rc_bbr_lastbtlbw = 0;
10049 bbr->r_ctl.rc_pkt_epoch_del = 0;
10050 bbr->r_ctl.rc_pkt_epoch = 0;
10051 bbr->r_ctl.rc_lowest_rtt = 0xffffffff;
10052 bbr->r_ctl.rc_bbr_hptsi_gain = bbr_high_gain;
10053 bbr->r_ctl.rc_bbr_cwnd_gain = bbr_high_gain;
10054 bbr->r_ctl.rc_went_idle_time = cts;
10055 bbr->rc_pacer_started = cts;
10056 bbr->r_ctl.rc_pkt_epoch_time = cts;
10057 bbr->r_ctl.rc_rcvtime = cts;
10058 bbr->r_ctl.rc_bbr_state_time = cts;
10059 bbr->r_ctl.rc_del_time = cts;
10060 bbr->r_ctl.rc_tlp_rxt_last_time = cts;
10061 bbr->r_ctl.last_in_probertt = cts;
10062 bbr->skip_gain = 0;
10063 bbr->gain_is_limited = 0;
10064 bbr->no_pacing_until = bbr_no_pacing_until;
10065 if (bbr->no_pacing_until)
10066 bbr->rc_no_pacing = 1;
10067 if (bbr_use_google_algo) {
10068 bbr->rc_no_pacing = 0;
10069 bbr->rc_use_google = 1;
10070 bbr->r_ctl.bbr_google_discount = bbr_google_discount;
10071 bbr->r_use_policer = bbr_policer_detection_enabled;
10072 } else {
10073 bbr->rc_use_google = 0;
10074 bbr->r_ctl.bbr_google_discount = 0;
10075 bbr->r_use_policer = 0;
10076 }
10077 if (bbr_ts_limiting)
10078 bbr->rc_use_ts_limit = 1;
10079 else
10080 bbr->rc_use_ts_limit = 0;
10081 if (bbr_ts_can_raise)
10082 bbr->ts_can_raise = 1;
10083 else
10084 bbr->ts_can_raise = 0;
10085 if (V_tcp_delack_enabled == 1)
10086 tp->t_delayed_ack = 2;
10087 else if (V_tcp_delack_enabled == 0)
10088 tp->t_delayed_ack = 0;
10089 else if (V_tcp_delack_enabled < 100)
10090 tp->t_delayed_ack = V_tcp_delack_enabled;
10091 else
10092 tp->t_delayed_ack = 2;
10093 if (bbr->rc_use_google == 0)
10094 bbr->r_ctl.rc_probertt_int = bbr_rtt_probe_limit;
10095 else
10096 bbr->r_ctl.rc_probertt_int = (USECS_IN_SECOND * 10);
10097 bbr->r_ctl.rc_min_rto_ms = bbr_rto_min_ms;
10098 bbr->rc_max_rto_sec = bbr_rto_max_sec;
10099 bbr->rc_init_win = bbr_def_init_win;
10100 if (tp->t_flags & TF_REQ_TSTMP)
10101 bbr->rc_last_options = TCP_TS_OVERHEAD;
10102 bbr->r_ctl.rc_pace_max_segs = tp->t_maxseg - bbr->rc_last_options;
10103 bbr->r_ctl.rc_high_rwnd = tp->snd_wnd;
10104 bbr->r_init_rtt = 1;
10105
10106 counter_u64_add(bbr_flows_nohdwr_pacing, 1);
10107 if (bbr_allow_hdwr_pacing)
10108 bbr->bbr_hdw_pace_ena = 1;
10109 else
10110 bbr->bbr_hdw_pace_ena = 0;
10111 if (bbr_sends_full_iwnd)
10112 bbr->bbr_init_win_cheat = 1;
10113 else
10114 bbr->bbr_init_win_cheat = 0;
10115 bbr->r_ctl.bbr_utter_max = bbr_hptsi_utter_max;
10116 bbr->r_ctl.rc_drain_pg = bbr_drain_gain;
10117 bbr->r_ctl.rc_startup_pg = bbr_high_gain;
10118 bbr->rc_loss_exit = bbr_exit_startup_at_loss;
10119 bbr->r_ctl.bbr_rttprobe_gain_val = bbr_rttprobe_gain;
10120 bbr->r_ctl.bbr_hptsi_per_second = bbr_hptsi_per_second;
10121 bbr->r_ctl.bbr_hptsi_segments_delay_tar = bbr_hptsi_segments_delay_tar;
10122 bbr->r_ctl.bbr_hptsi_segments_max = bbr_hptsi_segments_max;
10123 bbr->r_ctl.bbr_hptsi_segments_floor = bbr_hptsi_segments_floor;
10124 bbr->r_ctl.bbr_hptsi_bytes_min = bbr_hptsi_bytes_min;
10125 bbr->r_ctl.bbr_cross_over = bbr_cross_over;
10126 bbr->r_ctl.rc_rtt_shrinks = cts;
10127 if (bbr->rc_use_google) {
10128 setup_time_filter(&bbr->r_ctl.rc_delrate,
10129 FILTER_TYPE_MAX,
10130 BBR_NUM_RTTS_FOR_GOOG_DEL_LIMIT);
10131 setup_time_filter_small(&bbr->r_ctl.rc_rttprop,
10132 FILTER_TYPE_MIN, (11 * USECS_IN_SECOND));
10133 } else {
10134 setup_time_filter(&bbr->r_ctl.rc_delrate,
10135 FILTER_TYPE_MAX,
10136 bbr_num_pktepo_for_del_limit);
10137 setup_time_filter_small(&bbr->r_ctl.rc_rttprop,
10138 FILTER_TYPE_MIN, (bbr_filter_len_sec * USECS_IN_SECOND));
10139 }
10140 bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_INIT, 0);
10141 if (bbr_uses_idle_restart)
10142 bbr->rc_use_idle_restart = 1;
10143 else
10144 bbr->rc_use_idle_restart = 0;
10145 bbr->r_ctl.rc_bbr_cur_del_rate = 0;
10146 bbr->r_ctl.rc_initial_hptsi_bw = bbr_initial_bw_bps;
10147 if (bbr_resends_use_tso)
10148 bbr->rc_resends_use_tso = 1;
10149#ifdef NETFLIX_PEAKRATE
10150 tp->t_peakrate_thr = tp->t_maxpeakrate;
10151#endif
10152 if (tp->snd_una != tp->snd_max) {
10153 /* Create a send map for the current outstanding data */
10154 struct bbr_sendmap *rsm;
10155
10156 rsm = bbr_alloc(bbr);
10157 if (rsm == NULL) {
10158 uma_zfree(bbr_pcb_zone, tp->t_fb_ptr);
10159 tp->t_fb_ptr = NULL;
10160 return (ENOMEM);
10161 }
10162 rsm->r_rtt_not_allowed = 1;
10163 rsm->r_tim_lastsent[0] = cts;
10164 rsm->r_rtr_cnt = 1;
10165 rsm->r_rtr_bytes = 0;
10166 rsm->r_start = tp->snd_una;
10167 rsm->r_end = tp->snd_max;
10168 rsm->r_dupack = 0;
10169 rsm->r_delivered = bbr->r_ctl.rc_delivered;
10170 rsm->r_ts_valid = 0;
10171 rsm->r_del_ack_ts = tp->ts_recent;
10172 rsm->r_del_time = cts;
10173 if (bbr->r_ctl.r_app_limited_until)
10174 rsm->r_app_limited = 1;
10175 else
10176 rsm->r_app_limited = 0;
10177 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_map, rsm, r_next);
10178 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
10179 rsm->r_in_tmap = 1;
10180 if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW)
10181 rsm->r_bbr_state = bbr_state_val(bbr);
10182 else
10183 rsm->r_bbr_state = 8;
10184 }
10185 if (bbr_use_rack_resend_cheat && (bbr->rc_use_google == 0))
10186 bbr->bbr_use_rack_cheat = 1;
10187 if (bbr_incr_timers && (bbr->rc_use_google == 0))
10188 bbr->r_ctl.rc_incr_tmrs = 1;
10189 if (bbr_include_tcp_oh && (bbr->rc_use_google == 0))
10190 bbr->r_ctl.rc_inc_tcp_oh = 1;
10191 if (bbr_include_ip_oh && (bbr->rc_use_google == 0))
10192 bbr->r_ctl.rc_inc_ip_oh = 1;
10193 if (bbr_include_enet_oh && (bbr->rc_use_google == 0))
10194 bbr->r_ctl.rc_inc_enet_oh = 1;
10195
10196 bbr_log_type_statechange(bbr, cts, __LINE__);
10197 if (TCPS_HAVEESTABLISHED(tp->t_state) &&
10198 (tp->t_srtt)) {
10199 uint32_t rtt;
10200
10201 rtt = (TICKS_2_USEC(tp->t_srtt) >> TCP_RTT_SHIFT);
10202 apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, cts);
10203 }
10204 /* announce the settings and state */
10205 bbr_log_settings_change(bbr, BBR_RECOVERY_LOWRTT);
10206 tcp_bbr_tso_size_check(bbr, cts);
10207 /*
10208 * Now call the generic function to start a timer. This will place
10209 * the TCB on the hptsi wheel if a timer is needed with appropriate
10210 * flags.
10211 */
10212 bbr_stop_all_timers(tp);
10213 bbr_start_hpts_timer(bbr, tp, cts, 5, 0, 0);
10214 return (0);
10215}
10216
10217/*
10218 * Return 0 if we can accept the connection. Return
10219 * non-zero if we can't handle the connection. A EAGAIN
10220 * means you need to wait until the connection is up.
10221 * a EADDRNOTAVAIL means we can never handle the connection
10222 * (no SACK).
10223 */
10224static int
10225bbr_handoff_ok(struct tcpcb *tp)
10226{
10227 if ((tp->t_state == TCPS_CLOSED) ||
10228 (tp->t_state == TCPS_LISTEN)) {
10229 /* Sure no problem though it may not stick */
10230 return (0);
10231 }
10232 if ((tp->t_state == TCPS_SYN_SENT) ||
10233 (tp->t_state == TCPS_SYN_RECEIVED)) {
10234 /*
10235 * We really don't know you have to get to ESTAB or beyond
10236 * to tell.
10237 */
10238 return (EAGAIN);
10239 }
10240 if (tp->t_flags & TF_SENTFIN)
10241 return (EINVAL);
10242 if ((tp->t_flags & TF_SACK_PERMIT) || bbr_sack_not_required) {
10243 return (0);
10244 }
10245 /*
10246 * If we reach here we don't do SACK on this connection so we can
10247 * never do rack.
10248 */
10249 return (EINVAL);
10250}
10251
10252static void
10253bbr_fini(struct tcpcb *tp, int32_t tcb_is_purged)
10254{
10255 if (tp->t_fb_ptr) {
10256 uint32_t calc;
10257 struct tcp_bbr *bbr;
10258 struct bbr_sendmap *rsm;
10259
10260 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
10261 if (bbr->r_ctl.crte)
10262 tcp_rel_pacing_rate(bbr->r_ctl.crte, bbr->rc_tp);
10263 bbr_log_flowend(bbr);
10264 bbr->rc_tp = NULL;
10265 if (tp->t_inpcb) {
10266 /* Backout any flags2 we applied */
10267 tp->t_inpcb->inp_flags2 &= ~INP_CANNOT_DO_ECN;
10268 tp->t_inpcb->inp_flags2 &= ~INP_SUPPORTS_MBUFQ;
10269 tp->t_inpcb->inp_flags2 &= ~INP_MBUF_QUEUE_READY;
10270 }
10271 if (bbr->bbr_hdrw_pacing)
10272 counter_u64_add(bbr_flows_whdwr_pacing, -1);
10273 else
10274 counter_u64_add(bbr_flows_nohdwr_pacing, -1);
10275 if (bbr->r_ctl.crte != NULL) {
10276 tcp_rel_pacing_rate(bbr->r_ctl.crte, tp);
10277 bbr->r_ctl.crte = NULL;
10278 }
10279 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
10280 while (rsm) {
10281 TAILQ_REMOVE(&bbr->r_ctl.rc_map, rsm, r_next);
10282 uma_zfree(bbr_zone, rsm);
10283 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
10284 }
10285 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_free);
10286 while (rsm) {
10287 TAILQ_REMOVE(&bbr->r_ctl.rc_free, rsm, r_next);
10288 uma_zfree(bbr_zone, rsm);
10289 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_free);
10290 }
10291 calc = bbr->r_ctl.rc_high_rwnd - bbr->r_ctl.rc_init_rwnd;
10292 if (calc > (bbr->r_ctl.rc_init_rwnd / 10))
10293 BBR_STAT_INC(bbr_dynamic_rwnd);
10294 else
10295 BBR_STAT_INC(bbr_static_rwnd);
10296 bbr->r_ctl.rc_free_cnt = 0;
10297 uma_zfree(bbr_pcb_zone, tp->t_fb_ptr);
10298 tp->t_fb_ptr = NULL;
10299 }
10300 /* Make sure snd_nxt is correctly set */
10301 tp->snd_nxt = tp->snd_max;
10302}
10303
10304static void
10305bbr_set_state(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t win)
10306{
10307 switch (tp->t_state) {
10308 case TCPS_SYN_SENT:
10309 bbr->r_state = TCPS_SYN_SENT;
10310 bbr->r_substate = bbr_do_syn_sent;
10311 break;
10312 case TCPS_SYN_RECEIVED:
10313 bbr->r_state = TCPS_SYN_RECEIVED;
10314 bbr->r_substate = bbr_do_syn_recv;
10315 break;
10316 case TCPS_ESTABLISHED:
10317 bbr->r_ctl.rc_init_rwnd = max(win, bbr->rc_tp->snd_wnd);
10318 bbr->r_state = TCPS_ESTABLISHED;
10319 bbr->r_substate = bbr_do_established;
10320 break;
10321 case TCPS_CLOSE_WAIT:
10322 bbr->r_state = TCPS_CLOSE_WAIT;
10323 bbr->r_substate = bbr_do_close_wait;
10324 break;
10325 case TCPS_FIN_WAIT_1:
10326 bbr->r_state = TCPS_FIN_WAIT_1;
10327 bbr->r_substate = bbr_do_fin_wait_1;
10328 break;
10329 case TCPS_CLOSING:
10330 bbr->r_state = TCPS_CLOSING;
10331 bbr->r_substate = bbr_do_closing;
10332 break;
10333 case TCPS_LAST_ACK:
10334 bbr->r_state = TCPS_LAST_ACK;
10335 bbr->r_substate = bbr_do_lastack;
10336 break;
10337 case TCPS_FIN_WAIT_2:
10338 bbr->r_state = TCPS_FIN_WAIT_2;
10339 bbr->r_substate = bbr_do_fin_wait_2;
10340 break;
10341 case TCPS_LISTEN:
10342 case TCPS_CLOSED:
10343 case TCPS_TIME_WAIT:
10344 default:
10345 break;
10346 };
10347}
10348
10349static void
10350bbr_substate_change(struct tcp_bbr *bbr, uint32_t cts, int32_t line, int dolog)
10351{
10352 /*
10353 * Now what state are we going into now? Is there adjustments
10354 * needed?
10355 */
10356 int32_t old_state;
10357
10358 old_state = bbr_state_val(bbr);
10359 if (bbr_state_val(bbr) == BBR_SUB_LEVEL1) {
10360 /* Save the lowest srtt we saw in our end of the sub-state */
10361 bbr->rc_hit_state_1 = 0;
10362 if (bbr->r_ctl.bbr_smallest_srtt_this_state != 0xffffffff)
10363 bbr->r_ctl.bbr_smallest_srtt_state2 = bbr->r_ctl.bbr_smallest_srtt_this_state;
10364 }
10365 bbr->rc_bbr_substate++;
10366 if (bbr->rc_bbr_substate >= BBR_SUBSTATE_COUNT) {
10367 /* Cycle back to first state-> gain */
10368 bbr->rc_bbr_substate = 0;
10369 }
10370 if (bbr_state_val(bbr) == BBR_SUB_GAIN) {
10371 /*
10372 * We enter the gain(5/4) cycle (possibly less if
10373 * shallow buffer detection is enabled)
10374 */
10375 if (bbr->skip_gain) {
10376 /*
10377 * Hardware pacing has set our rate to
10378 * the max and limited our b/w just
10379 * do level i.e. no gain.
10380 */
10381 bbr->r_ctl.rc_bbr_hptsi_gain = bbr_hptsi_gain[BBR_SUB_LEVEL1];
10382 } else if (bbr->gain_is_limited &&
10383 bbr->bbr_hdrw_pacing &&
10384 bbr->r_ctl.crte) {
10385 /*
10386 * We can't gain above the hardware pacing
10387 * rate which is less than our rate + the gain
10388 * calculate the gain needed to reach the hardware
10389 * pacing rate..
10390 */
10391 uint64_t bw, rate, gain_calc;
10392
10393 bw = bbr_get_bw(bbr);
10394 rate = bbr->r_ctl.crte->rate;
10395 if ((rate > bw) &&
10396 (((bw * (uint64_t)bbr_hptsi_gain[BBR_SUB_GAIN]) / (uint64_t)BBR_UNIT) > rate)) {
10397 gain_calc = (rate * BBR_UNIT) / bw;
10398 if (gain_calc < BBR_UNIT)
10399 gain_calc = BBR_UNIT;
10400 bbr->r_ctl.rc_bbr_hptsi_gain = (uint16_t)gain_calc;
10401 } else {
10402 bbr->r_ctl.rc_bbr_hptsi_gain = bbr_hptsi_gain[BBR_SUB_GAIN];
10403 }
10404 } else
10405 bbr->r_ctl.rc_bbr_hptsi_gain = bbr_hptsi_gain[BBR_SUB_GAIN];
10406 if ((bbr->rc_use_google == 0) && (bbr_gain_to_target == 0)) {
10407 bbr->r_ctl.rc_bbr_state_atflight = cts;
10408 } else
10409 bbr->r_ctl.rc_bbr_state_atflight = 0;
10410 } else if (bbr_state_val(bbr) == BBR_SUB_DRAIN) {
10411 bbr->rc_hit_state_1 = 1;
10412 bbr->r_ctl.rc_exta_time_gd = 0;
10413 bbr->r_ctl.flightsize_at_drain = ctf_flight_size(bbr->rc_tp,
10414 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
10415 if (bbr_state_drain_2_tar) {
10416 bbr->r_ctl.rc_bbr_state_atflight = 0;
10417 } else
10418 bbr->r_ctl.rc_bbr_state_atflight = cts;
10419 bbr->r_ctl.rc_bbr_hptsi_gain = bbr_hptsi_gain[BBR_SUB_DRAIN];
10420 } else {
10421 /* All other cycles hit here 2-7 */
10422 if ((old_state == BBR_SUB_DRAIN) && bbr->rc_hit_state_1) {
10423 if (bbr_sub_drain_slam_cwnd &&
10424 (bbr->rc_use_google == 0) &&
10425 (bbr->rc_tp->snd_cwnd < bbr->r_ctl.rc_saved_cwnd)) {
10426 bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_saved_cwnd;
10427 bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__);
10428 }
10429 if ((cts - bbr->r_ctl.rc_bbr_state_time) > bbr_get_rtt(bbr, BBR_RTT_PROP))
10430 bbr->r_ctl.rc_exta_time_gd += ((cts - bbr->r_ctl.rc_bbr_state_time) -
10431 bbr_get_rtt(bbr, BBR_RTT_PROP));
10432 else
10433 bbr->r_ctl.rc_exta_time_gd = 0;
10434 if (bbr->r_ctl.rc_exta_time_gd) {
10435 bbr->r_ctl.rc_level_state_extra = bbr->r_ctl.rc_exta_time_gd;
10436 /* Now chop up the time for each state (div by 7) */
10437 bbr->r_ctl.rc_level_state_extra /= 7;
10438 if (bbr_rand_ot && bbr->r_ctl.rc_level_state_extra) {
10439 /* Add a randomization */
10440 bbr_randomize_extra_state_time(bbr);
10441 }
10442 }
10443 }
10444 bbr->r_ctl.rc_bbr_state_atflight = max(1, cts);
10445 bbr->r_ctl.rc_bbr_hptsi_gain = bbr_hptsi_gain[bbr_state_val(bbr)];
10446 }
10447 if (bbr->rc_use_google) {
10448 bbr->r_ctl.rc_bbr_state_atflight = max(1, cts);
10449 }
10450 bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost;
10451 bbr->r_ctl.rc_bbr_cwnd_gain = bbr_cwnd_gain;
10452 if (dolog)
10453 bbr_log_type_statechange(bbr, cts, line);
10454
10455 if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) {
10456 uint32_t time_in;
10457
10458 time_in = cts - bbr->r_ctl.rc_bbr_state_time;
10459 if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) {
10460 counter_u64_add(bbr_state_time[(old_state + 5)], time_in);
10461 } else {
10462 counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in);
10463 }
10464 }
10465 bbr->r_ctl.bbr_smallest_srtt_this_state = 0xffffffff;
10466 bbr_set_state_target(bbr, __LINE__);
10467 if (bbr_sub_drain_slam_cwnd &&
10468 (bbr->rc_use_google == 0) &&
10469 (bbr_state_val(bbr) == BBR_SUB_DRAIN)) {
10470 /* Slam down the cwnd */
10471 bbr->r_ctl.rc_saved_cwnd = bbr->rc_tp->snd_cwnd;
10472 bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state;
10473 if (bbr_sub_drain_app_limit) {
10474 /* Go app limited if we are on a long drain */
10475 bbr->r_ctl.r_app_limited_until = (bbr->r_ctl.rc_delivered +
10476 ctf_flight_size(bbr->rc_tp,
10477 (bbr->r_ctl.rc_sacked +
10478 bbr->r_ctl.rc_lost_bytes)));
10479 }
10480 bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__);
10481 }
10482 if (bbr->rc_lt_use_bw) {
10483 /* In policed mode we clamp pacing_gain to BBR_UNIT */
10484 bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT;
10485 }
10486 /* Google changes TSO size every cycle */
10487 if (bbr->rc_use_google)
10488 tcp_bbr_tso_size_check(bbr, cts);
10489 bbr->r_ctl.gain_epoch = cts;
10490 bbr->r_ctl.rc_bbr_state_time = cts;
10491 bbr->r_ctl.substate_pe = bbr->r_ctl.rc_pkt_epoch;
10492}
10493
10494static void
10495bbr_set_probebw_google_gains(struct tcp_bbr *bbr, uint32_t cts, uint32_t losses)
10496{
10497 if ((bbr_state_val(bbr) == BBR_SUB_DRAIN) &&
10498 (google_allow_early_out == 1) &&
10499 (bbr->r_ctl.rc_flight_at_input <= bbr->r_ctl.rc_target_at_state)) {
10500 /* We have reached out target flight size possibly early */
10501 goto change_state;
10502 }
10503 if (TSTMP_LT(cts, bbr->r_ctl.rc_bbr_state_time)) {
10504 return;
10505 }
10506 if ((cts - bbr->r_ctl.rc_bbr_state_time) < bbr_get_rtt(bbr, BBR_RTT_PROP)) {
10507 /*
10508 * Must be a rttProp movement forward before
10509 * we can change states.
10510 */
10511 return;
10512 }
10513 if (bbr_state_val(bbr) == BBR_SUB_GAIN) {
10514 /*
10515 * The needed time has passed but for
10516 * the gain cycle extra rules apply:
10517 * 1) If we have seen loss, we exit
10518 * 2) If we have not reached the target
10519 * we stay in GAIN (gain-to-target).
10520 */
10521 if (google_consider_lost && losses)
10522 goto change_state;
10523 if (bbr->r_ctl.rc_target_at_state > bbr->r_ctl.rc_flight_at_input) {
10524 return;
10525 }
10526 }
10527change_state:
10528 /* For gain we must reach our target, all others last 1 rttProp */
10529 bbr_substate_change(bbr, cts, __LINE__, 1);
10530}
10531
10532static void
10533bbr_set_probebw_gains(struct tcp_bbr *bbr, uint32_t cts, uint32_t losses)
10534{
10535 uint32_t flight, bbr_cur_cycle_time;
10536
10537 if (bbr->rc_use_google) {
10538 bbr_set_probebw_google_gains(bbr, cts, losses);
10539 return;
10540 }
10541 if (cts == 0) {
10542 /*
10543 * Never alow cts to be 0 we
10544 * do this so we can judge if
10545 * we have set a timestamp.
10546 */
10547 cts = 1;
10548 }
10549 if (bbr_state_is_pkt_epoch)
10550 bbr_cur_cycle_time = bbr_get_rtt(bbr, BBR_RTT_PKTRTT);
10551 else
10552 bbr_cur_cycle_time = bbr_get_rtt(bbr, BBR_RTT_PROP);
10553
10554 if (bbr->r_ctl.rc_bbr_state_atflight == 0) {
10555 if (bbr_state_val(bbr) == BBR_SUB_DRAIN) {
10556 flight = ctf_flight_size(bbr->rc_tp,
10557 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
10558 if (bbr_sub_drain_slam_cwnd && bbr->rc_hit_state_1) {
10559 /* Keep it slam down */
10560 if (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_target_at_state) {
10561 bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state;
10562 bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__);
10563 }
10564 if (bbr_sub_drain_app_limit) {
10565 /* Go app limited if we are on a long drain */
10566 bbr->r_ctl.r_app_limited_until = (bbr->r_ctl.rc_delivered + flight);
10567 }
10568 }
10569 if (TSTMP_GT(cts, bbr->r_ctl.gain_epoch) &&
10570 (((cts - bbr->r_ctl.gain_epoch) > bbr_get_rtt(bbr, BBR_RTT_PROP)) ||
10571 (flight >= bbr->r_ctl.flightsize_at_drain))) {
10572 /*
10573 * Still here after the same time as
10574 * the gain. We need to drain harder
10575 * for the next srtt. Reduce by a set amount
10576 * the gain drop is capped at DRAIN states
10577 * value (88).
10578 */
10579 bbr->r_ctl.flightsize_at_drain = flight;
10580 if (bbr_drain_drop_mul &&
10581 bbr_drain_drop_div &&
10582 (bbr_drain_drop_mul < bbr_drain_drop_div)) {
10583 /* Use your specific drop value (def 4/5 = 20%) */
10584 bbr->r_ctl.rc_bbr_hptsi_gain *= bbr_drain_drop_mul;
10585 bbr->r_ctl.rc_bbr_hptsi_gain /= bbr_drain_drop_div;
10586 } else {
10587 /* You get drop of 20% */
10588 bbr->r_ctl.rc_bbr_hptsi_gain *= 4;
10589 bbr->r_ctl.rc_bbr_hptsi_gain /= 5;
10590 }
10591 if (bbr->r_ctl.rc_bbr_hptsi_gain <= bbr_drain_floor) {
10592 /* Reduce our gain again to the bottom */
10593 bbr->r_ctl.rc_bbr_hptsi_gain = max(bbr_drain_floor, 1);
10594 }
10595 bbr_log_exit_gain(bbr, cts, 4);
10596 /*
10597 * Extend out so we wait another
10598 * epoch before dropping again.
10599 */
10600 bbr->r_ctl.gain_epoch = cts;
10601 }
10602 if (flight <= bbr->r_ctl.rc_target_at_state) {
10603 if (bbr_sub_drain_slam_cwnd &&
10604 (bbr->rc_use_google == 0) &&
10605 (bbr->rc_tp->snd_cwnd < bbr->r_ctl.rc_saved_cwnd)) {
10606 bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_saved_cwnd;
10607 bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__);
10608 }
10609 bbr->r_ctl.rc_bbr_state_atflight = max(cts, 1);
10610 bbr_log_exit_gain(bbr, cts, 3);
10611 }
10612 } else {
10613 /* Its a gain */
10614 if (bbr->r_ctl.rc_lost > bbr->r_ctl.bbr_lost_at_state) {
10615 bbr->r_ctl.rc_bbr_state_atflight = max(cts, 1);
10616 goto change_state;
10617 }
10618 if ((ctf_outstanding(bbr->rc_tp) >= bbr->r_ctl.rc_target_at_state) ||
10619 ((ctf_outstanding(bbr->rc_tp) + bbr->rc_tp->t_maxseg - 1) >=
10620 bbr->rc_tp->snd_wnd)) {
10621 bbr->r_ctl.rc_bbr_state_atflight = max(cts, 1);
10622 bbr_log_exit_gain(bbr, cts, 2);
10623 }
10624 }
10634 return;
10635 }
10636change_state:
10637 if (TSTMP_LT(cts, bbr->r_ctl.rc_bbr_state_time))
10638 return;
10639 if ((cts - bbr->r_ctl.rc_bbr_state_time) < bbr_cur_cycle_time) {
10640 /* Less than a full time-period has passed */
10641 return;
10642 }
10643 if (bbr->r_ctl.rc_level_state_extra &&
10644 (bbr_state_val(bbr) > BBR_SUB_DRAIN) &&
10645 ((cts - bbr->r_ctl.rc_bbr_state_time) <
10646 (bbr_cur_cycle_time + bbr->r_ctl.rc_level_state_extra))) {
10647 /* Less than a full time-period + extra has passed */
10648 return;
10649 }
10650 if (bbr_gain_gets_extra_too &&
10651 bbr->r_ctl.rc_level_state_extra &&
10652 (bbr_state_val(bbr) == BBR_SUB_GAIN) &&
10653 ((cts - bbr->r_ctl.rc_bbr_state_time) <
10654 (bbr_cur_cycle_time + bbr->r_ctl.rc_level_state_extra))) {
10655 /* Less than a full time-period + extra has passed */
10656 return;
10657 }
10658 bbr_substate_change(bbr, cts, __LINE__, 1);
10659}
10660
10661static uint32_t
10662bbr_get_a_state_target(struct tcp_bbr *bbr, uint32_t gain)
10663{
10664 uint32_t mss, tar;
10665
10666 if (bbr->rc_use_google) {
10667 /* Google just uses the cwnd target */
10668 tar = bbr_get_target_cwnd(bbr, bbr_get_bw(bbr), gain);
10669 } else {
10670 mss = min((bbr->rc_tp->t_maxseg - bbr->rc_last_options),
10671 bbr->r_ctl.rc_pace_max_segs);
10672 /* Get the base cwnd with gain rounded to a mss */
10673 tar = roundup(bbr_get_raw_target_cwnd(bbr, bbr_get_bw(bbr),
10674 gain), mss);
10675 /* Make sure it is within our min */
10676 if (tar < get_min_cwnd(bbr))
10677 return (get_min_cwnd(bbr));
10678 }
10679 return (tar);
10680}
10681
10682static void
10683bbr_set_state_target(struct tcp_bbr *bbr, int line)
10684{
10685 uint32_t tar, meth;
10686
10687 if ((bbr->rc_bbr_state == BBR_STATE_PROBE_RTT) &&
10688 ((bbr->r_ctl.bbr_rttprobe_gain_val == 0) || bbr->rc_use_google)) {
10689 /* Special case using old probe-rtt method */
10690 tar = bbr_rtt_probe_cwndtarg * (bbr->rc_tp->t_maxseg - bbr->rc_last_options);
10691 meth = 1;
10692 } else {
10693 /* Non-probe-rtt case and reduced probe-rtt */
10694 if ((bbr->rc_bbr_state == BBR_STATE_PROBE_BW) &&
10695 (bbr->r_ctl.rc_bbr_hptsi_gain > BBR_UNIT)) {
10696 /* For gain cycle we use the hptsi gain */
10697 tar = bbr_get_a_state_target(bbr, bbr->r_ctl.rc_bbr_hptsi_gain);
10698 meth = 2;
10699 } else if ((bbr_target_is_bbunit) || bbr->rc_use_google) {
10700 /*
10701 * If configured, or for google all other states
10702 * get BBR_UNIT.
10703 */
10704 tar = bbr_get_a_state_target(bbr, BBR_UNIT);
10705 meth = 3;
10706 } else {
10707 /*
10708 * Or we set a target based on the pacing gain
10709 * for non-google mode and default (non-configured).
10710 * Note we don't set a target goal below drain (192).
10711 */
10712 if (bbr->r_ctl.rc_bbr_hptsi_gain < bbr_hptsi_gain[BBR_SUB_DRAIN]) {
10713 tar = bbr_get_a_state_target(bbr, bbr_hptsi_gain[BBR_SUB_DRAIN]);
10714 meth = 4;
10715 } else {
10716 tar = bbr_get_a_state_target(bbr, bbr->r_ctl.rc_bbr_hptsi_gain);
10717 meth = 5;
10718 }
10719 }
10720 }
10721 bbr_log_set_of_state_target(bbr, tar, line, meth);
10722 bbr->r_ctl.rc_target_at_state = tar;
10723}
10724
10725static void
10726bbr_enter_probe_rtt(struct tcp_bbr *bbr, uint32_t cts, int32_t line)
10727{
10728 /* Change to probe_rtt */
10729 uint32_t time_in;
10730
10731 bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost;
10732 bbr->r_ctl.flightsize_at_drain = ctf_flight_size(bbr->rc_tp,
10733 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
10734 bbr->r_ctl.r_app_limited_until = (bbr->r_ctl.flightsize_at_drain
10735 + bbr->r_ctl.rc_delivered);
10736 /* Setup so we force feed the filter */
10737 if (bbr->rc_use_google || bbr_probertt_sets_rtt)
10738 bbr->rc_prtt_set_ts = 1;
10739 if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) {
10740 time_in = cts - bbr->r_ctl.rc_bbr_state_time;
10741 counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in);
10742 }
10743 bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_ENTERPROBE, 0);
10744 bbr->r_ctl.rc_rtt_shrinks = cts;
10745 bbr->r_ctl.last_in_probertt = cts;
10746 bbr->r_ctl.rc_probertt_srttchktim = cts;
10747 bbr->r_ctl.rc_bbr_state_time = cts;
10748 bbr->rc_bbr_state = BBR_STATE_PROBE_RTT;
10749 /* We need to force the filter to update */
10750
10751 if ((bbr_sub_drain_slam_cwnd) &&
10752 bbr->rc_hit_state_1 &&
10753 (bbr->rc_use_google == 0) &&
10754 (bbr_state_val(bbr) == BBR_SUB_DRAIN)) {
10755 if (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_saved_cwnd)
10756 bbr->r_ctl.rc_saved_cwnd = bbr->rc_tp->snd_cwnd;
10757 } else
10758 bbr->r_ctl.rc_saved_cwnd = bbr->rc_tp->snd_cwnd;
10759 /* Update the lost */
10760 bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost;
10761 if ((bbr->r_ctl.bbr_rttprobe_gain_val == 0) || bbr->rc_use_google){
10762 /* Set to the non-configurable default of 4 (PROBE_RTT_MIN) */
10763 bbr->rc_tp->snd_cwnd = bbr_rtt_probe_cwndtarg * (bbr->rc_tp->t_maxseg - bbr->rc_last_options);
10764 bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__);
10765 bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT;
10766 bbr->r_ctl.rc_bbr_cwnd_gain = BBR_UNIT;
10767 bbr_log_set_of_state_target(bbr, bbr->rc_tp->snd_cwnd, __LINE__, 6);
10768 bbr->r_ctl.rc_target_at_state = bbr->rc_tp->snd_cwnd;
10769 } else {
10770 /*
10771 * We bring it down slowly by using a hptsi gain that is
10772 * probably 75%. This will slowly float down our outstanding
10773 * without tampering with the cwnd.
10774 */
10775 bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.bbr_rttprobe_gain_val;
10776 bbr->r_ctl.rc_bbr_cwnd_gain = BBR_UNIT;
10777 bbr_set_state_target(bbr, __LINE__);
10778 if (bbr_prtt_slam_cwnd &&
10779 (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_target_at_state)) {
10780 bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state;
10781 bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__);
10782 }
10783 }
10784 if (ctf_flight_size(bbr->rc_tp,
10785 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) <=
10786 bbr->r_ctl.rc_target_at_state) {
10787 /* We are at target */
10788 bbr->r_ctl.rc_bbr_enters_probertt = cts;
10789 } else {
10790 /* We need to come down to reach target before our time begins */
10791 bbr->r_ctl.rc_bbr_enters_probertt = 0;
10792 }
10793 bbr->r_ctl.rc_pe_of_prtt = bbr->r_ctl.rc_pkt_epoch;
10794 BBR_STAT_INC(bbr_enter_probertt);
10795 bbr_log_exit_gain(bbr, cts, 0);
10796 bbr_log_type_statechange(bbr, cts, line);
10797}
10798
10799static void
10800bbr_check_probe_rtt_limits(struct tcp_bbr *bbr, uint32_t cts)
10801{
10802 /*
10803 * Sanity check on probe-rtt intervals.
10804 * In crazy situations where we are competing
10805 * against new-reno flows with huge buffers
10806 * our rtt-prop interval could come to dominate
10807 * things if we can't get through a full set
10808 * of cycles, we need to adjust it.
10809 */
10810 if (bbr_can_adjust_probertt &&
10811 (bbr->rc_use_google == 0)) {
10812 uint16_t val = 0;
10813 uint32_t cur_rttp, fval, newval, baseval;
10814
10815 /* Are we to small and go into probe-rtt to often? */
10816 baseval = (bbr_get_rtt(bbr, BBR_RTT_PROP) * (BBR_SUBSTATE_COUNT + 1));
10817 cur_rttp = roundup(baseval, USECS_IN_SECOND);
10818 fval = bbr_filter_len_sec * USECS_IN_SECOND;
10819 if (bbr_is_ratio == 0) {
10820 if (fval > bbr_rtt_probe_limit)
10821 newval = cur_rttp + (fval - bbr_rtt_probe_limit);
10822 else
10823 newval = cur_rttp;
10824 } else {
10825 int mul;
10826
10827 mul = fval / bbr_rtt_probe_limit;
10828 newval = cur_rttp * mul;
10829 }
10830 if (cur_rttp > bbr->r_ctl.rc_probertt_int) {
10831 bbr->r_ctl.rc_probertt_int = cur_rttp;
10832 reset_time_small(&bbr->r_ctl.rc_rttprop, newval);
10833 val = 1;
10834 } else {
10835 /*
10836 * No adjustments were made
10837 * do we need to shrink it?
10838 */
10839 if (bbr->r_ctl.rc_probertt_int > bbr_rtt_probe_limit) {
10840 if (cur_rttp <= bbr_rtt_probe_limit) {
10841 /*
10842 * Things have calmed down lets
10843 * shrink all the way to default
10844 */
10845 bbr->r_ctl.rc_probertt_int = bbr_rtt_probe_limit;
10846 reset_time_small(&bbr->r_ctl.rc_rttprop,
10847 (bbr_filter_len_sec * USECS_IN_SECOND));
10848 cur_rttp = bbr_rtt_probe_limit;
10849 newval = (bbr_filter_len_sec * USECS_IN_SECOND);
10850 val = 2;
10851 } else {
10852 /*
10853 * Well does some adjustment make sense?
10854 */
10855 if (cur_rttp < bbr->r_ctl.rc_probertt_int) {
10856 /* We can reduce interval time some */
10857 bbr->r_ctl.rc_probertt_int = cur_rttp;
10858 reset_time_small(&bbr->r_ctl.rc_rttprop, newval);
10859 val = 3;
10860 }
10861 }
10862 }
10863 }
10864 if (val)
10865 bbr_log_rtt_shrinks(bbr, cts, cur_rttp, newval, __LINE__, BBR_RTTS_RESETS_VALUES, val);
10866 }
10867}
10868
10869static void
10870bbr_exit_probe_rtt(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts)
10871{
10872 /* Exit probe-rtt */
10873
10874 if (tp->snd_cwnd < bbr->r_ctl.rc_saved_cwnd) {
10875 tp->snd_cwnd = bbr->r_ctl.rc_saved_cwnd;
10876 bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__);
10877 }
10878 bbr_log_exit_gain(bbr, cts, 1);
10879 bbr->rc_hit_state_1 = 0;
10880 bbr->r_ctl.rc_rtt_shrinks = cts;
10881 bbr->r_ctl.last_in_probertt = cts;
10882 bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_RTTPROBE, 0);
10883 bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost;
10884 bbr->r_ctl.r_app_limited_until = (ctf_flight_size(tp,
10885 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) +
10886 bbr->r_ctl.rc_delivered);
10887 if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) {
10888 uint32_t time_in;
10889
10890 time_in = cts - bbr->r_ctl.rc_bbr_state_time;
10891 counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in);
10892 }
10893 if (bbr->rc_filled_pipe) {
10894 /* Switch to probe_bw */
10895 bbr->rc_bbr_state = BBR_STATE_PROBE_BW;
10896 bbr->rc_bbr_substate = bbr_pick_probebw_substate(bbr, cts);
10897 bbr->r_ctl.rc_bbr_cwnd_gain = bbr_cwnd_gain;
10898 bbr_substate_change(bbr, cts, __LINE__, 0);
10899 bbr_log_type_statechange(bbr, cts, __LINE__);
10900 } else {
10901 /* Back to startup */
10902 bbr->rc_bbr_state = BBR_STATE_STARTUP;
10903 bbr->r_ctl.rc_bbr_state_time = cts;
10904 /*
10905 * We don't want to give a complete free 3
10906 * measurements until we exit, so we use
10907 * the number of pe's we were in probe-rtt
10908 * to add to the startup_epoch. That way
10909 * we will still retain the old state.
10910 */
10911 bbr->r_ctl.rc_bbr_last_startup_epoch += (bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_pe_of_prtt);
10912 bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost;
10913 /* Make sure to use the lower pg when shifting back in */
10914 if (bbr->r_ctl.rc_lost &&
10915 bbr_use_lower_gain_in_startup &&
10916 (bbr->rc_use_google == 0))
10917 bbr->r_ctl.rc_bbr_hptsi_gain = bbr_startup_lower;
10918 else
10919 bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.rc_startup_pg;
10920 bbr->r_ctl.rc_bbr_cwnd_gain = bbr->r_ctl.rc_startup_pg;
10921 /* Probably not needed but set it anyway */
10922 bbr_set_state_target(bbr, __LINE__);
10923 bbr_log_type_statechange(bbr, cts, __LINE__);
10924 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,
10925 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 0);
10926 }
10927 bbr_check_probe_rtt_limits(bbr, cts);
10928}
10929
10930static int32_t inline
10931bbr_should_enter_probe_rtt(struct tcp_bbr *bbr, uint32_t cts)
10932{
10933 if ((bbr->rc_past_init_win == 1) &&
10934 (bbr->rc_in_persist == 0) &&
10935 (bbr_calc_time(cts, bbr->r_ctl.rc_rtt_shrinks) >= bbr->r_ctl.rc_probertt_int)) {
10936 return (1);
10937 }
10938 if (bbr_can_force_probertt &&
10939 (bbr->rc_in_persist == 0) &&
10940 (TSTMP_GT(cts, bbr->r_ctl.last_in_probertt)) &&
10941 ((cts - bbr->r_ctl.last_in_probertt) > bbr->r_ctl.rc_probertt_int)) {
10942 return (1);
10943 }
10944 return (0);
10945}
10946
10947static int32_t
10948bbr_google_startup(struct tcp_bbr *bbr, uint32_t cts, int32_t pkt_epoch)
10949{
10950 uint64_t btlbw, gain;
10951 if (pkt_epoch == 0) {
10952 /*
10953 * Need to be on a pkt-epoch to continue.
10954 */
10955 return (0);
10956 }
10957 btlbw = bbr_get_full_bw(bbr);
10958 gain = ((bbr->r_ctl.rc_bbr_lastbtlbw *
10959 (uint64_t)bbr_start_exit) / (uint64_t)100) + bbr->r_ctl.rc_bbr_lastbtlbw;
10960 if (btlbw >= gain) {
10961 bbr->r_ctl.rc_bbr_last_startup_epoch = bbr->r_ctl.rc_pkt_epoch;
10962 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,
10963 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 3);
10964 bbr->r_ctl.rc_bbr_lastbtlbw = btlbw;
10965 }
10966 if ((bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_bbr_last_startup_epoch) >= BBR_STARTUP_EPOCHS)
10967 return (1);
10968 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,
10969 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 8);
10970 return(0);
10971}
10972
10973static int32_t inline
10974bbr_state_startup(struct tcp_bbr *bbr, uint32_t cts, int32_t epoch, int32_t pkt_epoch)
10975{
10976 /* Have we gained 25% in the last 3 packet based epoch's? */
10977 uint64_t btlbw, gain;
10978 int do_exit;
10979 int delta, rtt_gain;
10980
10981 if ((bbr->rc_tp->snd_una == bbr->rc_tp->snd_max) &&
10982 (bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time) >= bbr_rtt_probe_time)) {
10983 /*
10984 * This qualifies as a RTT_PROBE session since we drop the
10985 * data outstanding to nothing and waited more than
10986 * bbr_rtt_probe_time.
10987 */
10988 bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_WASIDLE, 0);
10989 bbr_set_reduced_rtt(bbr, cts, __LINE__);
10990 }
10991 if (bbr_should_enter_probe_rtt(bbr, cts)) {
10992 bbr_enter_probe_rtt(bbr, cts, __LINE__);
10993 return (0);
10994 }
10995 if (bbr->rc_use_google)
10996 return (bbr_google_startup(bbr, cts, pkt_epoch));
10997
10998 if ((bbr->r_ctl.rc_lost > bbr->r_ctl.rc_lost_at_startup) &&
10999 (bbr_use_lower_gain_in_startup)) {
11000 /* Drop to a lower gain 1.5 x since we saw loss */
11001 bbr->r_ctl.rc_bbr_hptsi_gain = bbr_startup_lower;
11002 }
11003 if (pkt_epoch == 0) {
11004 /*
11005 * Need to be on a pkt-epoch to continue.
11006 */
11007 return (0);
11008 }
11009 if (bbr_rtt_gain_thresh) {
11010 /*
11011 * Do we allow a flow to stay
11012 * in startup with no loss and no
11013 * gain in rtt over a set threshold?
11014 */
11015 if (bbr->r_ctl.rc_pkt_epoch_rtt &&
11016 bbr->r_ctl.startup_last_srtt &&
11017 (bbr->r_ctl.rc_pkt_epoch_rtt > bbr->r_ctl.startup_last_srtt)) {
11018 delta = bbr->r_ctl.rc_pkt_epoch_rtt - bbr->r_ctl.startup_last_srtt;
11019 rtt_gain = (delta * 100) / bbr->r_ctl.startup_last_srtt;
11020 } else
11021 rtt_gain = 0;
11022 if ((bbr->r_ctl.startup_last_srtt == 0) ||
11023 (bbr->r_ctl.rc_pkt_epoch_rtt < bbr->r_ctl.startup_last_srtt))
11024 /* First time or new lower value */
11025 bbr->r_ctl.startup_last_srtt = bbr->r_ctl.rc_pkt_epoch_rtt;
11026
11027 if ((bbr->r_ctl.rc_lost == 0) &&
11028 (rtt_gain < bbr_rtt_gain_thresh)) {
11029 /*
11030 * No loss, and we are under
11031 * our gain threhold for
11032 * increasing RTT.
11033 */
11034 if (bbr->r_ctl.rc_bbr_last_startup_epoch < bbr->r_ctl.rc_pkt_epoch)
11035 bbr->r_ctl.rc_bbr_last_startup_epoch++;
11036 bbr_log_startup_event(bbr, cts, rtt_gain,
11037 delta, bbr->r_ctl.startup_last_srtt, 10);
11038 return (0);
11039 }
11040 }
11041 if ((bbr->r_ctl.r_measurement_count == bbr->r_ctl.last_startup_measure) &&
11042 (bbr->r_ctl.rc_lost_at_startup == bbr->r_ctl.rc_lost) &&
11043 (!IN_RECOVERY(bbr->rc_tp->t_flags))) {
11044 /*
11045 * We only assess if we have a new measurement when
11046 * we have no loss and are not in recovery.
11047 * Drag up by one our last_startup epoch so we will hold
11048 * the number of non-gain we have already accumulated.
11049 */
11050 if (bbr->r_ctl.rc_bbr_last_startup_epoch < bbr->r_ctl.rc_pkt_epoch)
11051 bbr->r_ctl.rc_bbr_last_startup_epoch++;
11052 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,
11053 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 9);
11054 return (0);
11055 }
11056 /* Case where we reduced the lost (bad retransmit) */
11057 if (bbr->r_ctl.rc_lost_at_startup > bbr->r_ctl.rc_lost)
11058 bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost;
11059 bbr->r_ctl.last_startup_measure = bbr->r_ctl.r_measurement_count;
11060 btlbw = bbr_get_full_bw(bbr);
11061 if (bbr->r_ctl.rc_bbr_hptsi_gain == bbr_startup_lower)
11062 gain = ((bbr->r_ctl.rc_bbr_lastbtlbw *
11063 (uint64_t)bbr_low_start_exit) / (uint64_t)100) + bbr->r_ctl.rc_bbr_lastbtlbw;
11064 else
11065 gain = ((bbr->r_ctl.rc_bbr_lastbtlbw *
11066 (uint64_t)bbr_start_exit) / (uint64_t)100) + bbr->r_ctl.rc_bbr_lastbtlbw;
11067 do_exit = 0;
11068 if (btlbw > bbr->r_ctl.rc_bbr_lastbtlbw)
11069 bbr->r_ctl.rc_bbr_lastbtlbw = btlbw;
11070 if (btlbw >= gain) {
11071 bbr->r_ctl.rc_bbr_last_startup_epoch = bbr->r_ctl.rc_pkt_epoch;
11072 /* Update the lost so we won't exit in next set of tests */
11073 bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost;
11074 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,
11075 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 3);
11076 }
11077 if ((bbr->rc_loss_exit &&
11078 (bbr->r_ctl.rc_lost > bbr->r_ctl.rc_lost_at_startup) &&
11079 (bbr->r_ctl.rc_pkt_epoch_loss_rate > bbr_startup_loss_thresh)) &&
11080 ((bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_bbr_last_startup_epoch) >= BBR_STARTUP_EPOCHS)) {
11081 /*
11082 * If we had no gain, we had loss and that loss was above
11083 * our threshould, the rwnd is not constrained, and we have
11084 * had at least 3 packet epochs exit. Note that this is
11085 * switched off by sysctl. Google does not do this by the
11086 * way.
11087 */
11088 if ((ctf_flight_size(bbr->rc_tp,
11089 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) +
11090 (2 * max(bbr->r_ctl.rc_pace_max_segs, bbr->rc_tp->t_maxseg))) <= bbr->rc_tp->snd_wnd) {
11091 do_exit = 1;
11092 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,
11093 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 4);
11094 } else {
11095 /* Just record an updated loss value */
11096 bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost;
11097 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,
11098 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 5);
11099 }
11100 } else
11101 bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost;
11102 if (((bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_bbr_last_startup_epoch) >= BBR_STARTUP_EPOCHS) ||
11103 do_exit) {
11104 /* Return 1 to exit the startup state. */
11105 return (1);
11106 }
11107 /* Stay in startup */
11108 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,
11109 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 8);
11110 return (0);
11111}
11112
11113static void
11114bbr_state_change(struct tcp_bbr *bbr, uint32_t cts, int32_t epoch, int32_t pkt_epoch, uint32_t losses)
11115{
11116 /*
11117 * A tick occurred in the rtt epoch do we need to do anything?
11118 */
11119#ifdef BBR_INVARIANTS
11120 if ((bbr->rc_bbr_state != BBR_STATE_STARTUP) &&
11121 (bbr->rc_bbr_state != BBR_STATE_DRAIN) &&
11122 (bbr->rc_bbr_state != BBR_STATE_PROBE_RTT) &&
11123 (bbr->rc_bbr_state != BBR_STATE_IDLE_EXIT) &&
11124 (bbr->rc_bbr_state != BBR_STATE_PROBE_BW)) {
11125 /* Debug code? */
11126 panic("Unknown BBR state %d?\n", bbr->rc_bbr_state);
11127 }
11128#endif
11129 if (bbr->rc_bbr_state == BBR_STATE_STARTUP) {
11130 /* Do we exit the startup state? */
11131 if (bbr_state_startup(bbr, cts, epoch, pkt_epoch)) {
11132 uint32_t time_in;
11133
11134 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,
11135 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 6);
11136 bbr->rc_filled_pipe = 1;
11137 bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost;
11138 if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) {
11139 time_in = cts - bbr->r_ctl.rc_bbr_state_time;
11140 counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in);
11141 } else
11142 time_in = 0;
11143 if (bbr->rc_no_pacing)
11144 bbr->rc_no_pacing = 0;
11145 bbr->r_ctl.rc_bbr_state_time = cts;
11146 bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.rc_drain_pg;
11147 bbr->rc_bbr_state = BBR_STATE_DRAIN;
11148 bbr_set_state_target(bbr, __LINE__);
11149 if ((bbr->rc_use_google == 0) &&
11150 bbr_slam_cwnd_in_main_drain) {
11151 /* Here we don't have to worry about probe-rtt */
11152 bbr->r_ctl.rc_saved_cwnd = bbr->rc_tp->snd_cwnd;
11153 bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state;
11154 bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__);
11155 }
11156 bbr->r_ctl.rc_bbr_cwnd_gain = bbr_high_gain;
11157 bbr_log_type_statechange(bbr, cts, __LINE__);
11158 if (ctf_flight_size(bbr->rc_tp,
11159 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) <=
11160 bbr->r_ctl.rc_target_at_state) {
11161 /*
11162 * Switch to probe_bw if we are already
11163 * there
11164 */
11165 bbr->rc_bbr_substate = bbr_pick_probebw_substate(bbr, cts);
11166 bbr_substate_change(bbr, cts, __LINE__, 0);
11167 bbr->rc_bbr_state = BBR_STATE_PROBE_BW;
11168 bbr_log_type_statechange(bbr, cts, __LINE__);
11169 }
11170 }
11171 } else if (bbr->rc_bbr_state == BBR_STATE_IDLE_EXIT) {
11172 uint32_t inflight;
11173 struct tcpcb *tp;
11174
11175 tp = bbr->rc_tp;
11176 inflight = ctf_flight_size(tp,
11177 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
11178 if (inflight >= bbr->r_ctl.rc_target_at_state) {
11179 /* We have reached a flight of the cwnd target */
11180 bbr->rc_bbr_state = BBR_STATE_PROBE_BW;
11181 bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT;
11182 bbr->r_ctl.rc_bbr_cwnd_gain = BBR_UNIT;
11183 bbr_set_state_target(bbr, __LINE__);
11184 /*
11185 * Rig it so we don't do anything crazy and
11186 * start fresh with a new randomization.
11187 */
11188 bbr->r_ctl.bbr_smallest_srtt_this_state = 0xffffffff;
11189 bbr->rc_bbr_substate = BBR_SUB_LEVEL6;
11190 bbr_substate_change(bbr, cts, __LINE__, 1);
11191 }
11192 } else if (bbr->rc_bbr_state == BBR_STATE_DRAIN) {
11193 /* Has in-flight reached the bdp (or less)? */
11194 uint32_t inflight;
11195 struct tcpcb *tp;
11196
11197 tp = bbr->rc_tp;
11198 inflight = ctf_flight_size(tp,
11199 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
11200 if ((bbr->rc_use_google == 0) &&
11201 bbr_slam_cwnd_in_main_drain &&
11202 (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_target_at_state)) {
11203 /*
11204 * Here we don't have to worry about probe-rtt
11205 * re-slam it, but keep it slammed down.
11206 */
11207 bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state;
11208 bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__);
11209 }
11210 if (inflight <= bbr->r_ctl.rc_target_at_state) {
11211 /* We have drained */
11212 bbr->rc_bbr_state = BBR_STATE_PROBE_BW;
11213 bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost;
11214 if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) {
11215 uint32_t time_in;
11216
11217 time_in = cts - bbr->r_ctl.rc_bbr_state_time;
11218 counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in);
11219 }
11220 if ((bbr->rc_use_google == 0) &&
11221 bbr_slam_cwnd_in_main_drain &&
11222 (tp->snd_cwnd < bbr->r_ctl.rc_saved_cwnd)) {
11223 /* Restore the cwnd */
11224 tp->snd_cwnd = bbr->r_ctl.rc_saved_cwnd;
11225 bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__);
11226 }
11227 /* Setup probe-rtt has being done now RRS-HERE */
11228 bbr->r_ctl.rc_rtt_shrinks = cts;
11229 bbr->r_ctl.last_in_probertt = cts;
11230 bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_LEAVE_DRAIN, 0);
11231 /* Randomly pick a sub-state */
11232 bbr->rc_bbr_substate = bbr_pick_probebw_substate(bbr, cts);
11233 bbr_substate_change(bbr, cts, __LINE__, 0);
11234 bbr_log_type_statechange(bbr, cts, __LINE__);
11235 }
11236 } else if (bbr->rc_bbr_state == BBR_STATE_PROBE_RTT) {
11237 uint32_t flight;
11238
11239 flight = ctf_flight_size(bbr->rc_tp,
11240 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
11241 bbr->r_ctl.r_app_limited_until = (flight + bbr->r_ctl.rc_delivered);
11242 if (((bbr->r_ctl.bbr_rttprobe_gain_val == 0) || bbr->rc_use_google) &&
11243 (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_target_at_state)) {
11244 /*
11245 * We must keep cwnd at the desired MSS.
11246 */
11247 bbr->rc_tp->snd_cwnd = bbr_rtt_probe_cwndtarg * (bbr->rc_tp->t_maxseg - bbr->rc_last_options);
11248 bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__);
11249 } else if ((bbr_prtt_slam_cwnd) &&
11250 (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_target_at_state)) {
11251 /* Re-slam it */
11252 bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state;
11253 bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__);
11254 }
11255 if (bbr->r_ctl.rc_bbr_enters_probertt == 0) {
11256 /* Has outstanding reached our target? */
11257 if (flight <= bbr->r_ctl.rc_target_at_state) {
11258 bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_REACHTAR, 0);
11259 bbr->r_ctl.rc_bbr_enters_probertt = cts;
11260 /* If time is exactly 0, be 1usec off */
11261 if (bbr->r_ctl.rc_bbr_enters_probertt == 0)
11262 bbr->r_ctl.rc_bbr_enters_probertt = 1;
11263 if (bbr->rc_use_google == 0) {
11264 /*
11265 * Restore any lowering that as occurred to
11266 * reach here
11267 */
11268 if (bbr->r_ctl.bbr_rttprobe_gain_val)
11269 bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.bbr_rttprobe_gain_val;
11270 else
11271 bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT;
11272 }
11273 }
11274 if ((bbr->r_ctl.rc_bbr_enters_probertt == 0) &&
11275 (bbr->rc_use_google == 0) &&
11276 bbr->r_ctl.bbr_rttprobe_gain_val &&
11277 (((cts - bbr->r_ctl.rc_probertt_srttchktim) > bbr_get_rtt(bbr, bbr_drain_rtt)) ||
11278 (flight >= bbr->r_ctl.flightsize_at_drain))) {
11279 /*
11280 * We have doddled with our current hptsi
11281 * gain an srtt and have still not made it
11282 * to target, or we have increased our flight.
11283 * Lets reduce the gain by xx%
11284 * flooring the reduce at DRAIN (based on
11285 * mul/div)
11286 */
11287 int red;
11288
11289 bbr->r_ctl.flightsize_at_drain = flight;
11290 bbr->r_ctl.rc_probertt_srttchktim = cts;
11291 red = max((bbr->r_ctl.bbr_rttprobe_gain_val / 10), 1);
11292 if ((bbr->r_ctl.rc_bbr_hptsi_gain - red) > max(bbr_drain_floor, 1)) {
11293 /* Reduce our gain again */
11294 bbr->r_ctl.rc_bbr_hptsi_gain -= red;
11295 bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_SHRINK_PG, 0);
11296 } else if (bbr->r_ctl.rc_bbr_hptsi_gain > max(bbr_drain_floor, 1)) {
11297 /* one more chance before we give up */
11298 bbr->r_ctl.rc_bbr_hptsi_gain = max(bbr_drain_floor, 1);
11299 bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_SHRINK_PG_FINAL, 0);
11300 } else {
11301 /* At the very bottom */
11302 bbr->r_ctl.rc_bbr_hptsi_gain = max((bbr_drain_floor-1), 1);
11303 }
11304 }
11305 }
11306 if (bbr->r_ctl.rc_bbr_enters_probertt &&
11307 (TSTMP_GT(cts, bbr->r_ctl.rc_bbr_enters_probertt)) &&
11308 ((cts - bbr->r_ctl.rc_bbr_enters_probertt) >= bbr_rtt_probe_time)) {
11309 /* Time to exit probe RTT normally */
11310 bbr_exit_probe_rtt(bbr->rc_tp, bbr, cts);
11311 }
11312 } else if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) {
11313 if ((bbr->rc_tp->snd_una == bbr->rc_tp->snd_max) &&
11314 (bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time) >= bbr_rtt_probe_time)) {
11315 /*
11316 * This qualifies as a RTT_PROBE session since we
11317 * drop the data outstanding to nothing and waited
11318 * more than bbr_rtt_probe_time.
11319 */
11320 bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_WASIDLE, 0);
11321 bbr_set_reduced_rtt(bbr, cts, __LINE__);
11322 }
11323 if (bbr_should_enter_probe_rtt(bbr, cts)) {
11324 bbr_enter_probe_rtt(bbr, cts, __LINE__);
11325 } else {
11326 bbr_set_probebw_gains(bbr, cts, losses);
11327 }
11328 }
11329}
11330
11331static void
11332bbr_check_bbr_for_state(struct tcp_bbr *bbr, uint32_t cts, int32_t line, uint32_t losses)
11333{
11334 int32_t epoch = 0;
11335
11336 if ((cts - bbr->r_ctl.rc_rcv_epoch_start) >= bbr_get_rtt(bbr, BBR_RTT_PROP)) {
11337 bbr_set_epoch(bbr, cts, line);
11338 /* At each epoch doe lt bw sampling */
11339 epoch = 1;
11340 }
11341 bbr_state_change(bbr, cts, epoch, bbr->rc_is_pkt_epoch_now, losses);
11342}
11343
11344static int
11345bbr_do_segment_nounlock(struct mbuf *m, struct tcphdr *th, struct socket *so,
11346 struct tcpcb *tp, int32_t drop_hdrlen, int32_t tlen, uint8_t iptos,
11347 int32_t nxt_pkt, struct timeval *tv)
11348{
11349 int32_t thflags, retval;
11350 uint32_t cts, lcts;
11351 uint32_t tiwin;
11352 struct tcpopt to;
11353 struct tcp_bbr *bbr;
11354 struct bbr_sendmap *rsm;
11355 struct timeval ltv;
11356 int32_t did_out = 0;
11357 uint16_t nsegs;
11358 int32_t prev_state;
11359 uint32_t lost;
11360
11361 nsegs = max(1, m->m_pkthdr.lro_nsegs);
11362 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
11363 /* add in our stats */
11364 kern_prefetch(bbr, &prev_state);
11365 prev_state = 0;
11366 thflags = tcp_get_flags(th);
11367 /*
11368 * If this is either a state-changing packet or current state isn't
11369 * established, we require a write lock on tcbinfo. Otherwise, we
11370 * allow the tcbinfo to be in either alocked or unlocked, as the
11371 * caller may have unnecessarily acquired a write lock due to a
11372 * race.
11373 */
11374 INP_WLOCK_ASSERT(tp->t_inpcb);
11375 KASSERT(tp->t_state > TCPS_LISTEN, ("%s: TCPS_LISTEN",
11376 __func__));
11377 KASSERT(tp->t_state != TCPS_TIME_WAIT, ("%s: TCPS_TIME_WAIT",
11378 __func__));
11379
11380 tp->t_rcvtime = ticks;
11381 /*
11382 * Unscale the window into a 32-bit value. For the SYN_SENT state
11383 * the scale is zero.
11384 */
11385 tiwin = th->th_win << tp->snd_scale;
11386#ifdef STATS
11387 stats_voi_update_abs_ulong(tp->t_stats, VOI_TCP_FRWIN, tiwin);
11388#endif
11389
11390 if (m->m_flags & M_TSTMP) {
11391 /* Prefer the hardware timestamp if present */
11392 struct timespec ts;
11393
11394 mbuf_tstmp2timespec(m, &ts);
11395 bbr->rc_tv.tv_sec = ts.tv_sec;
11396 bbr->rc_tv.tv_usec = ts.tv_nsec / 1000;
11397 bbr->r_ctl.rc_rcvtime = cts = tcp_tv_to_usectick(&bbr->rc_tv);
11398 } else if (m->m_flags & M_TSTMP_LRO) {
11399 /* Next the arrival timestamp */
11400 struct timespec ts;
11401
11402 mbuf_tstmp2timespec(m, &ts);
11403 bbr->rc_tv.tv_sec = ts.tv_sec;
11404 bbr->rc_tv.tv_usec = ts.tv_nsec / 1000;
11405 bbr->r_ctl.rc_rcvtime = cts = tcp_tv_to_usectick(&bbr->rc_tv);
11406 } else {
11407 /*
11408 * Ok just get the current time.
11409 */
11410 bbr->r_ctl.rc_rcvtime = lcts = cts = tcp_get_usecs(&bbr->rc_tv);
11411 }
11412 /*
11413 * Parse options on any incoming segment.
11414 */
11415 tcp_dooptions(&to, (u_char *)(th + 1),
11416 (th->th_off << 2) - sizeof(struct tcphdr),
11417 (thflags & TH_SYN) ? TO_SYN : 0);
11418
11419 /*
11420 * If timestamps were negotiated during SYN/ACK and a
11421 * segment without a timestamp is received, silently drop
11422 * the segment, unless it is a RST segment or missing timestamps are
11423 * tolerated.
11424 * See section 3.2 of RFC 7323.
11425 */
11426 if ((tp->t_flags & TF_RCVD_TSTMP) && !(to.to_flags & TOF_TS) &&
11427 ((thflags & TH_RST) == 0) && (V_tcp_tolerate_missing_ts == 0)) {
11428 retval = 0;
11429 m_freem(m);
11430 goto done_with_input;
11431 }
11432 /*
11433 * If echoed timestamp is later than the current time, fall back to
11434 * non RFC1323 RTT calculation. Normalize timestamp if syncookies
11435 * were used when this connection was established.
11436 */
11437 if ((to.to_flags & TOF_TS) && (to.to_tsecr != 0)) {
11438 to.to_tsecr -= tp->ts_offset;
11439 if (TSTMP_GT(to.to_tsecr, tcp_tv_to_mssectick(&bbr->rc_tv)))
11440 to.to_tsecr = 0;
11441 }
11442 /*
11443 * If its the first time in we need to take care of options and
11444 * verify we can do SACK for rack!
11445 */
11446 if (bbr->r_state == 0) {
11447 /*
11448 * Process options only when we get SYN/ACK back. The SYN
11449 * case for incoming connections is handled in tcp_syncache.
11450 * According to RFC1323 the window field in a SYN (i.e., a
11451 * <SYN> or <SYN,ACK>) segment itself is never scaled. XXX
11452 * this is traditional behavior, may need to be cleaned up.
11453 */
11454 if (bbr->rc_inp == NULL) {
11455 bbr->rc_inp = tp->t_inpcb;
11456 }
11457 /*
11458 * We need to init rc_inp here since its not init'd when
11459 * bbr_init is called
11460 */
11461 if (tp->t_state == TCPS_SYN_SENT && (thflags & TH_SYN)) {
11462 if ((to.to_flags & TOF_SCALE) &&
11463 (tp->t_flags & TF_REQ_SCALE)) {
11464 tp->t_flags |= TF_RCVD_SCALE;
11465 tp->snd_scale = to.to_wscale;
11466 } else
11467 tp->t_flags &= ~TF_REQ_SCALE;
11468 /*
11469 * Initial send window. It will be updated with the
11470 * next incoming segment to the scaled value.
11471 */
11472 tp->snd_wnd = th->th_win;
11473 if ((to.to_flags & TOF_TS) &&
11474 (tp->t_flags & TF_REQ_TSTMP)) {
11475 tp->t_flags |= TF_RCVD_TSTMP;
11476 tp->ts_recent = to.to_tsval;
11477 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv);
11478 } else
11479 tp->t_flags &= ~TF_REQ_TSTMP;
11480 if (to.to_flags & TOF_MSS)
11481 tcp_mss(tp, to.to_mss);
11482 if ((tp->t_flags & TF_SACK_PERMIT) &&
11483 (to.to_flags & TOF_SACKPERM) == 0)
11484 tp->t_flags &= ~TF_SACK_PERMIT;
11485 if (IS_FASTOPEN(tp->t_flags)) {
11486 if (to.to_flags & TOF_FASTOPEN) {
11487 uint16_t mss;
11488
11489 if (to.to_flags & TOF_MSS)
11490 mss = to.to_mss;
11491 else
11492 if ((tp->t_inpcb->inp_vflag & INP_IPV6) != 0)
11493 mss = TCP6_MSS;
11494 else
11495 mss = TCP_MSS;
11496 tcp_fastopen_update_cache(tp, mss,
11497 to.to_tfo_len, to.to_tfo_cookie);
11498 } else
11499 tcp_fastopen_disable_path(tp);
11500 }
11501 }
11502 /*
11503 * At this point we are at the initial call. Here we decide
11504 * if we are doing RACK or not. We do this by seeing if
11505 * TF_SACK_PERMIT is set, if not rack is *not* possible and
11506 * we switch to the default code.
11507 */
11508 if ((tp->t_flags & TF_SACK_PERMIT) == 0) {
11509 /* Bail */
11510 tcp_switch_back_to_default(tp);
11511 (*tp->t_fb->tfb_tcp_do_segment) (m, th, so, tp, drop_hdrlen,
11512 tlen, iptos);
11513 return (1);
11514 }
11515 /* Set the flag */
11516 bbr->r_is_v6 = (tp->t_inpcb->inp_vflag & INP_IPV6) != 0;
11517 tcp_set_hpts(tp->t_inpcb);
11518 sack_filter_clear(&bbr->r_ctl.bbr_sf, th->th_ack);
11519 }
11520 if (thflags & TH_ACK) {
11521 /* Track ack types */
11522 if (to.to_flags & TOF_SACK)
11523 BBR_STAT_INC(bbr_acks_with_sacks);
11524 else
11525 BBR_STAT_INC(bbr_plain_acks);
11526 }
11527 /*
11528 * This is the one exception case where we set the rack state
11529 * always. All other times (timers etc) we must have a rack-state
11530 * set (so we assure we have done the checks above for SACK).
11531 */
11532 if (thflags & TH_FIN)
11533 tcp_log_end_status(tp, TCP_EI_STATUS_CLIENT_FIN);
11534 if (bbr->r_state != tp->t_state)
11535 bbr_set_state(tp, bbr, tiwin);
11536
11537 if (SEQ_GT(th->th_ack, tp->snd_una) && (rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map)) != NULL)
11538 kern_prefetch(rsm, &prev_state);
11539 prev_state = bbr->r_state;
11540 bbr->rc_ack_was_delayed = 0;
11541 lost = bbr->r_ctl.rc_lost;
11542 bbr->rc_is_pkt_epoch_now = 0;
11543 if (m->m_flags & (M_TSTMP|M_TSTMP_LRO)) {
11544 /* Get the real time into lcts and figure the real delay */
11545 lcts = tcp_get_usecs(&ltv);
11546 if (TSTMP_GT(lcts, cts)) {
11547 bbr->r_ctl.rc_ack_hdwr_delay = lcts - cts;
11548 bbr->rc_ack_was_delayed = 1;
11549 if (TSTMP_GT(bbr->r_ctl.rc_ack_hdwr_delay,
11550 bbr->r_ctl.highest_hdwr_delay))
11551 bbr->r_ctl.highest_hdwr_delay = bbr->r_ctl.rc_ack_hdwr_delay;
11552 } else {
11553 bbr->r_ctl.rc_ack_hdwr_delay = 0;
11554 bbr->rc_ack_was_delayed = 0;
11555 }
11556 } else {
11557 bbr->r_ctl.rc_ack_hdwr_delay = 0;
11558 bbr->rc_ack_was_delayed = 0;
11559 }
11560 bbr_log_ack_event(bbr, th, &to, tlen, nsegs, cts, nxt_pkt, m);
11561 if ((thflags & TH_SYN) && (thflags & TH_FIN) && V_drop_synfin) {
11562 retval = 0;
11563 m_freem(m);
11564 goto done_with_input;
11565 }
11566 /*
11567 * If a segment with the ACK-bit set arrives in the SYN-SENT state
11568 * check SEQ.ACK first as described on page 66 of RFC 793, section 3.9.
11569 */
11570 if ((tp->t_state == TCPS_SYN_SENT) && (thflags & TH_ACK) &&
11571 (SEQ_LEQ(th->th_ack, tp->iss) || SEQ_GT(th->th_ack, tp->snd_max))) {
11572 tcp_log_end_status(tp, TCP_EI_STATUS_RST_IN_FRONT);
11573 ctf_do_dropwithreset_conn(m, tp, th, BANDLIM_RST_OPENPORT, tlen);
11574 return (1);
11575 }
11576 if (tiwin > bbr->r_ctl.rc_high_rwnd)
11577 bbr->r_ctl.rc_high_rwnd = tiwin;
11578#ifdef BBR_INVARIANTS
11579 if ((tp->t_inpcb->inp_flags & INP_DROPPED) ||
11580 (tp->t_inpcb->inp_flags2 & INP_FREED)) {
11581 panic("tp:%p bbr:%p given a dropped inp:%p",
11582 tp, bbr, tp->t_inpcb);
11583 }
11584#endif
11585 bbr->r_ctl.rc_flight_at_input = ctf_flight_size(tp,
11586 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
11587 bbr->rtt_valid = 0;
11588 if (to.to_flags & TOF_TS) {
11589 bbr->rc_ts_valid = 1;
11590 bbr->r_ctl.last_inbound_ts = to.to_tsval;
11591 } else {
11592 bbr->rc_ts_valid = 0;
11593 bbr->r_ctl.last_inbound_ts = 0;
11594 }
11595 retval = (*bbr->r_substate) (m, th, so,
11596 tp, &to, drop_hdrlen,
11597 tlen, tiwin, thflags, nxt_pkt, iptos);
11598#ifdef BBR_INVARIANTS
11599 if ((retval == 0) &&
11600 (tp->t_inpcb == NULL)) {
11601 panic("retval:%d tp:%p t_inpcb:NULL state:%d",
11602 retval, tp, prev_state);
11603 }
11604#endif
11605 if (nxt_pkt == 0)
11606 BBR_STAT_INC(bbr_rlock_left_ret0);
11607 else
11608 BBR_STAT_INC(bbr_rlock_left_ret1);
11609 if (retval == 0) {
11610 /*
11611 * If retval is 1 the tcb is unlocked and most likely the tp
11612 * is gone.
11613 */
11614 INP_WLOCK_ASSERT(tp->t_inpcb);
11615 tcp_bbr_xmit_timer_commit(bbr, tp, cts);
11616 if (bbr->rc_is_pkt_epoch_now)
11617 bbr_set_pktepoch(bbr, cts, __LINE__);
11618 bbr_check_bbr_for_state(bbr, cts, __LINE__, (bbr->r_ctl.rc_lost - lost));
11619 if (nxt_pkt == 0) {
11620 if (bbr->r_wanted_output != 0) {
11621 bbr->rc_output_starts_timer = 0;
11622 did_out = 1;
11623 if (tcp_output(tp) < 0)
11624 return (1);
11625 } else
11626 bbr_start_hpts_timer(bbr, tp, cts, 6, 0, 0);
11627 }
11628 if ((nxt_pkt == 0) &&
11629 ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK) == 0) &&
11630 (SEQ_GT(tp->snd_max, tp->snd_una) ||
11631 (tp->t_flags & TF_DELACK) ||
11632 ((V_tcp_always_keepalive || bbr->rc_inp->inp_socket->so_options & SO_KEEPALIVE) &&
11633 (tp->t_state <= TCPS_CLOSING)))) {
11634 /*
11635 * We could not send (probably in the hpts but
11636 * stopped the timer)?
11637 */
11638 if ((tp->snd_max == tp->snd_una) &&
11639 ((tp->t_flags & TF_DELACK) == 0) &&
11640 (tcp_in_hpts(bbr->rc_inp)) &&
11641 (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)) {
11642 /*
11643 * keep alive not needed if we are hptsi
11644 * output yet
11645 */
11646 ;
11647 } else {
11648 if (tcp_in_hpts(bbr->rc_inp)) {
11649 tcp_hpts_remove(bbr->rc_inp);
11650 if ((bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) &&
11651 (TSTMP_GT(lcts, bbr->rc_pacer_started))) {
11652 uint32_t del;
11653
11654 del = lcts - bbr->rc_pacer_started;
11655 if (bbr->r_ctl.rc_last_delay_val > del) {
11656 BBR_STAT_INC(bbr_force_timer_start);
11657 bbr->r_ctl.rc_last_delay_val -= del;
11658 bbr->rc_pacer_started = lcts;
11659 } else {
11660 /* We are late */
11661 bbr->r_ctl.rc_last_delay_val = 0;
11662 BBR_STAT_INC(bbr_force_output);
11663 if (tcp_output(tp) < 0)
11664 return (1);
11665 }
11666 }
11667 }
11668 bbr_start_hpts_timer(bbr, tp, cts, 8, bbr->r_ctl.rc_last_delay_val,
11669 0);
11670 }
11671 } else if ((bbr->rc_output_starts_timer == 0) && (nxt_pkt == 0)) {
11672 /* Do we have the correct timer running? */
11673 bbr_timer_audit(tp, bbr, lcts, &so->so_snd);
11674 }
11675 /* Do we have a new state */
11676 if (bbr->r_state != tp->t_state)
11677 bbr_set_state(tp, bbr, tiwin);
11678done_with_input:
11679 bbr_log_doseg_done(bbr, cts, nxt_pkt, did_out);
11680 if (did_out)
11681 bbr->r_wanted_output = 0;
11682#ifdef BBR_INVARIANTS
11683 if (tp->t_inpcb == NULL) {
11684 panic("OP:%d retval:%d tp:%p t_inpcb:NULL state:%d",
11685 did_out,
11686 retval, tp, prev_state);
11687 }
11688#endif
11689 }
11690 return (retval);
11691}
11692
11693static void
11694bbr_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so,
11695 struct tcpcb *tp, int32_t drop_hdrlen, int32_t tlen, uint8_t iptos)
11696{
11697 struct timeval tv;
11698 int retval;
11699
11700 /* First lets see if we have old packets */
11701 if (tp->t_in_pkt) {
11702 if (ctf_do_queued_segments(so, tp, 1)) {
11703 m_freem(m);
11704 return;
11705 }
11706 }
11707 if (m->m_flags & M_TSTMP_LRO) {
11708 tv.tv_sec = m->m_pkthdr.rcv_tstmp /1000000000;
11709 tv.tv_usec = (m->m_pkthdr.rcv_tstmp % 1000000000)/1000;
11710 } else {
11711 /* Should not be should we kassert instead? */
11712 tcp_get_usecs(&tv);
11713 }
11714 retval = bbr_do_segment_nounlock(m, th, so, tp,
11715 drop_hdrlen, tlen, iptos, 0, &tv);
11716 if (retval == 0) {
11717 INP_WUNLOCK(tp->t_inpcb);
11718 }
11719}
11720
11721/*
11722 * Return how much data can be sent without violating the
11723 * cwnd or rwnd.
11724 */
11725
11726static inline uint32_t
11727bbr_what_can_we_send(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t sendwin,
11728 uint32_t avail, int32_t sb_offset, uint32_t cts)
11729{
11730 uint32_t len;
11731
11732 if (ctf_outstanding(tp) >= tp->snd_wnd) {
11733 /* We never want to go over our peers rcv-window */
11734 len = 0;
11735 } else {
11736 uint32_t flight;
11737
11738 flight = ctf_flight_size(tp, (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
11739 if (flight >= sendwin) {
11740 /*
11741 * We have in flight what we are allowed by cwnd (if
11742 * it was rwnd blocking it would have hit above out
11743 * >= tp->snd_wnd).
11744 */
11745 return (0);
11746 }
11747 len = sendwin - flight;
11748 if ((len + ctf_outstanding(tp)) > tp->snd_wnd) {
11749 /* We would send too much (beyond the rwnd) */
11750 len = tp->snd_wnd - ctf_outstanding(tp);
11751 }
11752 if ((len + sb_offset) > avail) {
11753 /*
11754 * We don't have that much in the SB, how much is
11755 * there?
11756 */
11757 len = avail - sb_offset;
11758 }
11759 }
11760 return (len);
11761}
11762
11763static inline void
11764bbr_do_error_accounting(struct tcpcb *tp, struct tcp_bbr *bbr, struct bbr_sendmap *rsm, int32_t len, int32_t error)
11765{
11766#ifdef NETFLIX_STATS
11767 KMOD_TCPSTAT_INC(tcps_sndpack_error);
11768 KMOD_TCPSTAT_ADD(tcps_sndbyte_error, len);
11769#endif
11770}
11771
11772static inline void
11773bbr_do_send_accounting(struct tcpcb *tp, struct tcp_bbr *bbr, struct bbr_sendmap *rsm, int32_t len, int32_t error)
11774{
11775 if (error) {
11776 bbr_do_error_accounting(tp, bbr, rsm, len, error);
11777 return;
11778 }
11779 if (rsm) {
11780 if (rsm->r_flags & BBR_TLP) {
11781 /*
11782 * TLP should not count in retran count, but in its
11783 * own bin
11784 */
11785#ifdef NETFLIX_STATS
11786 KMOD_TCPSTAT_INC(tcps_tlpresends);
11787 KMOD_TCPSTAT_ADD(tcps_tlpresend_bytes, len);
11788#endif
11789 } else {
11790 /* Retransmit */
11791 tp->t_sndrexmitpack++;
11792 KMOD_TCPSTAT_INC(tcps_sndrexmitpack);
11793 KMOD_TCPSTAT_ADD(tcps_sndrexmitbyte, len);
11794#ifdef STATS
11795 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_RETXPB,
11796 len);
11797#endif
11798 }
11799 /*
11800 * Logs in 0 - 8, 8 is all non probe_bw states 0-7 is
11801 * sub-state
11802 */
11803 counter_u64_add(bbr_state_lost[rsm->r_bbr_state], len);
11804 if (bbr->rc_bbr_state != BBR_STATE_PROBE_BW) {
11805 /* Non probe_bw log in 1, 2, or 4. */
11806 counter_u64_add(bbr_state_resend[bbr->rc_bbr_state], len);
11807 } else {
11808 /*
11809 * Log our probe state 3, and log also 5-13 to show
11810 * us the recovery sub-state for the send. This
11811 * means that 3 == (5+6+7+8+9+10+11+12+13)
11812 */
11813 counter_u64_add(bbr_state_resend[BBR_STATE_PROBE_BW], len);
11814 counter_u64_add(bbr_state_resend[(bbr_state_val(bbr) + 5)], len);
11815 }
11816 /* Place in both 16's the totals of retransmitted */
11817 counter_u64_add(bbr_state_lost[16], len);
11818 counter_u64_add(bbr_state_resend[16], len);
11819 /* Place in 17's the total sent */
11820 counter_u64_add(bbr_state_resend[17], len);
11821 counter_u64_add(bbr_state_lost[17], len);
11822
11823 } else {
11824 /* New sends */
11825 KMOD_TCPSTAT_INC(tcps_sndpack);
11826 KMOD_TCPSTAT_ADD(tcps_sndbyte, len);
11827 /* Place in 17's the total sent */
11828 counter_u64_add(bbr_state_resend[17], len);
11829 counter_u64_add(bbr_state_lost[17], len);
11830#ifdef STATS
11831 stats_voi_update_abs_u64(tp->t_stats, VOI_TCP_TXPB,
11832 len);
11833#endif
11834 }
11835}
11836
11837static void
11838bbr_cwnd_limiting(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t in_level)
11839{
11840 if (bbr->rc_filled_pipe && bbr_target_cwnd_mult_limit && (bbr->rc_use_google == 0)) {
11841 /*
11842 * Limit the cwnd to not be above N x the target plus whats
11843 * is outstanding. The target is based on the current b/w
11844 * estimate.
11845 */
11846 uint32_t target;
11847
11848 target = bbr_get_target_cwnd(bbr, bbr_get_bw(bbr), BBR_UNIT);
11849 target += ctf_outstanding(tp);
11850 target *= bbr_target_cwnd_mult_limit;
11851 if (tp->snd_cwnd > target)
11852 tp->snd_cwnd = target;
11853 bbr_log_type_cwndupd(bbr, 0, 0, 0, 10, 0, 0, __LINE__);
11854 }
11855}
11856
11857static int
11858bbr_window_update_needed(struct tcpcb *tp, struct socket *so, uint32_t recwin, int32_t maxseg)
11859{
11860 /*
11861 * "adv" is the amount we could increase the window, taking into
11862 * account that we are limited by TCP_MAXWIN << tp->rcv_scale.
11863 */
11864 int32_t adv;
11865 int32_t oldwin;
11866
11867 adv = recwin;
11868 if (SEQ_GT(tp->rcv_adv, tp->rcv_nxt)) {
11869 oldwin = (tp->rcv_adv - tp->rcv_nxt);
11870 if (adv > oldwin)
11871 adv -= oldwin;
11872 else {
11873 /* We can't increase the window */
11874 adv = 0;
11875 }
11876 } else
11877 oldwin = 0;
11878
11879 /*
11880 * If the new window size ends up being the same as or less
11881 * than the old size when it is scaled, then don't force
11882 * a window update.
11883 */
11884 if (oldwin >> tp->rcv_scale >= (adv + oldwin) >> tp->rcv_scale)
11885 return (0);
11886
11887 if (adv >= (2 * maxseg) &&
11888 (adv >= (so->so_rcv.sb_hiwat / 4) ||
11889 recwin <= (so->so_rcv.sb_hiwat / 8) ||
11890 so->so_rcv.sb_hiwat <= 8 * maxseg)) {
11891 return (1);
11892 }
11893 if (2 * adv >= (int32_t) so->so_rcv.sb_hiwat)
11894 return (1);
11895 return (0);
11896}
11897
11898/*
11899 * Return 0 on success and a errno on failure to send.
11900 * Note that a 0 return may not mean we sent anything
11901 * if the TCB was on the hpts. A non-zero return
11902 * does indicate the error we got from ip[6]_output.
11903 */
11904static int
11905bbr_output_wtime(struct tcpcb *tp, const struct timeval *tv)
11906{
11907 struct socket *so;
11908 int32_t len;
11909 uint32_t cts;
11910 uint32_t recwin, sendwin;
11911 int32_t sb_offset;
11912 int32_t flags, abandon, error = 0;
11913 struct tcp_log_buffer *lgb = NULL;
11914 struct mbuf *m;
11915 struct mbuf *mb;
11916 uint32_t if_hw_tsomaxsegcount = 0;
11917 uint32_t if_hw_tsomaxsegsize = 0;
11918 uint32_t if_hw_tsomax = 0;
11919 struct ip *ip = NULL;
11920#ifdef TCPDEBUG
11921 struct ipovly *ipov = NULL;
11922#endif
11923 struct tcp_bbr *bbr;
11924 struct tcphdr *th;
11925 struct udphdr *udp = NULL;
11926 u_char opt[TCP_MAXOLEN];
11927 unsigned ipoptlen, optlen, hdrlen;
11928 unsigned ulen;
11929 uint32_t bbr_seq;
11930 uint32_t delay_calc=0;
11931 uint8_t doing_tlp = 0;
11932 uint8_t local_options;
11933#ifdef BBR_INVARIANTS
11934 uint8_t doing_retran_from = 0;
11935 uint8_t picked_up_retran = 0;
11936#endif
11937 uint8_t wanted_cookie = 0;
11938 uint8_t more_to_rxt=0;
11939 int32_t prefetch_so_done = 0;
11940 int32_t prefetch_rsm = 0;
11941 uint32_t tot_len = 0;
11942 uint32_t rtr_cnt = 0;
11943 uint32_t maxseg, pace_max_segs, p_maxseg;
11944 int32_t csum_flags = 0;
11945 int32_t hw_tls;
11946#if defined(IPSEC) || defined(IPSEC_SUPPORT)
11947 unsigned ipsec_optlen = 0;
11948
11949#endif
11950 volatile int32_t sack_rxmit;
11951 struct bbr_sendmap *rsm = NULL;
11952 int32_t tso, mtu;
11953 struct tcpopt to;
11954 int32_t slot = 0;
11955 struct inpcb *inp;
11956 struct sockbuf *sb;
11957 uint32_t hpts_calling;
11958#ifdef INET6
11959 struct ip6_hdr *ip6 = NULL;
11960 int32_t isipv6;
11961#endif
11962 uint8_t app_limited = BBR_JR_SENT_DATA;
11963 uint8_t filled_all = 0;
11964 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
11965 /* We take a cache hit here */
11966 memcpy(&bbr->rc_tv, tv, sizeof(struct timeval));
11967 cts = tcp_tv_to_usectick(&bbr->rc_tv);
11968 inp = bbr->rc_inp;
11969 so = inp->inp_socket;
11970 sb = &so->so_snd;
11971 if (sb->sb_flags & SB_TLS_IFNET)
11972 hw_tls = 1;
11973 else
11974 hw_tls = 0;
11975 kern_prefetch(sb, &maxseg);
11976 maxseg = tp->t_maxseg - bbr->rc_last_options;
11977 if (bbr_minseg(bbr) < maxseg) {
11978 tcp_bbr_tso_size_check(bbr, cts);
11979 }
11980 /* Remove any flags that indicate we are pacing on the inp */
11981 pace_max_segs = bbr->r_ctl.rc_pace_max_segs;
11982 p_maxseg = min(maxseg, pace_max_segs);
11983 INP_WLOCK_ASSERT(inp);
11984#ifdef TCP_OFFLOAD
11985 if (tp->t_flags & TF_TOE)
11986 return (tcp_offload_output(tp));
11987#endif
11988
11989#ifdef INET6
11990 if (bbr->r_state) {
11991 /* Use the cache line loaded if possible */
11992 isipv6 = bbr->r_is_v6;
11993 } else {
11994 isipv6 = (inp->inp_vflag & INP_IPV6) != 0;
11995 }
11996#endif
11997 if (((bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0) &&
11998 tcp_in_hpts(inp)) {
11999 /*
12000 * We are on the hpts for some timer but not hptsi output.
12001 * Possibly remove from the hpts so we can send/recv etc.
12002 */
12003 if ((tp->t_flags & TF_ACKNOW) == 0) {
12004 /*
12005 * No immediate demand right now to send an ack, but
12006 * the user may have read, making room for new data
12007 * (a window update). If so we may want to cancel
12008 * whatever timer is running (KEEP/DEL-ACK?) and
12009 * continue to send out a window update. Or we may
12010 * have gotten more data into the socket buffer to
12011 * send.
12012 */
12013 recwin = lmin(lmax(sbspace(&so->so_rcv), 0),
12014 (long)TCP_MAXWIN << tp->rcv_scale);
12015 if ((bbr_window_update_needed(tp, so, recwin, maxseg) == 0) &&
12016 ((tcp_outflags[tp->t_state] & TH_RST) == 0) &&
12017 ((sbavail(sb) + ((tcp_outflags[tp->t_state] & TH_FIN) ? 1 : 0)) <=
12018 (tp->snd_max - tp->snd_una))) {
12019 /*
12020 * Nothing new to send and no window update
12021 * is needed to send. Lets just return and
12022 * let the timer-run off.
12023 */
12024 return (0);
12025 }
12026 }
12027 tcp_hpts_remove(inp);
12028 bbr_timer_cancel(bbr, __LINE__, cts);
12029 }
12030 if (bbr->r_ctl.rc_last_delay_val) {
12031 /* Calculate a rough delay for early escape to sending */
12032 if (SEQ_GT(cts, bbr->rc_pacer_started))
12033 delay_calc = cts - bbr->rc_pacer_started;
12034 if (delay_calc >= bbr->r_ctl.rc_last_delay_val)
12035 delay_calc -= bbr->r_ctl.rc_last_delay_val;
12036 else
12037 delay_calc = 0;
12038 }
12039 /* Mark that we have called bbr_output(). */
12040 if ((bbr->r_timer_override) ||
12041 (tp->t_state < TCPS_ESTABLISHED)) {
12042 /* Timeouts or early states are exempt */
12043 if (tcp_in_hpts(inp))
12044 tcp_hpts_remove(inp);
12045 } else if (tcp_in_hpts(inp)) {
12046 if ((bbr->r_ctl.rc_last_delay_val) &&
12047 (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) &&
12048 delay_calc) {
12049 /*
12050 * We were being paced for output and the delay has
12051 * already exceeded when we were supposed to be
12052 * called, lets go ahead and pull out of the hpts
12053 * and call output.
12054 */
12055 counter_u64_add(bbr_out_size[TCP_MSS_ACCT_LATE], 1);
12056 bbr->r_ctl.rc_last_delay_val = 0;
12057 tcp_hpts_remove(inp);
12058 } else if (tp->t_state == TCPS_CLOSED) {
12059 bbr->r_ctl.rc_last_delay_val = 0;
12060 tcp_hpts_remove(inp);
12061 } else {
12062 /*
12063 * On the hpts, you shall not pass! even if ACKNOW
12064 * is on, we will when the hpts fires, unless of
12065 * course we are overdue.
12066 */
12067 counter_u64_add(bbr_out_size[TCP_MSS_ACCT_INPACE], 1);
12068 return (0);
12069 }
12070 }
12071 bbr->rc_cwnd_limited = 0;
12072 if (bbr->r_ctl.rc_last_delay_val) {
12073 /* recalculate the real delay and deal with over/under */
12074 if (SEQ_GT(cts, bbr->rc_pacer_started))
12075 delay_calc = cts - bbr->rc_pacer_started;
12076 else
12077 delay_calc = 0;
12078 if (delay_calc >= bbr->r_ctl.rc_last_delay_val)
12079 /* Setup the delay which will be added in */
12080 delay_calc -= bbr->r_ctl.rc_last_delay_val;
12081 else {
12082 /*
12083 * We are early setup to adjust
12084 * our slot time.
12085 */
12086 uint64_t merged_val;
12087
12088 bbr->r_ctl.rc_agg_early += (bbr->r_ctl.rc_last_delay_val - delay_calc);
12089 bbr->r_agg_early_set = 1;
12090 if (bbr->r_ctl.rc_hptsi_agg_delay) {
12091 if (bbr->r_ctl.rc_hptsi_agg_delay >= bbr->r_ctl.rc_agg_early) {
12092 /* Nope our previous late cancels out the early */
12093 bbr->r_ctl.rc_hptsi_agg_delay -= bbr->r_ctl.rc_agg_early;
12094 bbr->r_agg_early_set = 0;
12095 bbr->r_ctl.rc_agg_early = 0;
12096 } else {
12097 bbr->r_ctl.rc_agg_early -= bbr->r_ctl.rc_hptsi_agg_delay;
12098 bbr->r_ctl.rc_hptsi_agg_delay = 0;
12099 }
12100 }
12101 merged_val = bbr->rc_pacer_started;
12102 merged_val <<= 32;
12103 merged_val |= bbr->r_ctl.rc_last_delay_val;
12104 bbr_log_pacing_delay_calc(bbr, inp->inp_hpts_calls,
12105 bbr->r_ctl.rc_agg_early, cts, delay_calc, merged_val,
12106 bbr->r_agg_early_set, 3);
12107 bbr->r_ctl.rc_last_delay_val = 0;
12108 BBR_STAT_INC(bbr_early);
12109 delay_calc = 0;
12110 }
12111 } else {
12112 /* We were not delayed due to hptsi */
12113 if (bbr->r_agg_early_set)
12114 bbr->r_ctl.rc_agg_early = 0;
12115 bbr->r_agg_early_set = 0;
12116 delay_calc = 0;
12117 }
12118 if (delay_calc) {
12119 /*
12120 * We had a hptsi delay which means we are falling behind on
12121 * sending at the expected rate. Calculate an extra amount
12122 * of data we can send, if any, to put us back on track.
12123 */
12124 if ((bbr->r_ctl.rc_hptsi_agg_delay + delay_calc) < bbr->r_ctl.rc_hptsi_agg_delay)
12125 bbr->r_ctl.rc_hptsi_agg_delay = 0xffffffff;
12126 else
12127 bbr->r_ctl.rc_hptsi_agg_delay += delay_calc;
12128 }
12129 sendwin = min(tp->snd_wnd, tp->snd_cwnd);
12130 if ((tp->snd_una == tp->snd_max) &&
12131 (bbr->rc_bbr_state != BBR_STATE_IDLE_EXIT) &&
12132 (sbavail(sb))) {
12133 /*
12134 * Ok we have been idle with nothing outstanding
12135 * we possibly need to start fresh with either a new
12136 * suite of states or a fast-ramp up.
12137 */
12138 bbr_restart_after_idle(bbr,
12139 cts, bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time));
12140 }
12141 /*
12142 * Now was there a hptsi delay where we are behind? We only count
12143 * being behind if: a) We are not in recovery. b) There was a delay.
12144 * <and> c) We had room to send something.
12145 *
12146 */
12147 hpts_calling = inp->inp_hpts_calls;
12148 inp->inp_hpts_calls = 0;
12149 if (bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK) {
12150 int retval;
12151
12152 retval = bbr_process_timers(tp, bbr, cts, hpts_calling);
12153 if (retval != 0) {
12154 counter_u64_add(bbr_out_size[TCP_MSS_ACCT_ATIMER], 1);
12155 /*
12156 * If timers want tcp_drop(), then pass error out,
12157 * otherwise suppress it.
12158 */
12159 return (retval < 0 ? retval : 0);
12160 }
12161 }
12162 bbr->rc_inp->inp_flags2 &= ~INP_MBUF_QUEUE_READY;
12163 if (hpts_calling &&
12164 (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)) {
12165 bbr->r_ctl.rc_last_delay_val = 0;
12166 }
12167 bbr->r_timer_override = 0;
12168 bbr->r_wanted_output = 0;
12169 /*
12170 * For TFO connections in SYN_RECEIVED, only allow the initial
12171 * SYN|ACK and those sent by the retransmit timer.
12172 */
12173 if (IS_FASTOPEN(tp->t_flags) &&
12174 ((tp->t_state == TCPS_SYN_RECEIVED) ||
12175 (tp->t_state == TCPS_SYN_SENT)) &&
12176 SEQ_GT(tp->snd_max, tp->snd_una) && /* initial SYN or SYN|ACK sent */
12177 (tp->t_rxtshift == 0)) { /* not a retransmit */
12178 len = 0;
12179 goto just_return_nolock;
12180 }
12181 /*
12182 * Before sending anything check for a state update. For hpts
12183 * calling without input this is important. If its input calling
12184 * then this was already done.
12185 */
12186 if (bbr->rc_use_google == 0)
12187 bbr_check_bbr_for_state(bbr, cts, __LINE__, 0);
12188again:
12189 /*
12190 * If we've recently taken a timeout, snd_max will be greater than
12191 * snd_max. BBR in general does not pay much attention to snd_nxt
12192 * for historic reasons the persist timer still uses it. This means
12193 * we have to look at it. All retransmissions that are not persits
12194 * use the rsm that needs to be sent so snd_nxt is ignored. At the
12195 * end of this routine we pull snd_nxt always up to snd_max.
12196 */
12197 doing_tlp = 0;
12198#ifdef BBR_INVARIANTS
12199 doing_retran_from = picked_up_retran = 0;
12200#endif
12201 error = 0;
12202 tso = 0;
12203 slot = 0;
12204 mtu = 0;
12205 sendwin = min(tp->snd_wnd, tp->snd_cwnd);
12206 sb_offset = tp->snd_max - tp->snd_una;
12207 flags = tcp_outflags[tp->t_state];
12208 sack_rxmit = 0;
12209 len = 0;
12210 rsm = NULL;
12211 if (flags & TH_RST) {
12212 SOCKBUF_LOCK(sb);
12213 goto send;
12214 }
12215recheck_resend:
12216 while (bbr->r_ctl.rc_free_cnt < bbr_min_req_free) {
12217 /* We need to always have one in reserve */
12218 rsm = bbr_alloc(bbr);
12219 if (rsm == NULL) {
12220 error = ENOMEM;
12221 /* Lie to get on the hpts */
12222 tot_len = tp->t_maxseg;
12223 if (hpts_calling)
12224 /* Retry in a ms */
12225 slot = 1001;
12226 goto just_return_nolock;
12227 }
12228 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_free, rsm, r_next);
12229 bbr->r_ctl.rc_free_cnt++;
12230 rsm = NULL;
12231 }
12232 /* What do we send, a resend? */
12233 if (bbr->r_ctl.rc_resend == NULL) {
12234 /* Check for rack timeout */
12235 bbr->r_ctl.rc_resend = bbr_check_recovery_mode(tp, bbr, cts);
12236 if (bbr->r_ctl.rc_resend) {
12237#ifdef BBR_INVARIANTS
12238 picked_up_retran = 1;
12239#endif
12240 bbr_cong_signal(tp, NULL, CC_NDUPACK, bbr->r_ctl.rc_resend);
12241 }
12242 }
12243 if (bbr->r_ctl.rc_resend) {
12244 rsm = bbr->r_ctl.rc_resend;
12245#ifdef BBR_INVARIANTS
12246 doing_retran_from = 1;
12247#endif
12248 /* Remove any TLP flags its a RACK or T-O */
12249 rsm->r_flags &= ~BBR_TLP;
12250 bbr->r_ctl.rc_resend = NULL;
12251 if (SEQ_LT(rsm->r_start, tp->snd_una)) {
12252#ifdef BBR_INVARIANTS
12253 panic("Huh, tp:%p bbr:%p rsm:%p start:%u < snd_una:%u\n",
12254 tp, bbr, rsm, rsm->r_start, tp->snd_una);
12255 goto recheck_resend;
12256#else
12257 /* TSNH */
12258 rsm = NULL;
12259 goto recheck_resend;
12260#endif
12261 }
12262 rtr_cnt++;
12263 if (rsm->r_flags & BBR_HAS_SYN) {
12264 /* Only retransmit a SYN by itself */
12265 len = 0;
12266 if ((flags & TH_SYN) == 0) {
12267 /* Huh something is wrong */
12268 rsm->r_start++;
12269 if (rsm->r_start == rsm->r_end) {
12270 /* Clean it up, somehow we missed the ack? */
12271 bbr_log_syn(tp, NULL);
12272 } else {
12273 /* TFO with data? */
12274 rsm->r_flags &= ~BBR_HAS_SYN;
12275 len = rsm->r_end - rsm->r_start;
12276 }
12277 } else {
12278 /* Retransmitting SYN */
12279 rsm = NULL;
12280 SOCKBUF_LOCK(sb);
12281 goto send;
12282 }
12283 } else
12284 len = rsm->r_end - rsm->r_start;
12285 if ((bbr->rc_resends_use_tso == 0) &&
12286 (len > maxseg)) {
12287 len = maxseg;
12288 more_to_rxt = 1;
12289 }
12290 sb_offset = rsm->r_start - tp->snd_una;
12291 if (len > 0) {
12292 sack_rxmit = 1;
12293 KMOD_TCPSTAT_INC(tcps_sack_rexmits);
12294 KMOD_TCPSTAT_ADD(tcps_sack_rexmit_bytes,
12295 min(len, maxseg));
12296 } else {
12297 /* I dont think this can happen */
12298 rsm = NULL;
12299 goto recheck_resend;
12300 }
12301 BBR_STAT_INC(bbr_resends_set);
12302 } else if (bbr->r_ctl.rc_tlp_send) {
12303 /*
12304 * Tail loss probe
12305 */
12306 doing_tlp = 1;
12307 rsm = bbr->r_ctl.rc_tlp_send;
12308 bbr->r_ctl.rc_tlp_send = NULL;
12309 sack_rxmit = 1;
12310 len = rsm->r_end - rsm->r_start;
12311 rtr_cnt++;
12312 if ((bbr->rc_resends_use_tso == 0) && (len > maxseg))
12313 len = maxseg;
12314
12315 if (SEQ_GT(tp->snd_una, rsm->r_start)) {
12316#ifdef BBR_INVARIANTS
12317 panic("tp:%p bbc:%p snd_una:%u rsm:%p r_start:%u",
12318 tp, bbr, tp->snd_una, rsm, rsm->r_start);
12319#else
12320 /* TSNH */
12321 rsm = NULL;
12322 goto recheck_resend;
12323#endif
12324 }
12325 sb_offset = rsm->r_start - tp->snd_una;
12326 BBR_STAT_INC(bbr_tlp_set);
12327 }
12328 /*
12329 * Enforce a connection sendmap count limit if set
12330 * as long as we are not retransmiting.
12331 */
12332 if ((rsm == NULL) &&
12333 (V_tcp_map_entries_limit > 0) &&
12334 (bbr->r_ctl.rc_num_maps_alloced >= V_tcp_map_entries_limit)) {
12335 BBR_STAT_INC(bbr_alloc_limited);
12336 if (!bbr->alloc_limit_reported) {
12337 bbr->alloc_limit_reported = 1;
12338 BBR_STAT_INC(bbr_alloc_limited_conns);
12339 }
12340 goto just_return_nolock;
12341 }
12342#ifdef BBR_INVARIANTS
12343 if (rsm && SEQ_LT(rsm->r_start, tp->snd_una)) {
12344 panic("tp:%p bbr:%p rsm:%p sb_offset:%u len:%u",
12345 tp, bbr, rsm, sb_offset, len);
12346 }
12347#endif
12348 /*
12349 * Get standard flags, and add SYN or FIN if requested by 'hidden'
12350 * state flags.
12351 */
12352 if (tp->t_flags & TF_NEEDFIN && (rsm == NULL))
12353 flags |= TH_FIN;
12354 if (tp->t_flags & TF_NEEDSYN)
12355 flags |= TH_SYN;
12356
12357 if (rsm && (rsm->r_flags & BBR_HAS_FIN)) {
12358 /* we are retransmitting the fin */
12359 len--;
12360 if (len) {
12361 /*
12362 * When retransmitting data do *not* include the
12363 * FIN. This could happen from a TLP probe if we
12364 * allowed data with a FIN.
12365 */
12366 flags &= ~TH_FIN;
12367 }
12368 } else if (rsm) {
12369 if (flags & TH_FIN)
12370 flags &= ~TH_FIN;
12371 }
12372 if ((sack_rxmit == 0) && (prefetch_rsm == 0)) {
12373 void *end_rsm;
12374
12375 end_rsm = TAILQ_LAST_FAST(&bbr->r_ctl.rc_tmap, bbr_sendmap, r_tnext);
12376 if (end_rsm)
12377 kern_prefetch(end_rsm, &prefetch_rsm);
12378 prefetch_rsm = 1;
12379 }
12380 SOCKBUF_LOCK(sb);
12381 /*
12382 * If snd_nxt == snd_max and we have transmitted a FIN, the
12383 * sb_offset will be > 0 even if so_snd.sb_cc is 0, resulting in a
12384 * negative length. This can also occur when TCP opens up its
12385 * congestion window while receiving additional duplicate acks after
12386 * fast-retransmit because TCP will reset snd_nxt to snd_max after
12387 * the fast-retransmit.
12388 *
12389 * In the normal retransmit-FIN-only case, however, snd_nxt will be
12390 * set to snd_una, the sb_offset will be 0, and the length may wind
12391 * up 0.
12392 *
12393 * If sack_rxmit is true we are retransmitting from the scoreboard
12394 * in which case len is already set.
12395 */
12396 if (sack_rxmit == 0) {
12397 uint32_t avail;
12398
12399 avail = sbavail(sb);
12400 if (SEQ_GT(tp->snd_max, tp->snd_una))
12401 sb_offset = tp->snd_max - tp->snd_una;
12402 else
12403 sb_offset = 0;
12404 if (bbr->rc_tlp_new_data) {
12405 /* TLP is forcing out new data */
12406 uint32_t tlplen;
12407
12408 doing_tlp = 1;
12409 tlplen = maxseg;
12410
12411 if (tlplen > (uint32_t)(avail - sb_offset)) {
12412 tlplen = (uint32_t)(avail - sb_offset);
12413 }
12414 if (tlplen > tp->snd_wnd) {
12415 len = tp->snd_wnd;
12416 } else {
12417 len = tlplen;
12418 }
12419 bbr->rc_tlp_new_data = 0;
12420 } else {
12421 len = bbr_what_can_we_send(tp, bbr, sendwin, avail, sb_offset, cts);
12422 if ((len < p_maxseg) &&
12423 (bbr->rc_in_persist == 0) &&
12424 (ctf_outstanding(tp) >= (2 * p_maxseg)) &&
12425 ((avail - sb_offset) >= p_maxseg)) {
12426 /*
12427 * We are not completing whats in the socket
12428 * buffer (i.e. there is at least a segment
12429 * waiting to send) and we have 2 or more
12430 * segments outstanding. There is no sense
12431 * of sending a little piece. Lets defer and
12432 * and wait until we can send a whole
12433 * segment.
12434 */
12435 len = 0;
12436 }
12437 if (bbr->rc_in_persist) {
12438 /*
12439 * We are in persists, figure out if
12440 * a retransmit is available (maybe the previous
12441 * persists we sent) or if we have to send new
12442 * data.
12443 */
12444 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
12445 if (rsm) {
12446 len = rsm->r_end - rsm->r_start;
12447 if (rsm->r_flags & BBR_HAS_FIN)
12448 len--;
12449 if ((bbr->rc_resends_use_tso == 0) && (len > maxseg))
12450 len = maxseg;
12451 if (len > 1)
12452 BBR_STAT_INC(bbr_persist_reneg);
12453 /*
12454 * XXXrrs we could force the len to
12455 * 1 byte here to cause the chunk to
12456 * split apart.. but that would then
12457 * mean we always retransmit it as
12458 * one byte even after the window
12459 * opens.
12460 */
12461 sack_rxmit = 1;
12462 sb_offset = rsm->r_start - tp->snd_una;
12463 } else {
12464 /*
12465 * First time through in persists or peer
12466 * acked our one byte. Though we do have
12467 * to have something in the sb.
12468 */
12469 len = 1;
12470 sb_offset = 0;
12471 if (avail == 0)
12472 len = 0;
12473 }
12474 }
12475 }
12476 }
12477 if (prefetch_so_done == 0) {
12478 kern_prefetch(so, &prefetch_so_done);
12479 prefetch_so_done = 1;
12480 }
12481 /*
12482 * Lop off SYN bit if it has already been sent. However, if this is
12483 * SYN-SENT state and if segment contains data and if we don't know
12484 * that foreign host supports TAO, suppress sending segment.
12485 */
12486 if ((flags & TH_SYN) && (rsm == NULL) &&
12487 SEQ_GT(tp->snd_max, tp->snd_una)) {
12488 if (tp->t_state != TCPS_SYN_RECEIVED)
12489 flags &= ~TH_SYN;
12490 /*
12491 * When sending additional segments following a TFO SYN|ACK,
12492 * do not include the SYN bit.
12493 */
12494 if (IS_FASTOPEN(tp->t_flags) &&
12495 (tp->t_state == TCPS_SYN_RECEIVED))
12496 flags &= ~TH_SYN;
12497 sb_offset--, len++;
12498 if (sbavail(sb) == 0)
12499 len = 0;
12500 } else if ((flags & TH_SYN) && rsm) {
12501 /*
12502 * Subtract one from the len for the SYN being
12503 * retransmitted.
12504 */
12505 len--;
12506 }
12507 /*
12508 * Be careful not to send data and/or FIN on SYN segments. This
12509 * measure is needed to prevent interoperability problems with not
12510 * fully conformant TCP implementations.
12511 */
12512 if ((flags & TH_SYN) && (tp->t_flags & TF_NOOPT)) {
12513 len = 0;
12514 flags &= ~TH_FIN;
12515 }
12516 /*
12517 * On TFO sockets, ensure no data is sent in the following cases:
12518 *
12519 * - When retransmitting SYN|ACK on a passively-created socket
12520 * - When retransmitting SYN on an actively created socket
12521 * - When sending a zero-length cookie (cookie request) on an
12522 * actively created socket
12523 * - When the socket is in the CLOSED state (RST is being sent)
12524 */
12525 if (IS_FASTOPEN(tp->t_flags) &&
12526 (((flags & TH_SYN) && (tp->t_rxtshift > 0)) ||
12527 ((tp->t_state == TCPS_SYN_SENT) &&
12528 (tp->t_tfo_client_cookie_len == 0)) ||
12529 (flags & TH_RST))) {
12530 len = 0;
12531 sack_rxmit = 0;
12532 rsm = NULL;
12533 }
12534 /* Without fast-open there should never be data sent on a SYN */
12535 if ((flags & TH_SYN) && (!IS_FASTOPEN(tp->t_flags)))
12536 len = 0;
12537 if (len <= 0) {
12538 /*
12539 * If FIN has been sent but not acked, but we haven't been
12540 * called to retransmit, len will be < 0. Otherwise, window
12541 * shrank after we sent into it. If window shrank to 0,
12542 * cancel pending retransmit, pull snd_nxt back to (closed)
12543 * window, and set the persist timer if it isn't already
12544 * going. If the window didn't close completely, just wait
12545 * for an ACK.
12546 *
12547 * We also do a general check here to ensure that we will
12548 * set the persist timer when we have data to send, but a
12549 * 0-byte window. This makes sure the persist timer is set
12550 * even if the packet hits one of the "goto send" lines
12551 * below.
12552 */
12553 len = 0;
12554 if ((tp->snd_wnd == 0) &&
12555 (TCPS_HAVEESTABLISHED(tp->t_state)) &&
12556 (tp->snd_una == tp->snd_max) &&
12557 (sb_offset < (int)sbavail(sb))) {
12558 /*
12559 * Not enough room in the rwnd to send
12560 * a paced segment out.
12561 */
12562 bbr_enter_persist(tp, bbr, cts, __LINE__);
12563 }
12564 } else if ((rsm == NULL) &&
12565 (doing_tlp == 0) &&
12566 (len < bbr->r_ctl.rc_pace_max_segs)) {
12567 /*
12568 * We are not sending a full segment for
12569 * some reason. Should we not send anything (think
12570 * sws or persists)?
12571 */
12572 if ((tp->snd_wnd < min((bbr->r_ctl.rc_high_rwnd/2), bbr_minseg(bbr))) &&
12573 (TCPS_HAVEESTABLISHED(tp->t_state)) &&
12574 (len < (int)(sbavail(sb) - sb_offset))) {
12575 /*
12576 * Here the rwnd is less than
12577 * the pacing size, this is not a retransmit,
12578 * we are established and
12579 * the send is not the last in the socket buffer
12580 * lets not send, and possibly enter persists.
12581 */
12582 len = 0;
12583 if (tp->snd_max == tp->snd_una)
12584 bbr_enter_persist(tp, bbr, cts, __LINE__);
12585 } else if ((tp->snd_cwnd >= bbr->r_ctl.rc_pace_max_segs) &&
12586 (ctf_flight_size(tp, (bbr->r_ctl.rc_sacked +
12587 bbr->r_ctl.rc_lost_bytes)) > (2 * maxseg)) &&
12588 (len < (int)(sbavail(sb) - sb_offset)) &&
12589 (len < bbr_minseg(bbr))) {
12590 /*
12591 * Here we are not retransmitting, and
12592 * the cwnd is not so small that we could
12593 * not send at least a min size (rxt timer
12594 * not having gone off), We have 2 segments or
12595 * more already in flight, its not the tail end
12596 * of the socket buffer and the cwnd is blocking
12597 * us from sending out minimum pacing segment size.
12598 * Lets not send anything.
12599 */
12600 bbr->rc_cwnd_limited = 1;
12601 len = 0;
12602 } else if (((tp->snd_wnd - ctf_outstanding(tp)) <
12603 min((bbr->r_ctl.rc_high_rwnd/2), bbr_minseg(bbr))) &&
12604 (ctf_flight_size(tp, (bbr->r_ctl.rc_sacked +
12605 bbr->r_ctl.rc_lost_bytes)) > (2 * maxseg)) &&
12606 (len < (int)(sbavail(sb) - sb_offset)) &&
12607 (TCPS_HAVEESTABLISHED(tp->t_state))) {
12608 /*
12609 * Here we have a send window but we have
12610 * filled it up and we can't send another pacing segment.
12611 * We also have in flight more than 2 segments
12612 * and we are not completing the sb i.e. we allow
12613 * the last bytes of the sb to go out even if
12614 * its not a full pacing segment.
12615 */
12616 len = 0;
12617 }
12618 }
12619 /* len will be >= 0 after this point. */
12620 KASSERT(len >= 0, ("[%s:%d]: len < 0", __func__, __LINE__));
12621 tcp_sndbuf_autoscale(tp, so, sendwin);
12622 /*
12623 *
12624 */
12625 if (bbr->rc_in_persist &&
12626 len &&
12627 (rsm == NULL) &&
12628 (len < min((bbr->r_ctl.rc_high_rwnd/2), bbr->r_ctl.rc_pace_max_segs))) {
12629 /*
12630 * We are in persist, not doing a retransmit and don't have enough space
12631 * yet to send a full TSO. So is it at the end of the sb
12632 * if so we need to send else nuke to 0 and don't send.
12633 */
12634 int sbleft;
12635 if (sbavail(sb) > sb_offset)
12636 sbleft = sbavail(sb) - sb_offset;
12637 else
12638 sbleft = 0;
12639 if (sbleft >= min((bbr->r_ctl.rc_high_rwnd/2), bbr->r_ctl.rc_pace_max_segs)) {
12640 /* not at end of sb lets not send */
12641 len = 0;
12642 }
12643 }
12644 /*
12645 * Decide if we can use TCP Segmentation Offloading (if supported by
12646 * hardware).
12647 *
12648 * TSO may only be used if we are in a pure bulk sending state. The
12649 * presence of TCP-MD5, SACK retransmits, SACK advertizements and IP
12650 * options prevent using TSO. With TSO the TCP header is the same
12651 * (except for the sequence number) for all generated packets. This
12652 * makes it impossible to transmit any options which vary per
12653 * generated segment or packet.
12654 *
12655 * IPv4 handling has a clear separation of ip options and ip header
12656 * flags while IPv6 combines both in in6p_outputopts. ip6_optlen()
12657 * does the right thing below to provide length of just ip options
12658 * and thus checking for ipoptlen is enough to decide if ip options
12659 * are present.
12660 */
12661#ifdef INET6
12662 if (isipv6)
12663 ipoptlen = ip6_optlen(inp);
12664 else
12665#endif
12666 if (inp->inp_options)
12667 ipoptlen = inp->inp_options->m_len -
12668 offsetof(struct ipoption, ipopt_list);
12669 else
12670 ipoptlen = 0;
12671#if defined(IPSEC) || defined(IPSEC_SUPPORT)
12672 /*
12673 * Pre-calculate here as we save another lookup into the darknesses
12674 * of IPsec that way and can actually decide if TSO is ok.
12675 */
12676#ifdef INET6
12677 if (isipv6 && IPSEC_ENABLED(ipv6))
12678 ipsec_optlen = IPSEC_HDRSIZE(ipv6, inp);
12679#ifdef INET
12680 else
12681#endif
12682#endif /* INET6 */
12683#ifdef INET
12684 if (IPSEC_ENABLED(ipv4))
12685 ipsec_optlen = IPSEC_HDRSIZE(ipv4, inp);
12686#endif /* INET */
12687#endif /* IPSEC */
12688#if defined(IPSEC) || defined(IPSEC_SUPPORT)
12689 ipoptlen += ipsec_optlen;
12690#endif
12691 if ((tp->t_flags & TF_TSO) && V_tcp_do_tso &&
12692 (len > maxseg) &&
12693 (tp->t_port == 0) &&
12694 ((tp->t_flags & TF_SIGNATURE) == 0) &&
12695 tp->rcv_numsacks == 0 &&
12696 ipoptlen == 0)
12697 tso = 1;
12698
12699 recwin = lmin(lmax(sbspace(&so->so_rcv), 0),
12700 (long)TCP_MAXWIN << tp->rcv_scale);
12701 /*
12702 * Sender silly window avoidance. We transmit under the following
12703 * conditions when len is non-zero:
12704 *
12705 * - We have a full segment (or more with TSO) - This is the last
12706 * buffer in a write()/send() and we are either idle or running
12707 * NODELAY - we've timed out (e.g. persist timer) - we have more
12708 * then 1/2 the maximum send window's worth of data (receiver may be
12709 * limited the window size) - we need to retransmit
12710 */
12711 if (rsm)
12712 goto send;
12713 if (len) {
12714 if (sack_rxmit)
12715 goto send;
12716 if (len >= p_maxseg)
12717 goto send;
12718 /*
12719 * NOTE! on localhost connections an 'ack' from the remote
12720 * end may occur synchronously with the output and cause us
12721 * to flush a buffer queued with moretocome. XXX
12722 *
12723 */
12724 if (((tp->t_flags & TF_MORETOCOME) == 0) && /* normal case */
12725 ((tp->t_flags & TF_NODELAY) ||
12726 ((uint32_t)len + (uint32_t)sb_offset) >= sbavail(&so->so_snd)) &&
12727 (tp->t_flags & TF_NOPUSH) == 0) {
12728 goto send;
12729 }
12730 if ((tp->snd_una == tp->snd_max) && len) { /* Nothing outstanding */
12731 goto send;
12732 }
12733 if (len >= tp->max_sndwnd / 2 && tp->max_sndwnd > 0) {
12734 goto send;
12735 }
12736 }
12737 /*
12738 * Sending of standalone window updates.
12739 *
12740 * Window updates are important when we close our window due to a
12741 * full socket buffer and are opening it again after the application
12742 * reads data from it. Once the window has opened again and the
12743 * remote end starts to send again the ACK clock takes over and
12744 * provides the most current window information.
12745 *
12746 * We must avoid the silly window syndrome whereas every read from
12747 * the receive buffer, no matter how small, causes a window update
12748 * to be sent. We also should avoid sending a flurry of window
12749 * updates when the socket buffer had queued a lot of data and the
12750 * application is doing small reads.
12751 *
12752 * Prevent a flurry of pointless window updates by only sending an
12753 * update when we can increase the advertized window by more than
12754 * 1/4th of the socket buffer capacity. When the buffer is getting
12755 * full or is very small be more aggressive and send an update
12756 * whenever we can increase by two mss sized segments. In all other
12757 * situations the ACK's to new incoming data will carry further
12758 * window increases.
12759 *
12760 * Don't send an independent window update if a delayed ACK is
12761 * pending (it will get piggy-backed on it) or the remote side
12762 * already has done a half-close and won't send more data. Skip
12763 * this if the connection is in T/TCP half-open state.
12764 */
12765 if (recwin > 0 && !(tp->t_flags & TF_NEEDSYN) &&
12766 !(tp->t_flags & TF_DELACK) &&
12767 !TCPS_HAVERCVDFIN(tp->t_state)) {
12768 /* Check to see if we should do a window update */
12769 if (bbr_window_update_needed(tp, so, recwin, maxseg))
12770 goto send;
12771 }
12772 /*
12773 * Send if we owe the peer an ACK, RST, SYN. ACKNOW
12774 * is also a catch-all for the retransmit timer timeout case.
12775 */
12776 if (tp->t_flags & TF_ACKNOW) {
12777 goto send;
12778 }
12779 if (flags & TH_RST) {
12780 /* Always send a RST if one is due */
12781 goto send;
12782 }
12783 if ((flags & TH_SYN) && (tp->t_flags & TF_NEEDSYN) == 0) {
12784 goto send;
12785 }
12786 /*
12787 * If our state indicates that FIN should be sent and we have not
12788 * yet done so, then we need to send.
12789 */
12790 if (flags & TH_FIN &&
12791 ((tp->t_flags & TF_SENTFIN) == 0)) {
12792 goto send;
12793 }
12794 /*
12795 * No reason to send a segment, just return.
12796 */
12797just_return:
12798 SOCKBUF_UNLOCK(sb);
12799just_return_nolock:
12800 if (tot_len)
12801 slot = bbr_get_pacing_delay(bbr, bbr->r_ctl.rc_bbr_hptsi_gain, tot_len, cts, 0);
12802 if (bbr->rc_no_pacing)
12803 slot = 0;
12804 if (tot_len == 0) {
12805 if ((ctf_outstanding(tp) + min((bbr->r_ctl.rc_high_rwnd/2), bbr_minseg(bbr))) >=
12806 tp->snd_wnd) {
12807 BBR_STAT_INC(bbr_rwnd_limited);
12808 app_limited = BBR_JR_RWND_LIMITED;
12809 bbr_cwnd_limiting(tp, bbr, ctf_outstanding(tp));
12810 if ((bbr->rc_in_persist == 0) &&
12811 TCPS_HAVEESTABLISHED(tp->t_state) &&
12812 (tp->snd_max == tp->snd_una) &&
12813 sbavail(&tp->t_inpcb->inp_socket->so_snd)) {
12814 /* No send window.. we must enter persist */
12815 bbr_enter_persist(tp, bbr, bbr->r_ctl.rc_rcvtime, __LINE__);
12816 }
12817 } else if (ctf_outstanding(tp) >= sbavail(sb)) {
12818 BBR_STAT_INC(bbr_app_limited);
12819 app_limited = BBR_JR_APP_LIMITED;
12820 bbr_cwnd_limiting(tp, bbr, ctf_outstanding(tp));
12821 } else if ((ctf_flight_size(tp, (bbr->r_ctl.rc_sacked +
12822 bbr->r_ctl.rc_lost_bytes)) + p_maxseg) >= tp->snd_cwnd) {
12823 BBR_STAT_INC(bbr_cwnd_limited);
12824 app_limited = BBR_JR_CWND_LIMITED;
12825 bbr_cwnd_limiting(tp, bbr, ctf_flight_size(tp, (bbr->r_ctl.rc_sacked +
12826 bbr->r_ctl.rc_lost_bytes)));
12827 bbr->rc_cwnd_limited = 1;
12828 } else {
12829 BBR_STAT_INC(bbr_app_limited);
12830 app_limited = BBR_JR_APP_LIMITED;
12831 bbr_cwnd_limiting(tp, bbr, ctf_outstanding(tp));
12832 }
12833 bbr->r_ctl.rc_hptsi_agg_delay = 0;
12834 bbr->r_agg_early_set = 0;
12835 bbr->r_ctl.rc_agg_early = 0;
12836 bbr->r_ctl.rc_last_delay_val = 0;
12837 } else if (bbr->rc_use_google == 0)
12838 bbr_check_bbr_for_state(bbr, cts, __LINE__, 0);
12839 /* Are we app limited? */
12840 if ((app_limited == BBR_JR_APP_LIMITED) ||
12841 (app_limited == BBR_JR_RWND_LIMITED)) {
12845 bbr->r_ctl.r_app_limited_until = (ctf_flight_size(tp, (bbr->r_ctl.rc_sacked +
12846 bbr->r_ctl.rc_lost_bytes)) + bbr->r_ctl.rc_delivered);
12847 }
12848 if (tot_len == 0)
12849 counter_u64_add(bbr_out_size[TCP_MSS_ACCT_JUSTRET], 1);
12850 /* Dont update the time if we did not send */
12851 bbr->r_ctl.rc_last_delay_val = 0;
12852 bbr->rc_output_starts_timer = 1;
12853 bbr_start_hpts_timer(bbr, tp, cts, 9, slot, tot_len);
12854 bbr_log_type_just_return(bbr, cts, tot_len, hpts_calling, app_limited, p_maxseg, len);
12855 if (SEQ_LT(tp->snd_nxt, tp->snd_max)) {
12856 /* Make sure snd_nxt is drug up */
12857 tp->snd_nxt = tp->snd_max;
12858 }
12859 return (error);
12860
12861send:
12862 if (doing_tlp == 0) {
12863 /*
12864 * Data not a TLP, and its not the rxt firing. If it is the
12865 * rxt firing, we want to leave the tlp_in_progress flag on
12866 * so we don't send another TLP. It has to be a rack timer
12867 * or normal send (response to acked data) to clear the tlp
12868 * in progress flag.
12869 */
12870 bbr->rc_tlp_in_progress = 0;
12871 bbr->rc_tlp_rtx_out = 0;
12872 } else {
12873 /*
12874 * Its a TLP.
12875 */
12876 bbr->rc_tlp_in_progress = 1;
12877 }
12878 bbr_timer_cancel(bbr, __LINE__, cts);
12879 if (rsm == NULL) {
12880 if (sbused(sb) > 0) {
12881 /*
12882 * This is sub-optimal. We only send a stand alone
12883 * FIN on its own segment.
12884 */
12885 if (flags & TH_FIN) {
12886 flags &= ~TH_FIN;
12887 if ((len == 0) && ((tp->t_flags & TF_ACKNOW) == 0)) {
12888 /* Lets not send this */
12889 slot = 0;
12890 goto just_return;
12891 }
12892 }
12893 }
12894 } else {
12895 /*
12896 * We do *not* send a FIN on a retransmit if it has data.
12897 * The if clause here where len > 1 should never come true.
12898 */
12899 if ((len > 0) &&
12900 (((rsm->r_flags & BBR_HAS_FIN) == 0) &&
12901 (flags & TH_FIN))) {
12902 flags &= ~TH_FIN;
12903 len--;
12904 }
12905 }
12906 SOCKBUF_LOCK_ASSERT(sb);
12907 if (len > 0) {
12908 if ((tp->snd_una == tp->snd_max) &&
12909 (bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time) >= bbr_rtt_probe_time)) {
12910 /*
12911 * This qualifies as a RTT_PROBE session since we
12912 * drop the data outstanding to nothing and waited
12913 * more than bbr_rtt_probe_time.
12914 */
12915 bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_WASIDLE, 0);
12916 bbr_set_reduced_rtt(bbr, cts, __LINE__);
12917 }
12918 if (len >= maxseg)
12919 tp->t_flags2 |= TF2_PLPMTU_MAXSEGSNT;
12920 else
12921 tp->t_flags2 &= ~TF2_PLPMTU_MAXSEGSNT;
12922 }
12923 /*
12924 * Before ESTABLISHED, force sending of initial options unless TCP
12925 * set not to do any options. NOTE: we assume that the IP/TCP header
12926 * plus TCP options always fit in a single mbuf, leaving room for a
12927 * maximum link header, i.e. max_linkhdr + sizeof (struct tcpiphdr)
12928 * + optlen <= MCLBYTES
12929 */
12930 optlen = 0;
12931#ifdef INET6
12932 if (isipv6)
12933 hdrlen = sizeof(struct ip6_hdr) + sizeof(struct tcphdr);
12934 else
12935#endif
12936 hdrlen = sizeof(struct tcpiphdr);
12937
12938 /*
12939 * Compute options for segment. We only have to care about SYN and
12940 * established connection segments. Options for SYN-ACK segments
12941 * are handled in TCP syncache.
12942 */
12943 to.to_flags = 0;
12944 local_options = 0;
12945 if ((tp->t_flags & TF_NOOPT) == 0) {
12946 /* Maximum segment size. */
12947 if (flags & TH_SYN) {
12948 to.to_mss = tcp_mssopt(&inp->inp_inc);
12949 if (tp->t_port)
12950 to.to_mss -= V_tcp_udp_tunneling_overhead;
12951 to.to_flags |= TOF_MSS;
12952 /*
12953 * On SYN or SYN|ACK transmits on TFO connections,
12954 * only include the TFO option if it is not a
12955 * retransmit, as the presence of the TFO option may
12956 * have caused the original SYN or SYN|ACK to have
12957 * been dropped by a middlebox.
12958 */
12959 if (IS_FASTOPEN(tp->t_flags) &&
12960 (tp->t_rxtshift == 0)) {
12961 if (tp->t_state == TCPS_SYN_RECEIVED) {
12962 to.to_tfo_len = TCP_FASTOPEN_COOKIE_LEN;
12963 to.to_tfo_cookie =
12964 (u_int8_t *)&tp->t_tfo_cookie.server;
12965 to.to_flags |= TOF_FASTOPEN;
12966 wanted_cookie = 1;
12967 } else if (tp->t_state == TCPS_SYN_SENT) {
12968 to.to_tfo_len =
12969 tp->t_tfo_client_cookie_len;
12970 to.to_tfo_cookie =
12971 tp->t_tfo_cookie.client;
12972 to.to_flags |= TOF_FASTOPEN;
12973 wanted_cookie = 1;
12974 }
12975 }
12976 }
12977 /* Window scaling. */
12978 if ((flags & TH_SYN) && (tp->t_flags & TF_REQ_SCALE)) {
12979 to.to_wscale = tp->request_r_scale;
12980 to.to_flags |= TOF_SCALE;
12981 }
12982 /* Timestamps. */
12983 if ((tp->t_flags & TF_RCVD_TSTMP) ||
12984 ((flags & TH_SYN) && (tp->t_flags & TF_REQ_TSTMP))) {
12985 to.to_tsval = tcp_tv_to_mssectick(&bbr->rc_tv) + tp->ts_offset;
12986 to.to_tsecr = tp->ts_recent;
12987 to.to_flags |= TOF_TS;
12988 local_options += TCPOLEN_TIMESTAMP + 2;
12989 }
12990 /* Set receive buffer autosizing timestamp. */
12991 if (tp->rfbuf_ts == 0 &&
12992 (so->so_rcv.sb_flags & SB_AUTOSIZE))
12993 tp->rfbuf_ts = tcp_tv_to_mssectick(&bbr->rc_tv);
12994 /* Selective ACK's. */
12995 if (flags & TH_SYN)
12996 to.to_flags |= TOF_SACKPERM;
12997 else if (TCPS_HAVEESTABLISHED(tp->t_state) &&
12998 tp->rcv_numsacks > 0) {
12999 to.to_flags |= TOF_SACK;
13000 to.to_nsacks = tp->rcv_numsacks;
13001 to.to_sacks = (u_char *)tp->sackblks;
13002 }
13003#if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE)
13004 /* TCP-MD5 (RFC2385). */
13005 if (tp->t_flags & TF_SIGNATURE)
13006 to.to_flags |= TOF_SIGNATURE;
13007#endif /* TCP_SIGNATURE */
13008
13009 /* Processing the options. */
13010 hdrlen += (optlen = tcp_addoptions(&to, opt));
13011 /*
13012 * If we wanted a TFO option to be added, but it was unable
13013 * to fit, ensure no data is sent.
13014 */
13015 if (IS_FASTOPEN(tp->t_flags) && wanted_cookie &&
13016 !(to.to_flags & TOF_FASTOPEN))
13017 len = 0;
13018 }
13019 if (tp->t_port) {
13020 if (V_tcp_udp_tunneling_port == 0) {
13021 /* The port was removed?? */
13022 SOCKBUF_UNLOCK(&so->so_snd);
13023 return (EHOSTUNREACH);
13024 }
13025 hdrlen += sizeof(struct udphdr);
13026 }
13027#ifdef INET6
13028 if (isipv6)
13029 ipoptlen = ip6_optlen(tp->t_inpcb);
13030 else
13031#endif
13032 if (tp->t_inpcb->inp_options)
13033 ipoptlen = tp->t_inpcb->inp_options->m_len -
13034 offsetof(struct ipoption, ipopt_list);
13035 else
13036 ipoptlen = 0;
13037 ipoptlen = 0;
13038#if defined(IPSEC) || defined(IPSEC_SUPPORT)
13039 ipoptlen += ipsec_optlen;
13040#endif
13041 if (bbr->rc_last_options != local_options) {
13042 /*
13043 * Cache the options length this generally does not change
13044 * on a connection. We use this to calculate TSO.
13045 */
13046 bbr->rc_last_options = local_options;
13047 }
13048 maxseg = tp->t_maxseg - (ipoptlen + optlen);
13049 p_maxseg = min(maxseg, pace_max_segs);
13050 /*
13051 * Adjust data length if insertion of options will bump the packet
13052 * length beyond the t_maxseg length. Clear the FIN bit because we
13053 * cut off the tail of the segment.
13054 */
13055 if (len > maxseg) {
13056 if (len != 0 && (flags & TH_FIN)) {
13057 flags &= ~TH_FIN;
13058 }
13059 if (tso) {
13060 uint32_t moff;
13061 int32_t max_len;
13062
13063 /* extract TSO information */
13064 if_hw_tsomax = tp->t_tsomax;
13065 if_hw_tsomaxsegcount = tp->t_tsomaxsegcount;
13066 if_hw_tsomaxsegsize = tp->t_tsomaxsegsize;
13067 KASSERT(ipoptlen == 0,
13068 ("%s: TSO can't do IP options", __func__));
13069
13070 /*
13071 * Check if we should limit by maximum payload
13072 * length:
13073 */
13074 if (if_hw_tsomax != 0) {
13075 /* compute maximum TSO length */
13076 max_len = (if_hw_tsomax - hdrlen -
13077 max_linkhdr);
13078 if (max_len <= 0) {
13079 len = 0;
13080 } else if (len > max_len) {
13081 len = max_len;
13082 }
13083 }
13084 /*
13085 * Prevent the last segment from being fractional
13086 * unless the send sockbuf can be emptied:
13087 */
13088 if ((sb_offset + len) < sbavail(sb)) {
13089 moff = len % (uint32_t)maxseg;
13090 if (moff != 0) {
13091 len -= moff;
13092 }
13093 }
13094 /*
13095 * In case there are too many small fragments don't
13096 * use TSO:
13097 */
13098 if (len <= maxseg) {
13099 len = maxseg;
13100 tso = 0;
13101 }
13102 } else {
13103 /* Not doing TSO */
13104 if (optlen + ipoptlen >= tp->t_maxseg) {
13105 /*
13106 * Since we don't have enough space to put
13107 * the IP header chain and the TCP header in
13108 * one packet as required by RFC 7112, don't
13109 * send it. Also ensure that at least one
13110 * byte of the payload can be put into the
13111 * TCP segment.
13112 */
13113 SOCKBUF_UNLOCK(&so->so_snd);
13114 error = EMSGSIZE;
13115 sack_rxmit = 0;
13116 goto out;
13117 }
13118 len = maxseg;
13119 }
13120 } else {
13121 /* Not doing TSO */
13122 if_hw_tsomaxsegcount = 0;
13123 tso = 0;
13124 }
13125 KASSERT(len + hdrlen + ipoptlen <= IP_MAXPACKET,
13126 ("%s: len > IP_MAXPACKET", __func__));
13127#ifdef DIAGNOSTIC
13128#ifdef INET6
13129 if (max_linkhdr + hdrlen > MCLBYTES)
13130#else
13131 if (max_linkhdr + hdrlen > MHLEN)
13132#endif
13133 panic("tcphdr too big");
13134#endif
13135 /*
13136 * This KASSERT is here to catch edge cases at a well defined place.
13137 * Before, those had triggered (random) panic conditions further
13138 * down.
13139 */
13140#ifdef BBR_INVARIANTS
13141 if (sack_rxmit) {
13142 if (SEQ_LT(rsm->r_start, tp->snd_una)) {
13143 panic("RSM:%p TP:%p bbr:%p start:%u is < snd_una:%u",
13144 rsm, tp, bbr, rsm->r_start, tp->snd_una);
13145 }
13146 }
13147#endif
13148 KASSERT(len >= 0, ("[%s:%d]: len < 0", __func__, __LINE__));
13149 if ((len == 0) &&
13150 (flags & TH_FIN) &&
13151 (sbused(sb))) {
13152 /*
13153 * We have outstanding data, don't send a fin by itself!.
13154 */
13155 slot = 0;
13156 goto just_return;
13157 }
13158 /*
13159 * Grab a header mbuf, attaching a copy of data to be transmitted,
13160 * and initialize the header from the template for sends on this
13161 * connection.
13162 */
13163 if (len) {
13164 uint32_t moff;
13165
13166 /*
13167 * We place a limit on sending with hptsi.
13168 */
13169 if ((rsm == NULL) && len > pace_max_segs)
13170 len = pace_max_segs;
13171 if (len <= maxseg)
13172 tso = 0;
13173#ifdef INET6
13174 if (MHLEN < hdrlen + max_linkhdr)
13175 m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
13176 else
13177#endif
13178 m = m_gethdr(M_NOWAIT, MT_DATA);
13179
13180 if (m == NULL) {
13181 BBR_STAT_INC(bbr_failed_mbuf_aloc);
13182 bbr_log_enobuf_jmp(bbr, len, cts, __LINE__, len, 0, 0);
13183 SOCKBUF_UNLOCK(sb);
13184 error = ENOBUFS;
13185 sack_rxmit = 0;
13186 goto out;
13187 }
13188 m->m_data += max_linkhdr;
13189 m->m_len = hdrlen;
13190 /*
13191 * Start the m_copy functions from the closest mbuf to the
13192 * sb_offset in the socket buffer chain.
13193 */
13194 if ((sb_offset > sbavail(sb)) || ((len + sb_offset) > sbavail(sb))) {
13195#ifdef BBR_INVARIANTS
13196 if ((len + sb_offset) > (sbavail(sb) + ((flags & (TH_FIN | TH_SYN)) ? 1 : 0)))
13197 panic("tp:%p bbr:%p len:%u sb_offset:%u sbavail:%u rsm:%p %u:%u:%u",
13198 tp, bbr, len, sb_offset, sbavail(sb), rsm,
13199 doing_retran_from,
13200 picked_up_retran,
13201 doing_tlp);
13202
13203#endif
13204 /*
13205 * In this messed up situation we have two choices,
13206 * a) pretend the send worked, and just start timers
13207 * and what not (not good since that may lead us
13208 * back here a lot). <or> b) Send the lowest segment
13209 * in the map. <or> c) Drop the connection. Lets do
13210 * <b> which if it continues to happen will lead to
13211 * <c> via timeouts.
13212 */
13213 BBR_STAT_INC(bbr_offset_recovery);
13214 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
13215 sb_offset = 0;
13216 if (rsm == NULL) {
13217 sack_rxmit = 0;
13218 len = sbavail(sb);
13219 } else {
13220 sack_rxmit = 1;
13221 if (rsm->r_start != tp->snd_una) {
13222 /*
13223 * Things are really messed up, <c>
13224 * is the only thing to do.
13225 */
13226 BBR_STAT_INC(bbr_offset_drop);
13227 SOCKBUF_UNLOCK(sb);
13228 (void)m_free(m);
13229 return (-EFAULT); /* tcp_drop() */
13230 }
13231 len = rsm->r_end - rsm->r_start;
13232 }
13233 if (len > sbavail(sb))
13234 len = sbavail(sb);
13235 if (len > maxseg)
13236 len = maxseg;
13237 }
13238 mb = sbsndptr_noadv(sb, sb_offset, &moff);
13239 if (len <= MHLEN - hdrlen - max_linkhdr && !hw_tls) {
13240 m_copydata(mb, moff, (int)len,
13241 mtod(m, caddr_t)+hdrlen);
13242 if (rsm == NULL)
13243 sbsndptr_adv(sb, mb, len);
13244 m->m_len += len;
13245 } else {
13246 struct sockbuf *msb;
13247
13248 if (rsm)
13249 msb = NULL;
13250 else
13251 msb = sb;
13252#ifdef BBR_INVARIANTS
13253 if ((len + moff) > (sbavail(sb) + ((flags & (TH_FIN | TH_SYN)) ? 1 : 0))) {
13254 if (rsm) {
13255 panic("tp:%p bbr:%p len:%u moff:%u sbavail:%u rsm:%p snd_una:%u rsm_start:%u flg:%x %u:%u:%u sr:%d ",
13256 tp, bbr, len, moff,
13257 sbavail(sb), rsm,
13258 tp->snd_una, rsm->r_flags, rsm->r_start,
13259 doing_retran_from,
13260 picked_up_retran,
13261 doing_tlp, sack_rxmit);
13262 } else {
13263 panic("tp:%p bbr:%p len:%u moff:%u sbavail:%u sb_offset:%u snd_una:%u",
13264 tp, bbr, len, moff, sbavail(sb), sb_offset, tp->snd_una);
13265 }
13266 }
13267#endif
13268 m->m_next = tcp_m_copym(
13269 mb, moff, &len,
13270 if_hw_tsomaxsegcount,
13271 if_hw_tsomaxsegsize, msb,
13272 ((rsm == NULL) ? hw_tls : 0)
13273#ifdef NETFLIX_COPY_ARGS
13274 , &filled_all
13275#endif
13276 );
13277 if (len <= maxseg) {
13278 /*
13279 * Must have ran out of mbufs for the copy
13280 * shorten it to no longer need tso. Lets
13281 * not put on sendalot since we are low on
13282 * mbufs.
13283 */
13284 tso = 0;
13285 }
13286 if (m->m_next == NULL) {
13287 SOCKBUF_UNLOCK(sb);
13288 (void)m_free(m);
13289 error = ENOBUFS;
13290 sack_rxmit = 0;
13291 goto out;
13292 }
13293 }
13294#ifdef BBR_INVARIANTS
13295 if (tso && len < maxseg) {
13296 panic("tp:%p tso on, but len:%d < maxseg:%d",
13297 tp, len, maxseg);
13298 }
13299 if (tso && if_hw_tsomaxsegcount) {
13300 int32_t seg_cnt = 0;
13301 struct mbuf *foo;
13302
13303 foo = m;
13304 while (foo) {
13305 seg_cnt++;
13306 foo = foo->m_next;
13307 }
13308 if (seg_cnt > if_hw_tsomaxsegcount) {
13309 panic("seg_cnt:%d > max:%d", seg_cnt, if_hw_tsomaxsegcount);
13310 }
13311 }
13312#endif
13313 /*
13314 * If we're sending everything we've got, set PUSH. (This
13315 * will keep happy those implementations which only give
13316 * data to the user when a buffer fills or a PUSH comes in.)
13317 */
13318 if (sb_offset + len == sbused(sb) &&
13319 sbused(sb) &&
13320 !(flags & TH_SYN)) {
13321 flags |= TH_PUSH;
13322 }
13323 SOCKBUF_UNLOCK(sb);
13324 } else {
13325 SOCKBUF_UNLOCK(sb);
13326 if (tp->t_flags & TF_ACKNOW)
13327 KMOD_TCPSTAT_INC(tcps_sndacks);
13328 else if (flags & (TH_SYN | TH_FIN | TH_RST))
13329 KMOD_TCPSTAT_INC(tcps_sndctrl);
13330 else
13331 KMOD_TCPSTAT_INC(tcps_sndwinup);
13332
13333 m = m_gethdr(M_NOWAIT, MT_DATA);
13334 if (m == NULL) {
13335 BBR_STAT_INC(bbr_failed_mbuf_aloc);
13336 bbr_log_enobuf_jmp(bbr, len, cts, __LINE__, len, 0, 0);
13337 error = ENOBUFS;
13338 /* Fudge the send time since we could not send */
13339 sack_rxmit = 0;
13340 goto out;
13341 }
13342#ifdef INET6
13343 if (isipv6 && (MHLEN < hdrlen + max_linkhdr) &&
13344 MHLEN >= hdrlen) {
13345 M_ALIGN(m, hdrlen);
13346 } else
13347#endif
13348 m->m_data += max_linkhdr;
13349 m->m_len = hdrlen;
13350 }
13351 SOCKBUF_UNLOCK_ASSERT(sb);
13352 m->m_pkthdr.rcvif = (struct ifnet *)0;
13353#ifdef MAC
13354 mac_inpcb_create_mbuf(inp, m);
13355#endif
13356#ifdef INET6
13357 if (isipv6) {
13358 ip6 = mtod(m, struct ip6_hdr *);
13359 if (tp->t_port) {
13360 udp = (struct udphdr *)((caddr_t)ip6 + sizeof(struct ip6_hdr));
13361 udp->uh_sport = htons(V_tcp_udp_tunneling_port);
13362 udp->uh_dport = tp->t_port;
13363 ulen = hdrlen + len - sizeof(struct ip6_hdr);
13364 udp->uh_ulen = htons(ulen);
13365 th = (struct tcphdr *)(udp + 1);
13366 } else {
13367 th = (struct tcphdr *)(ip6 + 1);
13368 }
13369 tcpip_fillheaders(inp, tp->t_port, ip6, th);
13370 } else
13371#endif /* INET6 */
13372 {
13373 ip = mtod(m, struct ip *);
13374#ifdef TCPDEBUG
13375 ipov = (struct ipovly *)ip;
13376#endif
13377 if (tp->t_port) {
13378 udp = (struct udphdr *)((caddr_t)ip + sizeof(struct ip));
13379 udp->uh_sport = htons(V_tcp_udp_tunneling_port);
13380 udp->uh_dport = tp->t_port;
13381 ulen = hdrlen + len - sizeof(struct ip);
13382 udp->uh_ulen = htons(ulen);
13383 th = (struct tcphdr *)(udp + 1);
13384 } else {
13385 th = (struct tcphdr *)(ip + 1);
13386 }
13387 tcpip_fillheaders(inp, tp->t_port, ip, th);
13388 }
13389 /*
13390 * If we are doing retransmissions, then snd_nxt will not reflect
13391 * the first unsent octet. For ACK only packets, we do not want the
13392 * sequence number of the retransmitted packet, we want the sequence
13393 * number of the next unsent octet. So, if there is no data (and no
13394 * SYN or FIN), use snd_max instead of snd_nxt when filling in
13395 * ti_seq. But if we are in persist state, snd_max might reflect
13396 * one byte beyond the right edge of the window, so use snd_nxt in
13397 * that case, since we know we aren't doing a retransmission.
13398 * (retransmit and persist are mutually exclusive...)
13399 */
13400 if (sack_rxmit == 0) {
13401 if (len && ((flags & (TH_FIN | TH_SYN | TH_RST)) == 0)) {
13402 /* New data (including new persists) */
13403 th->th_seq = htonl(tp->snd_max);
13404 bbr_seq = tp->snd_max;
13405 } else if (flags & TH_SYN) {
13406 /* Syn's always send from iss */
13407 th->th_seq = htonl(tp->iss);
13408 bbr_seq = tp->iss;
13409 } else if (flags & TH_FIN) {
13410 if (flags & TH_FIN && tp->t_flags & TF_SENTFIN) {
13411 /*
13412 * If we sent the fin already its 1 minus
13413 * snd_max
13414 */
13415 th->th_seq = (htonl(tp->snd_max - 1));
13416 bbr_seq = (tp->snd_max - 1);
13417 } else {
13418 /* First time FIN use snd_max */
13419 th->th_seq = htonl(tp->snd_max);
13420 bbr_seq = tp->snd_max;
13421 }
13422 } else {
13423 /*
13424 * len == 0 and not persist we use snd_max, sending
13425 * an ack unless we have sent the fin then its 1
13426 * minus.
13427 */
13428 /*
13429 * XXXRRS Question if we are in persists and we have
13430 * nothing outstanding to send and we have not sent
13431 * a FIN, we will send an ACK. In such a case it
13432 * might be better to send (tp->snd_una - 1) which
13433 * would force the peer to ack.
13434 */
13435 if (tp->t_flags & TF_SENTFIN) {
13436 th->th_seq = htonl(tp->snd_max - 1);
13437 bbr_seq = (tp->snd_max - 1);
13438 } else {
13439 th->th_seq = htonl(tp->snd_max);
13440 bbr_seq = tp->snd_max;
13441 }
13442 }
13443 } else {
13444 /* All retransmits use the rsm to guide the send */
13445 th->th_seq = htonl(rsm->r_start);
13446 bbr_seq = rsm->r_start;
13447 }
13448 th->th_ack = htonl(tp->rcv_nxt);
13449 if (optlen) {
13450 bcopy(opt, th + 1, optlen);
13451 th->th_off = (sizeof(struct tcphdr) + optlen) >> 2;
13452 }
13453 tcp_set_flags(th, flags);
13454 /*
13455 * Calculate receive window. Don't shrink window, but avoid silly
13456 * window syndrome.
13457 */
13458 if ((flags & TH_RST) || ((recwin < (so->so_rcv.sb_hiwat / 4) &&
13459 recwin < maxseg)))
13460 recwin = 0;
13461 if (SEQ_GT(tp->rcv_adv, tp->rcv_nxt) &&
13462 recwin < (tp->rcv_adv - tp->rcv_nxt))
13463 recwin = (tp->rcv_adv - tp->rcv_nxt);
13464 if (recwin > TCP_MAXWIN << tp->rcv_scale)
13465 recwin = TCP_MAXWIN << tp->rcv_scale;
13466
13467 /*
13468 * According to RFC1323 the window field in a SYN (i.e., a <SYN> or
13469 * <SYN,ACK>) segment itself is never scaled. The <SYN,ACK> case is
13470 * handled in syncache.
13471 */
13472 if (flags & TH_SYN)
13473 th->th_win = htons((u_short)
13474 (min(sbspace(&so->so_rcv), TCP_MAXWIN)));
13475 else {
13476 /* Avoid shrinking window with window scaling. */
13477 recwin = roundup2(recwin, 1 << tp->rcv_scale);
13478 th->th_win = htons((u_short)(recwin >> tp->rcv_scale));
13479 }
13480 /*
13481 * Adjust the RXWIN0SENT flag - indicate that we have advertised a 0
13482 * window. This may cause the remote transmitter to stall. This
13483 * flag tells soreceive() to disable delayed acknowledgements when
13484 * draining the buffer. This can occur if the receiver is
13485 * attempting to read more data than can be buffered prior to
13486 * transmitting on the connection.
13487 */
13488 if (th->th_win == 0) {
13489 tp->t_sndzerowin++;
13490 tp->t_flags |= TF_RXWIN0SENT;
13491 } else
13492 tp->t_flags &= ~TF_RXWIN0SENT;
13493 /*
13494 * We don't support urgent data, but drag along
13495 * the pointer in case of a stack switch.
13496 */
13497 tp->snd_up = tp->snd_una;
13498
13499#if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE)
13500 if (to.to_flags & TOF_SIGNATURE) {
13501 /*
13502 * Calculate MD5 signature and put it into the place
13503 * determined before. NOTE: since TCP options buffer doesn't
13504 * point into mbuf's data, calculate offset and use it.
13505 */
13506 if (!TCPMD5_ENABLED() || TCPMD5_OUTPUT(m, th,
13507 (u_char *)(th + 1) + (to.to_signature - opt)) != 0) {
13508 /*
13509 * Do not send segment if the calculation of MD5
13510 * digest has failed.
13511 */
13512 goto out;
13513 }
13514 }
13515#endif
13516
13517 /*
13518 * Put TCP length in extended header, and then checksum extended
13519 * header and data.
13520 */
13521 m->m_pkthdr.len = hdrlen + len; /* in6_cksum() need this */
13522#ifdef INET6
13523 if (isipv6) {
13524 /*
13525 * ip6_plen is not need to be filled now, and will be filled
13526 * in ip6_output.
13527 */
13528 if (tp->t_port) {
13529 m->m_pkthdr.csum_flags = CSUM_UDP_IPV6;
13530 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);
13531 udp->uh_sum = in6_cksum_pseudo(ip6, ulen, IPPROTO_UDP, 0);
13532 th->th_sum = htons(0);
13533 UDPSTAT_INC(udps_opackets);
13534 } else {
13535 csum_flags = m->m_pkthdr.csum_flags = CSUM_TCP_IPV6;
13536 m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);
13537 th->th_sum = in6_cksum_pseudo(ip6, sizeof(struct tcphdr) +
13538 optlen + len, IPPROTO_TCP, 0);
13539 }
13540 }
13541#endif
13542#if defined(INET6) && defined(INET)
13543 else
13544#endif
13545#ifdef INET
13546 {
13547 if (tp->t_port) {
13548 m->m_pkthdr.csum_flags = CSUM_UDP;
13549 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);
13550 udp->uh_sum = in_pseudo(ip->ip_src.s_addr,
13551 ip->ip_dst.s_addr, htons(ulen + IPPROTO_UDP));
13552 th->th_sum = htons(0);
13553 UDPSTAT_INC(udps_opackets);
13554 } else {
13555 csum_flags = m->m_pkthdr.csum_flags = CSUM_TCP;
13556 m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);
13557 th->th_sum = in_pseudo(ip->ip_src.s_addr,
13558 ip->ip_dst.s_addr, htons(sizeof(struct tcphdr) +
13559 IPPROTO_TCP + len + optlen));
13560 }
13561 /* IP version must be set here for ipv4/ipv6 checking later */
13562 KASSERT(ip->ip_v == IPVERSION,
13563 ("%s: IP version incorrect: %d", __func__, ip->ip_v));
13564 }
13565#endif
13566
13567 /*
13568 * Enable TSO and specify the size of the segments. The TCP pseudo
13569 * header checksum is always provided. XXX: Fixme: This is currently
13570 * not the case for IPv6.
13571 */
13572 if (tso) {
13573 KASSERT(len > maxseg,
13574 ("%s: len:%d <= tso_segsz:%d", __func__, len, maxseg));
13575 m->m_pkthdr.csum_flags |= CSUM_TSO;
13576 csum_flags |= CSUM_TSO;
13577 m->m_pkthdr.tso_segsz = maxseg;
13578 }
13579 KASSERT(len + hdrlen == m_length(m, NULL),
13580 ("%s: mbuf chain different than expected: %d + %u != %u",
13581 __func__, len, hdrlen, m_length(m, NULL)));
13582
13583#ifdef TCP_HHOOK
13584 /* Run HHOOK_TC_ESTABLISHED_OUT helper hooks. */
13585 hhook_run_tcp_est_out(tp, th, &to, len, tso);
13586#endif
13587#ifdef TCPDEBUG
13588 /*
13589 * Trace.
13590 */
13591 if (so->so_options & SO_DEBUG) {
13592 u_short save = 0;
13593
13594#ifdef INET6
13595 if (!isipv6)
13596#endif
13597 {
13598 save = ipov->ih_len;
13599 ipov->ih_len = htons(m->m_pkthdr.len /* - hdrlen +
13600 * (th->th_off << 2) */ );
13601 }
13602 tcp_trace(TA_OUTPUT, tp->t_state, tp, mtod(m, void *), th, 0);
13603#ifdef INET6
13604 if (!isipv6)
13605#endif
13606 ipov->ih_len = save;
13607 }
13608#endif /* TCPDEBUG */
13609
13610 /* Log to the black box */
13611 if (tp->t_logstate != TCP_LOG_STATE_OFF) {
13612 union tcp_log_stackspecific log;
13613
13614 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
13615 /* Record info on type of transmission */
13616 log.u_bbr.flex1 = bbr->r_ctl.rc_hptsi_agg_delay;
13617 log.u_bbr.flex2 = (bbr->r_recovery_bw << 3);
13618 log.u_bbr.flex3 = maxseg;
13619 log.u_bbr.flex4 = delay_calc;
13620 /* Encode filled_all into the upper flex5 bit */
13621 log.u_bbr.flex5 = bbr->rc_past_init_win;
13622 log.u_bbr.flex5 <<= 1;
13623 log.u_bbr.flex5 |= bbr->rc_no_pacing;
13624 log.u_bbr.flex5 <<= 29;
13625 if (filled_all)
13626 log.u_bbr.flex5 |= 0x80000000;
13627 log.u_bbr.flex5 |= tp->t_maxseg;
13628 log.u_bbr.flex6 = bbr->r_ctl.rc_pace_max_segs;
13629 log.u_bbr.flex7 = (bbr->rc_bbr_state << 8) | bbr_state_val(bbr);
13630 /* lets poke in the low and the high here for debugging */
13631 log.u_bbr.pkts_out = bbr->rc_tp->t_maxseg;
13632 if (rsm || sack_rxmit) {
13633 if (doing_tlp)
13634 log.u_bbr.flex8 = 2;
13635 else
13636 log.u_bbr.flex8 = 1;
13637 } else {
13638 log.u_bbr.flex8 = 0;
13639 }
13640 lgb = tcp_log_event_(tp, th, &so->so_rcv, &so->so_snd, TCP_LOG_OUT, ERRNO_UNK,
13641 len, &log, false, NULL, NULL, 0, tv);
13642 } else {
13643 lgb = NULL;
13644 }
13645 /*
13646 * Fill in IP length and desired time to live and send to IP level.
13647 * There should be a better way to handle ttl and tos; we could keep
13648 * them in the template, but need a way to checksum without them.
13649 */
13650 /*
13651 * m->m_pkthdr.len should have been set before cksum calcuration,
13652 * because in6_cksum() need it.
13653 */
13654#ifdef INET6
13655 if (isipv6) {
13656 /*
13657 * we separately set hoplimit for every segment, since the
13658 * user might want to change the value via setsockopt. Also,
13659 * desired default hop limit might be changed via Neighbor
13660 * Discovery.
13661 */
13662 ip6->ip6_hlim = in6_selecthlim(inp, NULL);
13663
13664 /*
13665 * Set the packet size here for the benefit of DTrace
13666 * probes. ip6_output() will set it properly; it's supposed
13667 * to include the option header lengths as well.
13668 */
13669 ip6->ip6_plen = htons(m->m_pkthdr.len - sizeof(*ip6));
13670
13671 if (V_path_mtu_discovery && maxseg > V_tcp_minmss)
13672 tp->t_flags2 |= TF2_PLPMTU_PMTUD;
13673 else
13674 tp->t_flags2 &= ~TF2_PLPMTU_PMTUD;
13675
13676 if (tp->t_state == TCPS_SYN_SENT)
13677 TCP_PROBE5(connect__request, NULL, tp, ip6, tp, th);
13678
13679 TCP_PROBE5(send, NULL, tp, ip6, tp, th);
13680 /* TODO: IPv6 IP6TOS_ECT bit on */
13681 error = ip6_output(m, inp->in6p_outputopts,
13682 &inp->inp_route6,
13683 ((rsm || sack_rxmit) ? IP_NO_SND_TAG_RL : 0),
13684 NULL, NULL, inp);
13685
13686 if (error == EMSGSIZE && inp->inp_route6.ro_nh != NULL)
13687 mtu = inp->inp_route6.ro_nh->nh_mtu;
13688 }
13689#endif /* INET6 */
13690#if defined(INET) && defined(INET6)
13691 else
13692#endif
13693#ifdef INET
13694 {
13695 ip->ip_len = htons(m->m_pkthdr.len);
13696#ifdef INET6
13697 if (isipv6)
13698 ip->ip_ttl = in6_selecthlim(inp, NULL);
13699#endif /* INET6 */
13700 /*
13701 * If we do path MTU discovery, then we set DF on every
13702 * packet. This might not be the best thing to do according
13703 * to RFC3390 Section 2. However the tcp hostcache migitates
13704 * the problem so it affects only the first tcp connection
13705 * with a host.
13706 *
13707 * NB: Don't set DF on small MTU/MSS to have a safe
13708 * fallback.
13709 */
13710 if (V_path_mtu_discovery && tp->t_maxseg > V_tcp_minmss) {
13711 tp->t_flags2 |= TF2_PLPMTU_PMTUD;
13712 if (tp->t_port == 0 || len < V_tcp_minmss) {
13713 ip->ip_off |= htons(IP_DF);
13714 }
13715 } else {
13716 tp->t_flags2 &= ~TF2_PLPMTU_PMTUD;
13717 }
13718
13719 if (tp->t_state == TCPS_SYN_SENT)
13720 TCP_PROBE5(connect__request, NULL, tp, ip, tp, th);
13721
13722 TCP_PROBE5(send, NULL, tp, ip, tp, th);
13723
13724 error = ip_output(m, inp->inp_options, &inp->inp_route,
13725 ((rsm || sack_rxmit) ? IP_NO_SND_TAG_RL : 0), 0,
13726 inp);
13727 if (error == EMSGSIZE && inp->inp_route.ro_nh != NULL)
13728 mtu = inp->inp_route.ro_nh->nh_mtu;
13729 }
13730#endif /* INET */
13731out:
13732
13733 if (lgb) {
13734 lgb->tlb_errno = error;
13735 lgb = NULL;
13736 }
13737 /*
13738 * In transmit state, time the transmission and arrange for the
13739 * retransmit. In persist state, just set snd_max.
13740 */
13741 if (error == 0) {
13742 tcp_account_for_send(tp, len, (rsm != NULL), doing_tlp, hw_tls);
13743 if (TCPS_HAVEESTABLISHED(tp->t_state) &&
13744 (tp->t_flags & TF_SACK_PERMIT) &&
13745 tp->rcv_numsacks > 0)
13746 tcp_clean_dsack_blocks(tp);
13747 /* We sent an ack clear the bbr_segs_rcvd count */
13748 bbr->output_error_seen = 0;
13749 bbr->oerror_cnt = 0;
13750 bbr->bbr_segs_rcvd = 0;
13751 if (len == 0)
13752 counter_u64_add(bbr_out_size[TCP_MSS_ACCT_SNDACK], 1);
13753 /* Do accounting for new sends */
13754 if ((len > 0) && (rsm == NULL)) {
13755 int idx;
13756 if (tp->snd_una == tp->snd_max) {
13757 /*
13758 * Special case to match google, when
13759 * nothing is in flight the delivered
13760 * time does get updated to the current
13761 * time (see tcp_rate_bsd.c).
13762 */
13763 bbr->r_ctl.rc_del_time = cts;
13764 }
13765 if (len >= maxseg) {
13766 idx = (len / maxseg) + 3;
13767 if (idx >= TCP_MSS_ACCT_ATIMER)
13768 counter_u64_add(bbr_out_size[(TCP_MSS_ACCT_ATIMER - 1)], 1);
13769 else
13770 counter_u64_add(bbr_out_size[idx], 1);
13771 } else {
13772 /* smaller than a MSS */
13773 idx = len / (bbr_hptsi_bytes_min - bbr->rc_last_options);
13774 if (idx >= TCP_MSS_SMALL_MAX_SIZE_DIV)
13775 idx = (TCP_MSS_SMALL_MAX_SIZE_DIV - 1);
13776 counter_u64_add(bbr_out_size[(idx + TCP_MSS_SMALL_SIZE_OFF)], 1);
13777 }
13778 }
13779 }
13780 abandon = 0;
13781 /*
13782 * We must do the send accounting before we log the output,
13783 * otherwise the state of the rsm could change and we account to the
13784 * wrong bucket.
13785 */
13786 if (len > 0) {
13787 bbr_do_send_accounting(tp, bbr, rsm, len, error);
13788 if (error == 0) {
13789 if (tp->snd_una == tp->snd_max)
13790 bbr->r_ctl.rc_tlp_rxt_last_time = cts;
13791 }
13792 }
13793 bbr_log_output(bbr, tp, &to, len, bbr_seq, (uint8_t) flags, error,
13794 cts, mb, &abandon, rsm, 0, sb);
13795 if (abandon) {
13796 /*
13797 * If bbr_log_output destroys the TCB or sees a TH_RST being
13798 * sent we should hit this condition.
13799 */
13800 return (0);
13801 }
13802 if (bbr->rc_in_persist == 0) {
13803 /*
13804 * Advance snd_nxt over sequence space of this segment.
13805 */
13806 if (error)
13807 /* We don't log or do anything with errors */
13808 goto skip_upd;
13809
13810 if (tp->snd_una == tp->snd_max &&
13811 (len || (flags & (TH_SYN | TH_FIN)))) {
13812 /*
13813 * Update the time we just added data since none was
13814 * outstanding.
13815 */
13816 bbr_log_progress_event(bbr, tp, ticks, PROGRESS_START, __LINE__);
13817 bbr->rc_tp->t_acktime = ticks;
13818 }
13819 if (flags & (TH_SYN | TH_FIN) && (rsm == NULL)) {
13820 if (flags & TH_SYN) {
13821 /*
13822 * Smack the snd_max to iss + 1
13823 * if its a FO we will add len below.
13824 */
13825 tp->snd_max = tp->iss + 1;
13826 }
13827 if ((flags & TH_FIN) && ((tp->t_flags & TF_SENTFIN) == 0)) {
13828 tp->snd_max++;
13829 tp->t_flags |= TF_SENTFIN;
13830 }
13831 }
13832 if (sack_rxmit == 0)
13833 tp->snd_max += len;
13834skip_upd:
13835 if ((error == 0) && len)
13836 tot_len += len;
13837 } else {
13838 /* Persists case */
13839 int32_t xlen = len;
13840
13841 if (error)
13842 goto nomore;
13843
13844 if (flags & TH_SYN)
13845 ++xlen;
13846 if ((flags & TH_FIN) && ((tp->t_flags & TF_SENTFIN) == 0)) {
13847 ++xlen;
13848 tp->t_flags |= TF_SENTFIN;
13849 }
13850 if (xlen && (tp->snd_una == tp->snd_max)) {
13851 /*
13852 * Update the time we just added data since none was
13853 * outstanding.
13854 */
13855 bbr_log_progress_event(bbr, tp, ticks, PROGRESS_START, __LINE__);
13856 bbr->rc_tp->t_acktime = ticks;
13857 }
13858 if (sack_rxmit == 0)
13859 tp->snd_max += xlen;
13860 tot_len += (len + optlen + ipoptlen);
13861 }
13862nomore:
13863 if (error) {
13864 /*
13865 * Failures do not advance the seq counter above. For the
13866 * case of ENOBUFS we will fall out and become ack-clocked.
13867 * capping the cwnd at the current flight.
13868 * Everything else will just have to retransmit with the timer
13869 * (no pacer).
13870 */
13871 SOCKBUF_UNLOCK_ASSERT(sb);
13872 BBR_STAT_INC(bbr_saw_oerr);
13873 /* Clear all delay/early tracks */
13874 bbr->r_ctl.rc_hptsi_agg_delay = 0;
13875 bbr->r_ctl.rc_agg_early = 0;
13876 bbr->r_agg_early_set = 0;
13877 bbr->output_error_seen = 1;
13878 if (bbr->oerror_cnt < 0xf)
13879 bbr->oerror_cnt++;
13880 if (bbr_max_net_error_cnt && (bbr->oerror_cnt >= bbr_max_net_error_cnt)) {
13881 /* drop the session */
13882 return (-ENETDOWN);
13883 }
13884 switch (error) {
13885 case ENOBUFS:
13886 /*
13887 * Make this guy have to get ack's to send
13888 * more but lets make sure we don't
13889 * slam him below a T-O (1MSS).
13890 */
13891 if (bbr->rc_bbr_state != BBR_STATE_PROBE_RTT) {
13892 tp->snd_cwnd = ctf_flight_size(tp, (bbr->r_ctl.rc_sacked +
13893 bbr->r_ctl.rc_lost_bytes)) - maxseg;
13894 if (tp->snd_cwnd < maxseg)
13895 tp->snd_cwnd = maxseg;
13896 }
13897 slot = (bbr_error_base_paceout + 1) << bbr->oerror_cnt;
13898 BBR_STAT_INC(bbr_saw_enobuf);
13899 if (bbr->bbr_hdrw_pacing)
13900 counter_u64_add(bbr_hdwr_pacing_enobuf, 1);
13901 else
13902 counter_u64_add(bbr_nohdwr_pacing_enobuf, 1);
13903 /*
13904 * Here even in the enobuf's case we want to do our
13905 * state update. The reason being we may have been
13906 * called by the input function. If so we have had
13907 * things change.
13908 */
13909 error = 0;
13910 goto enobufs;
13911 case EMSGSIZE:
13912 /*
13913 * For some reason the interface we used initially
13914 * to send segments changed to another or lowered
13915 * its MTU. If TSO was active we either got an
13916 * interface without TSO capabilits or TSO was
13917 * turned off. If we obtained mtu from ip_output()
13918 * then update it and try again.
13919 */
13920 /* Turn on tracing (or try to) */
13921 {
13922 int old_maxseg;
13923
13924 old_maxseg = tp->t_maxseg;
13925 BBR_STAT_INC(bbr_saw_emsgsiz);
13926 bbr_log_msgsize_fail(bbr, tp, len, maxseg, mtu, csum_flags, tso, cts);
13927 if (mtu != 0)
13928 tcp_mss_update(tp, -1, mtu, NULL, NULL);
13929 if (old_maxseg <= tp->t_maxseg) {
13930 /* Huh it did not shrink? */
13931 tp->t_maxseg = old_maxseg - 40;
13932 bbr_log_msgsize_fail(bbr, tp, len, maxseg, mtu, 0, tso, cts);
13933 }
13934 /*
13935 * Nuke all other things that can interfere
13936 * with slot
13937 */
13938 if ((tot_len + len) && (len >= tp->t_maxseg)) {
13939 slot = bbr_get_pacing_delay(bbr,
13940 bbr->r_ctl.rc_bbr_hptsi_gain,
13941 (tot_len + len), cts, 0);
13942 if (slot < bbr_error_base_paceout)
13943 slot = (bbr_error_base_paceout + 2) << bbr->oerror_cnt;
13944 } else
13945 slot = (bbr_error_base_paceout + 2) << bbr->oerror_cnt;
13946 bbr->rc_output_starts_timer = 1;
13947 bbr_start_hpts_timer(bbr, tp, cts, 10, slot,
13948 tot_len);
13949 return (error);
13950 }
13951 case EPERM:
13952 tp->t_softerror = error;
13953 /* Fall through */
13954 case EHOSTDOWN:
13955 case EHOSTUNREACH:
13956 case ENETDOWN:
13957 case ENETUNREACH:
13958 if (TCPS_HAVERCVDSYN(tp->t_state)) {
13959 tp->t_softerror = error;
13960 }
13961 /* FALLTHROUGH */
13962 default:
13963 slot = (bbr_error_base_paceout + 3) << bbr->oerror_cnt;
13964 bbr->rc_output_starts_timer = 1;
13965 bbr_start_hpts_timer(bbr, tp, cts, 11, slot, 0);
13966 return (error);
13967 }
13968#ifdef STATS
13969 } else if (((tp->t_flags & TF_GPUTINPROG) == 0) &&
13970 len &&
13971 (rsm == NULL) &&
13972 (bbr->rc_in_persist == 0)) {
13973 tp->gput_seq = bbr_seq;
13974 tp->gput_ack = bbr_seq +
13975 min(sbavail(&so->so_snd) - sb_offset, sendwin);
13976 tp->gput_ts = cts;
13977 tp->t_flags |= TF_GPUTINPROG;
13978#endif
13979 }
13980 KMOD_TCPSTAT_INC(tcps_sndtotal);
13981 if ((bbr->bbr_hdw_pace_ena) &&
13982 (bbr->bbr_attempt_hdwr_pace == 0) &&
13983 (bbr->rc_past_init_win) &&
13984 (bbr->rc_bbr_state != BBR_STATE_STARTUP) &&
13985 (get_filter_value(&bbr->r_ctl.rc_delrate)) &&
13986 (inp->inp_route.ro_nh &&
13987 inp->inp_route.ro_nh->nh_ifp)) {
13988 /*
13989 * We are past the initial window and
13990 * have at least one measurement so we
13991 * could use hardware pacing if its available.
13992 * We have an interface and we have not attempted
13993 * to setup hardware pacing, lets try to now.
13994 */
13995 uint64_t rate_wanted;
13996 int err = 0;
13997
13998 rate_wanted = bbr_get_hardware_rate(bbr);
13999 bbr->bbr_attempt_hdwr_pace = 1;
14000 bbr->r_ctl.crte = tcp_set_pacing_rate(bbr->rc_tp,
14001 inp->inp_route.ro_nh->nh_ifp,
14002 rate_wanted,
14003 (RS_PACING_GEQ|RS_PACING_SUB_OK),
14004 &err, NULL);
14005 if (bbr->r_ctl.crte) {
14006 bbr_type_log_hdwr_pacing(bbr,
14007 bbr->r_ctl.crte->ptbl->rs_ifp,
14008 rate_wanted,
14009 bbr->r_ctl.crte->rate,
14010 __LINE__, cts, err);
14011 BBR_STAT_INC(bbr_hdwr_rl_add_ok);
14012 counter_u64_add(bbr_flows_nohdwr_pacing, -1);
14013 counter_u64_add(bbr_flows_whdwr_pacing, 1);
14014 bbr->bbr_hdrw_pacing = 1;
14015 /* Now what is our gain status? */
14016 if (bbr->r_ctl.crte->rate < rate_wanted) {
14017 /* We have a problem */
14018 bbr_setup_less_of_rate(bbr, cts,
14019 bbr->r_ctl.crte->rate, rate_wanted);
14020 } else {
14021 /* We are good */
14022 bbr->gain_is_limited = 0;
14023 bbr->skip_gain = 0;
14024 }
14025 tcp_bbr_tso_size_check(bbr, cts);
14026 } else {
14027 bbr_type_log_hdwr_pacing(bbr,
14028 inp->inp_route.ro_nh->nh_ifp,
14029 rate_wanted,
14030 0,
14031 __LINE__, cts, err);
14032 BBR_STAT_INC(bbr_hdwr_rl_add_fail);
14033 }
14034 }
14035 if (bbr->bbr_hdrw_pacing) {
14036 /*
14037 * Worry about cases where the route
14038 * changes or something happened that we
14039 * lost our hardware pacing possibly during
14040 * the last ip_output call.
14041 */
14042 if (inp->inp_snd_tag == NULL) {
14043 /* A change during ip output disabled hw pacing? */
14044 bbr->bbr_hdrw_pacing = 0;
14045 } else if ((inp->inp_route.ro_nh == NULL) ||
14046 (inp->inp_route.ro_nh->nh_ifp != inp->inp_snd_tag->ifp)) {
14047 /*
14048 * We had an interface or route change,
14049 * detach from the current hdwr pacing
14050 * and setup to re-attempt next go
14051 * round.
14052 */
14053 bbr->bbr_hdrw_pacing = 0;
14054 bbr->bbr_attempt_hdwr_pace = 0;
14055 tcp_rel_pacing_rate(bbr->r_ctl.crte, bbr->rc_tp);
14056 tcp_bbr_tso_size_check(bbr, cts);
14057 }
14058 }
14059 /*
14060 * Data sent (as far as we can tell). If this advertises a larger
14061 * window than any other segment, then remember the size of the
14062 * advertised window. Any pending ACK has now been sent.
14063 */
14064 if (SEQ_GT(tp->rcv_nxt + recwin, tp->rcv_adv))
14065 tp->rcv_adv = tp->rcv_nxt + recwin;
14066
14067 tp->last_ack_sent = tp->rcv_nxt;
14068 if ((error == 0) &&
14069 (bbr->r_ctl.rc_pace_max_segs > tp->t_maxseg) &&
14070 (doing_tlp == 0) &&
14071 (tso == 0) &&
14072 (len > 0) &&
14073 ((flags & TH_RST) == 0) &&
14074 ((flags & TH_SYN) == 0) &&
14075 (IN_RECOVERY(tp->t_flags) == 0) &&
14076 (bbr->rc_in_persist == 0) &&
14077 (tot_len < bbr->r_ctl.rc_pace_max_segs)) {
14078 /*
14079 * For non-tso we need to goto again until we have sent out
14080 * enough data to match what we are hptsi out every hptsi
14081 * interval.
14082 */
14083 if (SEQ_LT(tp->snd_nxt, tp->snd_max)) {
14084 /* Make sure snd_nxt is drug up */
14085 tp->snd_nxt = tp->snd_max;
14086 }
14087 if (rsm != NULL) {
14088 rsm = NULL;
14089 goto skip_again;
14090 }
14091 rsm = NULL;
14092 sack_rxmit = 0;
14093 tp->t_flags &= ~(TF_ACKNOW | TF_DELACK);
14094 goto again;
14095 }
14096skip_again:
14097 if ((error == 0) && (flags & TH_FIN))
14098 tcp_log_end_status(tp, TCP_EI_STATUS_SERVER_FIN);
14099 if ((error == 0) && (flags & TH_RST))
14100 tcp_log_end_status(tp, TCP_EI_STATUS_SERVER_RST);
14101 if (((flags & (TH_RST | TH_SYN | TH_FIN)) == 0) && tot_len) {
14102 /*
14103 * Calculate/Re-Calculate the hptsi slot in usecs based on
14104 * what we have sent so far
14105 */
14106 slot = bbr_get_pacing_delay(bbr, bbr->r_ctl.rc_bbr_hptsi_gain, tot_len, cts, 0);
14107 if (bbr->rc_no_pacing)
14108 slot = 0;
14109 }
14110 tp->t_flags &= ~(TF_ACKNOW | TF_DELACK);
14111enobufs:
14112 if (bbr->rc_use_google == 0)
14113 bbr_check_bbr_for_state(bbr, cts, __LINE__, 0);
14114 bbr_cwnd_limiting(tp, bbr, ctf_flight_size(tp, (bbr->r_ctl.rc_sacked +
14115 bbr->r_ctl.rc_lost_bytes)));
14116 bbr->rc_output_starts_timer = 1;
14117 if (bbr->bbr_use_rack_cheat &&
14118 (more_to_rxt ||
14119 ((bbr->r_ctl.rc_resend = bbr_check_recovery_mode(tp, bbr, cts)) != NULL))) {
14120 /* Rack cheats and shotguns out all rxt's 1ms apart */
14121 if (slot > 1000)
14122 slot = 1000;
14123 }
14124 if (bbr->bbr_hdrw_pacing && (bbr->hw_pacing_set == 0)) {
14125 /*
14126 * We don't change the tso size until some number of sends
14127 * to give the hardware commands time to get down
14128 * to the interface.
14129 */
14130 bbr->r_ctl.bbr_hdwr_cnt_noset_snt++;
14131 if (bbr->r_ctl.bbr_hdwr_cnt_noset_snt >= bbr_hdwr_pacing_delay_cnt) {
14132 bbr->hw_pacing_set = 1;
14133 tcp_bbr_tso_size_check(bbr, cts);
14134 }
14135 }
14136 bbr_start_hpts_timer(bbr, tp, cts, 12, slot, tot_len);
14137 if (SEQ_LT(tp->snd_nxt, tp->snd_max)) {
14138 /* Make sure snd_nxt is drug up */
14139 tp->snd_nxt = tp->snd_max;
14140 }
14141 return (error);
14142
14143}
14144
14145/*
14146 * See bbr_output_wtime() for return values.
14147 */
14148static int
14149bbr_output(struct tcpcb *tp)
14150{
14151 int32_t ret;
14152 struct timeval tv;
14153
14154 NET_EPOCH_ASSERT();
14155
14156 INP_WLOCK_ASSERT(tp->t_inpcb);
14157 (void)tcp_get_usecs(&tv);
14158 ret = bbr_output_wtime(tp, &tv);
14159 return (ret);
14160}
14161
14162static void
14163bbr_mtu_chg(struct tcpcb *tp)
14164{
14165 struct tcp_bbr *bbr;
14166 struct bbr_sendmap *rsm, *frsm = NULL;
14167 uint32_t maxseg;
14168
14169 /*
14170 * The MTU has changed. a) Clear the sack filter. b) Mark everything
14171 * over the current size as SACK_PASS so a retransmit will occur.
14172 */
14173
14174 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
14175 maxseg = tp->t_maxseg - bbr->rc_last_options;
14176 sack_filter_clear(&bbr->r_ctl.bbr_sf, tp->snd_una);
14177 TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) {
14178 /* Don't mess with ones acked (by sack?) */
14179 if (rsm->r_flags & BBR_ACKED)
14180 continue;
14181 if ((rsm->r_end - rsm->r_start) > maxseg) {
14182 /*
14183 * We mark sack-passed on all the previous large
14184 * sends we did. This will force them to retransmit.
14185 */
14186 rsm->r_flags |= BBR_SACK_PASSED;
14187 if (((rsm->r_flags & BBR_MARKED_LOST) == 0) &&
14188 bbr_is_lost(bbr, rsm, bbr->r_ctl.rc_rcvtime)) {
14189 bbr->r_ctl.rc_lost_bytes += rsm->r_end - rsm->r_start;
14190 bbr->r_ctl.rc_lost += rsm->r_end - rsm->r_start;
14191 rsm->r_flags |= BBR_MARKED_LOST;
14192 }
14193 if (frsm == NULL)
14194 frsm = rsm;
14195 }
14196 }
14197 if (frsm) {
14198 bbr->r_ctl.rc_resend = frsm;
14199 }
14200}
14201
14202static int
14203bbr_pru_options(struct tcpcb *tp, int flags)
14204{
14205 if (flags & PRUS_OOB)
14206 return (EOPNOTSUPP);
14207 return (0);
14208}
14209
14210struct tcp_function_block __tcp_bbr = {
14211 .tfb_tcp_block_name = __XSTRING(STACKNAME),
14212 .tfb_tcp_output = bbr_output,
14213 .tfb_do_queued_segments = ctf_do_queued_segments,
14214 .tfb_do_segment_nounlock = bbr_do_segment_nounlock,
14215 .tfb_tcp_do_segment = bbr_do_segment,
14216 .tfb_tcp_ctloutput = bbr_ctloutput,
14217 .tfb_tcp_fb_init = bbr_init,
14218 .tfb_tcp_fb_fini = bbr_fini,
14219 .tfb_tcp_timer_stop_all = bbr_stopall,
14220 .tfb_tcp_timer_activate = bbr_timer_activate,
14221 .tfb_tcp_timer_active = bbr_timer_active,
14222 .tfb_tcp_timer_stop = bbr_timer_stop,
14223 .tfb_tcp_rexmit_tmr = bbr_remxt_tmr,
14224 .tfb_tcp_handoff_ok = bbr_handoff_ok,
14225 .tfb_tcp_mtu_chg = bbr_mtu_chg,
14226 .tfb_pru_options = bbr_pru_options,
14227 .tfb_flags = TCP_FUNC_OUTPUT_CANDROP,
14228};
14229
14230/*
14231 * bbr_ctloutput() must drop the inpcb lock before performing copyin on
14232 * socket option arguments. When it re-acquires the lock after the copy, it
14233 * has to revalidate that the connection is still valid for the socket
14234 * option.
14235 */
14236static int
14237bbr_set_sockopt(struct inpcb *inp, struct sockopt *sopt)
14238{
14239 struct epoch_tracker et;
14240 struct tcpcb *tp;
14241 struct tcp_bbr *bbr;
14242 int32_t error = 0, optval;
14243
14244 switch (sopt->sopt_level) {
14245 case IPPROTO_IPV6:
14246 case IPPROTO_IP:
14247 return (tcp_default_ctloutput(inp, sopt));
14248 }
14249
14250 switch (sopt->sopt_name) {
14251 case TCP_RACK_PACE_MAX_SEG:
14252 case TCP_RACK_MIN_TO:
14253 case TCP_RACK_REORD_THRESH:
14254 case TCP_RACK_REORD_FADE:
14255 case TCP_RACK_TLP_THRESH:
14256 case TCP_RACK_PKT_DELAY:
14257 case TCP_BBR_ALGORITHM:
14258 case TCP_BBR_TSLIMITS:
14259 case TCP_BBR_IWINTSO:
14260 case TCP_BBR_RECFORCE:
14261 case TCP_BBR_STARTUP_PG:
14262 case TCP_BBR_DRAIN_PG:
14263 case TCP_BBR_RWND_IS_APP:
14264 case TCP_BBR_PROBE_RTT_INT:
14265 case TCP_BBR_PROBE_RTT_GAIN:
14266 case TCP_BBR_PROBE_RTT_LEN:
14267 case TCP_BBR_STARTUP_LOSS_EXIT:
14268 case TCP_BBR_USEDEL_RATE:
14269 case TCP_BBR_MIN_RTO:
14270 case TCP_BBR_MAX_RTO:
14271 case TCP_BBR_PACE_PER_SEC:
14272 case TCP_DELACK:
14273 case TCP_BBR_PACE_DEL_TAR:
14274 case TCP_BBR_SEND_IWND_IN_TSO:
14275 case TCP_BBR_EXTRA_STATE:
14276 case TCP_BBR_UTTER_MAX_TSO:
14277 case TCP_BBR_MIN_TOPACEOUT:
14278 case TCP_BBR_FLOOR_MIN_TSO:
14279 case TCP_BBR_TSTMP_RAISES:
14280 case TCP_BBR_POLICER_DETECT:
14281 case TCP_BBR_USE_RACK_CHEAT:
14282 case TCP_DATA_AFTER_CLOSE:
14283 case TCP_BBR_HDWR_PACE:
14284 case TCP_BBR_PACE_SEG_MAX:
14285 case TCP_BBR_PACE_SEG_MIN:
14286 case TCP_BBR_PACE_CROSS:
14287 case TCP_BBR_PACE_OH:
14288#ifdef NETFLIX_PEAKRATE
14289 case TCP_MAXPEAKRATE:
14290#endif
14291 case TCP_BBR_TMR_PACE_OH:
14292 case TCP_BBR_RACK_RTT_USE:
14293 case TCP_BBR_RETRAN_WTSO:
14294 break;
14295 default:
14296 return (tcp_default_ctloutput(inp, sopt));
14297 break;
14298 }
14299 INP_WUNLOCK(inp);
14300 error = sooptcopyin(sopt, &optval, sizeof(optval), sizeof(optval));
14301 if (error)
14302 return (error);
14303 INP_WLOCK(inp);
14304 if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) {
14305 INP_WUNLOCK(inp);
14306 return (ECONNRESET);
14307 }
14308 tp = intotcpcb(inp);
14309 if (tp->t_fb != &__tcp_bbr) {
14310 INP_WUNLOCK(inp);
14311 return (ENOPROTOOPT);
14312 }
14313 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
14314 switch (sopt->sopt_name) {
14315 case TCP_BBR_PACE_PER_SEC:
14316 BBR_OPTS_INC(tcp_bbr_pace_per_sec);
14317 bbr->r_ctl.bbr_hptsi_per_second = optval;
14318 break;
14319 case TCP_BBR_PACE_DEL_TAR:
14320 BBR_OPTS_INC(tcp_bbr_pace_del_tar);
14321 bbr->r_ctl.bbr_hptsi_segments_delay_tar = optval;
14322 break;
14323 case TCP_BBR_PACE_SEG_MAX:
14324 BBR_OPTS_INC(tcp_bbr_pace_seg_max);
14325 bbr->r_ctl.bbr_hptsi_segments_max = optval;
14326 break;
14327 case TCP_BBR_PACE_SEG_MIN:
14328 BBR_OPTS_INC(tcp_bbr_pace_seg_min);
14329 bbr->r_ctl.bbr_hptsi_bytes_min = optval;
14330 break;
14331 case TCP_BBR_PACE_CROSS:
14332 BBR_OPTS_INC(tcp_bbr_pace_cross);
14333 bbr->r_ctl.bbr_cross_over = optval;
14334 break;
14335 case TCP_BBR_ALGORITHM:
14336 BBR_OPTS_INC(tcp_bbr_algorithm);
14337 if (optval && (bbr->rc_use_google == 0)) {
14338 /* Turn on the google mode */
14339 bbr_google_mode_on(bbr);
14340 if ((optval > 3) && (optval < 500)) {
14341 /*
14342 * Must be at least greater than .3%
14343 * and must be less than 50.0%.
14344 */
14345 bbr->r_ctl.bbr_google_discount = optval;
14346 }
14347 } else if ((optval == 0) && (bbr->rc_use_google == 1)) {
14348 /* Turn off the google mode */
14349 bbr_google_mode_off(bbr);
14350 }
14351 break;
14352 case TCP_BBR_TSLIMITS:
14353 BBR_OPTS_INC(tcp_bbr_tslimits);
14354 if (optval == 1)
14355 bbr->rc_use_ts_limit = 1;
14356 else if (optval == 0)
14357 bbr->rc_use_ts_limit = 0;
14358 else
14359 error = EINVAL;
14360 break;
14361
14362 case TCP_BBR_IWINTSO:
14363 BBR_OPTS_INC(tcp_bbr_iwintso);
14364 if ((optval >= 0) && (optval < 128)) {
14365 uint32_t twin;
14366
14367 bbr->rc_init_win = optval;
14368 twin = bbr_initial_cwnd(bbr, tp);
14369 if ((bbr->rc_past_init_win == 0) && (twin > tp->snd_cwnd))
14370 tp->snd_cwnd = twin;
14371 else
14372 error = EBUSY;
14373 } else
14374 error = EINVAL;
14375 break;
14376 case TCP_BBR_STARTUP_PG:
14377 BBR_OPTS_INC(tcp_bbr_startup_pg);
14378 if ((optval > 0) && (optval < BBR_MAX_GAIN_VALUE)) {
14379 bbr->r_ctl.rc_startup_pg = optval;
14380 if (bbr->rc_bbr_state == BBR_STATE_STARTUP) {
14381 bbr->r_ctl.rc_bbr_hptsi_gain = optval;
14382 }
14383 } else
14384 error = EINVAL;
14385 break;
14386 case TCP_BBR_DRAIN_PG:
14387 BBR_OPTS_INC(tcp_bbr_drain_pg);
14388 if ((optval > 0) && (optval < BBR_MAX_GAIN_VALUE))
14389 bbr->r_ctl.rc_drain_pg = optval;
14390 else
14391 error = EINVAL;
14392 break;
14393 case TCP_BBR_PROBE_RTT_LEN:
14394 BBR_OPTS_INC(tcp_bbr_probertt_len);
14395 if (optval <= 1)
14396 reset_time_small(&bbr->r_ctl.rc_rttprop, (optval * USECS_IN_SECOND));
14397 else
14398 error = EINVAL;
14399 break;
14400 case TCP_BBR_PROBE_RTT_GAIN:
14401 BBR_OPTS_INC(tcp_bbr_probertt_gain);
14402 if (optval <= BBR_UNIT)
14403 bbr->r_ctl.bbr_rttprobe_gain_val = optval;
14404 else
14405 error = EINVAL;
14406 break;
14407 case TCP_BBR_PROBE_RTT_INT:
14408 BBR_OPTS_INC(tcp_bbr_probe_rtt_int);
14409 if (optval > 1000)
14410 bbr->r_ctl.rc_probertt_int = optval;
14411 else
14412 error = EINVAL;
14413 break;
14414 case TCP_BBR_MIN_TOPACEOUT:
14415 BBR_OPTS_INC(tcp_bbr_topaceout);
14416 if (optval == 0) {
14417 bbr->no_pacing_until = 0;
14418 bbr->rc_no_pacing = 0;
14419 } else if (optval <= 0x00ff) {
14420 bbr->no_pacing_until = optval;
14421 if ((bbr->r_ctl.rc_pkt_epoch < bbr->no_pacing_until) &&
14422 (bbr->rc_bbr_state == BBR_STATE_STARTUP)){
14423 /* Turn on no pacing */
14424 bbr->rc_no_pacing = 1;
14425 }
14426 } else
14427 error = EINVAL;
14428 break;
14429 case TCP_BBR_STARTUP_LOSS_EXIT:
14430 BBR_OPTS_INC(tcp_bbr_startup_loss_exit);
14431 bbr->rc_loss_exit = optval;
14432 break;
14433 case TCP_BBR_USEDEL_RATE:
14434 error = EINVAL;
14435 break;
14436 case TCP_BBR_MIN_RTO:
14437 BBR_OPTS_INC(tcp_bbr_min_rto);
14438 bbr->r_ctl.rc_min_rto_ms = optval;
14439 break;
14440 case TCP_BBR_MAX_RTO:
14441 BBR_OPTS_INC(tcp_bbr_max_rto);
14442 bbr->rc_max_rto_sec = optval;
14443 break;
14444 case TCP_RACK_MIN_TO:
14445 /* Minimum time between rack t-o's in ms */
14446 BBR_OPTS_INC(tcp_rack_min_to);
14447 bbr->r_ctl.rc_min_to = optval;
14448 break;
14449 case TCP_RACK_REORD_THRESH:
14450 /* RACK reorder threshold (shift amount) */
14451 BBR_OPTS_INC(tcp_rack_reord_thresh);
14452 if ((optval > 0) && (optval < 31))
14453 bbr->r_ctl.rc_reorder_shift = optval;
14454 else
14455 error = EINVAL;
14456 break;
14457 case TCP_RACK_REORD_FADE:
14458 /* Does reordering fade after ms time */
14459 BBR_OPTS_INC(tcp_rack_reord_fade);
14460 bbr->r_ctl.rc_reorder_fade = optval;
14461 break;
14462 case TCP_RACK_TLP_THRESH:
14463 /* RACK TLP theshold i.e. srtt+(srtt/N) */
14464 BBR_OPTS_INC(tcp_rack_tlp_thresh);
14465 if (optval)
14466 bbr->rc_tlp_threshold = optval;
14467 else
14468 error = EINVAL;
14469 break;
14470 case TCP_BBR_USE_RACK_CHEAT:
14471 BBR_OPTS_INC(tcp_use_rackcheat);
14472 if (bbr->rc_use_google) {
14473 error = EINVAL;
14474 break;
14475 }
14476 BBR_OPTS_INC(tcp_rack_cheat);
14477 if (optval)
14478 bbr->bbr_use_rack_cheat = 1;
14479 else
14480 bbr->bbr_use_rack_cheat = 0;
14481 break;
14482 case TCP_BBR_FLOOR_MIN_TSO:
14483 BBR_OPTS_INC(tcp_utter_max_tso);
14484 if ((optval >= 0) && (optval < 40))
14485 bbr->r_ctl.bbr_hptsi_segments_floor = optval;
14486 else
14487 error = EINVAL;
14488 break;
14489 case TCP_BBR_UTTER_MAX_TSO:
14490 BBR_OPTS_INC(tcp_utter_max_tso);
14491 if ((optval >= 0) && (optval < 0xffff))
14492 bbr->r_ctl.bbr_utter_max = optval;
14493 else
14494 error = EINVAL;
14495 break;
14496
14497 case TCP_BBR_EXTRA_STATE:
14498 BBR_OPTS_INC(tcp_extra_state);
14499 if (optval)
14500 bbr->rc_use_idle_restart = 1;
14501 else
14502 bbr->rc_use_idle_restart = 0;
14503 break;
14504 case TCP_BBR_SEND_IWND_IN_TSO:
14505 BBR_OPTS_INC(tcp_iwnd_tso);
14506 if (optval) {
14507 bbr->bbr_init_win_cheat = 1;
14508 if (bbr->rc_past_init_win == 0) {
14509 uint32_t cts;
14510 cts = tcp_get_usecs(&bbr->rc_tv);
14511 tcp_bbr_tso_size_check(bbr, cts);
14512 }
14513 } else
14514 bbr->bbr_init_win_cheat = 0;
14515 break;
14516 case TCP_BBR_HDWR_PACE:
14517 BBR_OPTS_INC(tcp_hdwr_pacing);
14518 if (optval){
14519 bbr->bbr_hdw_pace_ena = 1;
14520 bbr->bbr_attempt_hdwr_pace = 0;
14521 } else {
14522 bbr->bbr_hdw_pace_ena = 0;
14523#ifdef RATELIMIT
14524 if (bbr->r_ctl.crte != NULL) {
14525 tcp_rel_pacing_rate(bbr->r_ctl.crte, tp);
14526 bbr->r_ctl.crte = NULL;
14527 }
14528#endif
14529 }
14530 break;
14531
14532 case TCP_DELACK:
14533 BBR_OPTS_INC(tcp_delack);
14534 if (optval < 100) {
14535 if (optval == 0) /* off */
14536 tp->t_delayed_ack = 0;
14537 else if (optval == 1) /* on which is 2 */
14538 tp->t_delayed_ack = 2;
14539 else /* higher than 2 and less than 100 */
14540 tp->t_delayed_ack = optval;
14541 if (tp->t_flags & TF_DELACK) {
14542 tp->t_flags &= ~TF_DELACK;
14543 tp->t_flags |= TF_ACKNOW;
14544 NET_EPOCH_ENTER(et);
14545 bbr_output(tp);
14546 NET_EPOCH_EXIT(et);
14547 }
14548 } else
14549 error = EINVAL;
14550 break;
14551 case TCP_RACK_PKT_DELAY:
14552 /* RACK added ms i.e. rack-rtt + reord + N */
14553 BBR_OPTS_INC(tcp_rack_pkt_delay);
14554 bbr->r_ctl.rc_pkt_delay = optval;
14555 break;
14556#ifdef NETFLIX_PEAKRATE
14557 case TCP_MAXPEAKRATE:
14558 BBR_OPTS_INC(tcp_maxpeak);
14559 error = tcp_set_maxpeakrate(tp, optval);
14560 if (!error)
14561 tp->t_peakrate_thr = tp->t_maxpeakrate;
14562 break;
14563#endif
14564 case TCP_BBR_RETRAN_WTSO:
14565 BBR_OPTS_INC(tcp_retran_wtso);
14566 if (optval)
14567 bbr->rc_resends_use_tso = 1;
14568 else
14569 bbr->rc_resends_use_tso = 0;
14570 break;
14571 case TCP_DATA_AFTER_CLOSE:
14572 BBR_OPTS_INC(tcp_data_ac);
14573 if (optval)
14574 bbr->rc_allow_data_af_clo = 1;
14575 else
14576 bbr->rc_allow_data_af_clo = 0;
14577 break;
14578 case TCP_BBR_POLICER_DETECT:
14579 BBR_OPTS_INC(tcp_policer_det);
14580 if (bbr->rc_use_google == 0)
14581 error = EINVAL;
14582 else if (optval)
14583 bbr->r_use_policer = 1;
14584 else
14585 bbr->r_use_policer = 0;
14586 break;
14587
14588 case TCP_BBR_TSTMP_RAISES:
14589 BBR_OPTS_INC(tcp_ts_raises);
14590 if (optval)
14591 bbr->ts_can_raise = 1;
14592 else
14593 bbr->ts_can_raise = 0;
14594 break;
14595 case TCP_BBR_TMR_PACE_OH:
14596 BBR_OPTS_INC(tcp_pacing_oh_tmr);
14597 if (bbr->rc_use_google) {
14598 error = EINVAL;
14599 } else {
14600 if (optval)
14601 bbr->r_ctl.rc_incr_tmrs = 1;
14602 else
14603 bbr->r_ctl.rc_incr_tmrs = 0;
14604 }
14605 break;
14606 case TCP_BBR_PACE_OH:
14607 BBR_OPTS_INC(tcp_pacing_oh);
14608 if (bbr->rc_use_google) {
14609 error = EINVAL;
14610 } else {
14611 if (optval > (BBR_INCL_TCP_OH|
14612 BBR_INCL_IP_OH|
14613 BBR_INCL_ENET_OH)) {
14614 error = EINVAL;
14615 break;
14616 }
14617 if (optval & BBR_INCL_TCP_OH)
14618 bbr->r_ctl.rc_inc_tcp_oh = 1;
14619 else
14620 bbr->r_ctl.rc_inc_tcp_oh = 0;
14621 if (optval & BBR_INCL_IP_OH)
14622 bbr->r_ctl.rc_inc_ip_oh = 1;
14623 else
14624 bbr->r_ctl.rc_inc_ip_oh = 0;
14625 if (optval & BBR_INCL_ENET_OH)
14626 bbr->r_ctl.rc_inc_enet_oh = 1;
14627 else
14628 bbr->r_ctl.rc_inc_enet_oh = 0;
14629 }
14630 break;
14631 default:
14632 return (tcp_default_ctloutput(inp, sopt));
14633 break;
14634 }
14635#ifdef NETFLIX_STATS
14636 tcp_log_socket_option(tp, sopt->sopt_name, optval, error);
14637#endif
14638 INP_WUNLOCK(inp);
14639 return (error);
14640}
14641
14642/*
14643 * return 0 on success, error-num on failure
14644 */
14645static int
14646bbr_get_sockopt(struct inpcb *inp, struct sockopt *sopt)
14647{
14648 struct tcpcb *tp;
14649 struct tcp_bbr *bbr;
14650 int32_t error, optval;
14651
14652 tp = intotcpcb(inp);
14653 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
14654 if (bbr == NULL) {
14655 INP_WUNLOCK(inp);
14656 return (EINVAL);
14657 }
14658 /*
14659 * Because all our options are either boolean or an int, we can just
14660 * pull everything into optval and then unlock and copy. If we ever
14661 * add a option that is not a int, then this will have quite an
14662 * impact to this routine.
14663 */
14664 switch (sopt->sopt_name) {
14665 case TCP_BBR_PACE_PER_SEC:
14666 optval = bbr->r_ctl.bbr_hptsi_per_second;
14667 break;
14668 case TCP_BBR_PACE_DEL_TAR:
14669 optval = bbr->r_ctl.bbr_hptsi_segments_delay_tar;
14670 break;
14671 case TCP_BBR_PACE_SEG_MAX:
14672 optval = bbr->r_ctl.bbr_hptsi_segments_max;
14673 break;
14674 case TCP_BBR_MIN_TOPACEOUT:
14675 optval = bbr->no_pacing_until;
14676 break;
14677 case TCP_BBR_PACE_SEG_MIN:
14678 optval = bbr->r_ctl.bbr_hptsi_bytes_min;
14679 break;
14680 case TCP_BBR_PACE_CROSS:
14681 optval = bbr->r_ctl.bbr_cross_over;
14682 break;
14683 case TCP_BBR_ALGORITHM:
14684 optval = bbr->rc_use_google;
14685 break;
14686 case TCP_BBR_TSLIMITS:
14687 optval = bbr->rc_use_ts_limit;
14688 break;
14689 case TCP_BBR_IWINTSO:
14690 optval = bbr->rc_init_win;
14691 break;
14692 case TCP_BBR_STARTUP_PG:
14693 optval = bbr->r_ctl.rc_startup_pg;
14694 break;
14695 case TCP_BBR_DRAIN_PG:
14696 optval = bbr->r_ctl.rc_drain_pg;
14697 break;
14698 case TCP_BBR_PROBE_RTT_INT:
14699 optval = bbr->r_ctl.rc_probertt_int;
14700 break;
14701 case TCP_BBR_PROBE_RTT_LEN:
14702 optval = (bbr->r_ctl.rc_rttprop.cur_time_limit / USECS_IN_SECOND);
14703 break;
14704 case TCP_BBR_PROBE_RTT_GAIN:
14705 optval = bbr->r_ctl.bbr_rttprobe_gain_val;
14706 break;
14707 case TCP_BBR_STARTUP_LOSS_EXIT:
14708 optval = bbr->rc_loss_exit;
14709 break;
14710 case TCP_BBR_USEDEL_RATE:
14711 error = EINVAL;
14712 break;
14713 case TCP_BBR_MIN_RTO:
14714 optval = bbr->r_ctl.rc_min_rto_ms;
14715 break;
14716 case TCP_BBR_MAX_RTO:
14717 optval = bbr->rc_max_rto_sec;
14718 break;
14719 case TCP_RACK_PACE_MAX_SEG:
14720 /* Max segments in a pace */
14721 optval = bbr->r_ctl.rc_pace_max_segs;
14722 break;
14723 case TCP_RACK_MIN_TO:
14724 /* Minimum time between rack t-o's in ms */
14725 optval = bbr->r_ctl.rc_min_to;
14726 break;
14727 case TCP_RACK_REORD_THRESH:
14728 /* RACK reorder threshold (shift amount) */
14729 optval = bbr->r_ctl.rc_reorder_shift;
14730 break;
14731 case TCP_RACK_REORD_FADE:
14732 /* Does reordering fade after ms time */
14733 optval = bbr->r_ctl.rc_reorder_fade;
14734 break;
14735 case TCP_BBR_USE_RACK_CHEAT:
14736 /* Do we use the rack cheat for rxt */
14737 optval = bbr->bbr_use_rack_cheat;
14738 break;
14739 case TCP_BBR_FLOOR_MIN_TSO:
14740 optval = bbr->r_ctl.bbr_hptsi_segments_floor;
14741 break;
14742 case TCP_BBR_UTTER_MAX_TSO:
14743 optval = bbr->r_ctl.bbr_utter_max;
14744 break;
14745 case TCP_BBR_SEND_IWND_IN_TSO:
14746 /* Do we send TSO size segments initially */
14747 optval = bbr->bbr_init_win_cheat;
14748 break;
14749 case TCP_BBR_EXTRA_STATE:
14750 optval = bbr->rc_use_idle_restart;
14751 break;
14752 case TCP_RACK_TLP_THRESH:
14753 /* RACK TLP theshold i.e. srtt+(srtt/N) */
14754 optval = bbr->rc_tlp_threshold;
14755 break;
14756 case TCP_RACK_PKT_DELAY:
14757 /* RACK added ms i.e. rack-rtt + reord + N */
14758 optval = bbr->r_ctl.rc_pkt_delay;
14759 break;
14760 case TCP_BBR_RETRAN_WTSO:
14761 optval = bbr->rc_resends_use_tso;
14762 break;
14763 case TCP_DATA_AFTER_CLOSE:
14764 optval = bbr->rc_allow_data_af_clo;
14765 break;
14766 case TCP_DELACK:
14767 optval = tp->t_delayed_ack;
14768 break;
14769 case TCP_BBR_HDWR_PACE:
14770 optval = bbr->bbr_hdw_pace_ena;
14771 break;
14772 case TCP_BBR_POLICER_DETECT:
14773 optval = bbr->r_use_policer;
14774 break;
14775 case TCP_BBR_TSTMP_RAISES:
14776 optval = bbr->ts_can_raise;
14777 break;
14778 case TCP_BBR_TMR_PACE_OH:
14779 optval = bbr->r_ctl.rc_incr_tmrs;
14780 break;
14781 case TCP_BBR_PACE_OH:
14782 optval = 0;
14783 if (bbr->r_ctl.rc_inc_tcp_oh)
14784 optval |= BBR_INCL_TCP_OH;
14785 if (bbr->r_ctl.rc_inc_ip_oh)
14786 optval |= BBR_INCL_IP_OH;
14787 if (bbr->r_ctl.rc_inc_enet_oh)
14788 optval |= BBR_INCL_ENET_OH;
14789 break;
14790 default:
14791 return (tcp_default_ctloutput(inp, sopt));
14792 break;
14793 }
14794 INP_WUNLOCK(inp);
14795 error = sooptcopyout(sopt, &optval, sizeof optval);
14796 return (error);
14797}
14798
14799/*
14800 * return 0 on success, error-num on failure
14801 */
14802static int
14803bbr_ctloutput(struct inpcb *inp, struct sockopt *sopt)
14804{
14805 if (sopt->sopt_dir == SOPT_SET) {
14806 return (bbr_set_sockopt(inp, sopt));
14807 } else if (sopt->sopt_dir == SOPT_GET) {
14808 return (bbr_get_sockopt(inp, sopt));
14809 } else {
14810 panic("%s: sopt_dir $%d", __func__, sopt->sopt_dir);
14811 }
14812}
14813
14814static const char *bbr_stack_names[] = {
14815 __XSTRING(STACKNAME),
14816#ifdef STACKALIAS
14817 __XSTRING(STACKALIAS),
14818#endif
14819};
14820
14821static bool bbr_mod_inited = false;
14822
14823static int
14824tcp_addbbr(module_t mod, int32_t type, void *data)
14825{
14826 int32_t err = 0;
14827 int num_stacks;
14828
14829 switch (type) {
14830 case MOD_LOAD:
14831 printf("Attempting to load " __XSTRING(MODNAME) "\n");
14832 bbr_zone = uma_zcreate(__XSTRING(MODNAME) "_map",
14833 sizeof(struct bbr_sendmap),
14834 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
14835 bbr_pcb_zone = uma_zcreate(__XSTRING(MODNAME) "_pcb",
14836 sizeof(struct tcp_bbr),
14837 NULL, NULL, NULL, NULL, UMA_ALIGN_CACHE, 0);
14838 sysctl_ctx_init(&bbr_sysctl_ctx);
14839 bbr_sysctl_root = SYSCTL_ADD_NODE(&bbr_sysctl_ctx,
14840 SYSCTL_STATIC_CHILDREN(_net_inet_tcp),
14841 OID_AUTO,
14842#ifdef STACKALIAS
14843 __XSTRING(STACKALIAS),
14844#else
14845 __XSTRING(STACKNAME),
14846#endif
14847 CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
14848 "");
14849 if (bbr_sysctl_root == NULL) {
14850 printf("Failed to add sysctl node\n");
14851 err = EFAULT;
14852 goto free_uma;
14853 }
14854 bbr_init_sysctls();
14855 num_stacks = nitems(bbr_stack_names);
14856 err = register_tcp_functions_as_names(&__tcp_bbr, M_WAITOK,
14857 bbr_stack_names, &num_stacks);
14858 if (err) {
14859 printf("Failed to register %s stack name for "
14860 "%s module\n", bbr_stack_names[num_stacks],
14861 __XSTRING(MODNAME));
14862 sysctl_ctx_free(&bbr_sysctl_ctx);
14863 free_uma:
14864 uma_zdestroy(bbr_zone);
14865 uma_zdestroy(bbr_pcb_zone);
14866 bbr_counter_destroy();
14867 printf("Failed to register " __XSTRING(MODNAME)
14868 " module err:%d\n", err);
14869 return (err);
14870 }
14871 tcp_lro_reg_mbufq();
14872 bbr_mod_inited = true;
14873 printf(__XSTRING(MODNAME) " is now available\n");
14874 break;
14875 case MOD_QUIESCE:
14876 err = deregister_tcp_functions(&__tcp_bbr, true, false);
14877 break;
14878 case MOD_UNLOAD:
14879 err = deregister_tcp_functions(&__tcp_bbr, false, true);
14880 if (err == EBUSY)
14881 break;
14882 if (bbr_mod_inited) {
14883 uma_zdestroy(bbr_zone);
14884 uma_zdestroy(bbr_pcb_zone);
14885 sysctl_ctx_free(&bbr_sysctl_ctx);
14886 bbr_counter_destroy();
14887 printf(__XSTRING(MODNAME)
14888 " is now no longer available\n");
14889 bbr_mod_inited = false;
14890 }
14891 tcp_lro_dereg_mbufq();
14892 err = 0;
14893 break;
14894 default:
14895 return (EOPNOTSUPP);
14896 }
14897 return (err);
14898}
14899
14900static moduledata_t tcp_bbr = {
14901 .name = __XSTRING(MODNAME),
14902 .evhand = tcp_addbbr,
14903 .priv = 0
14904};
14905
14906MODULE_VERSION(MODNAME, 1);
14907DECLARE_MODULE(MODNAME, tcp_bbr, SI_SUB_PROTO_DOMAIN, SI_ORDER_ANY);
14908MODULE_DEPEND(MODNAME, tcphpts, 1, 1, 1);
bbr_calc_time
static uint32_t bbr_calc_time(uint32_t cts, uint32_t earlier_time)
Definition: bbr.c:1107
tcp_bbr_xmit_timer_commit
static void tcp_bbr_xmit_timer_commit(struct tcp_bbr *bbr, struct tcpcb *tp, uint32_t cts)
Definition: bbr.c:6282
bbr_timer_activate
static void bbr_timer_activate(struct tcpcb *tp, uint32_t timer_type, uint32_t delta)
Definition: bbr.c:5311
bbr_what_can_we_send
static uint32_t bbr_what_can_we_send(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t sendwin, uint32_t avail, int32_t sb_offset, uint32_t cts)
Definition: bbr.c:11727
bbr_timer_stop
static void bbr_timer_stop(struct tcpcb *tp, uint32_t timer_type)
Definition: bbr.c:5292
tcp_bbr_xmit_timer
static void tcp_bbr_xmit_timer(struct tcp_bbr *bbr, uint32_t rtt_usecs, uint32_t rsm_send_time, uint32_t r_start, uint32_t tsin)
Definition: bbr.c:6143
bbr_do_syn_recv
static int bbr_do_syn_recv(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos)
Definition: bbr.c:8990
bbr_hptsi_utter_max
static int32_t bbr_hptsi_utter_max
Definition: bbr.c:401
bbr_hptsi_segments_delay_tar
static int32_t bbr_hptsi_segments_delay_tar
Definition: bbr.c:383
bbr_handoff_ok
static int bbr_handoff_ok(struct tcpcb *tp)
Definition: bbr.c:10225
bbr_include_tcp_oh
static int32_t bbr_include_tcp_oh
Definition: bbr.c:388
bbr_un_collapse_window
static void bbr_un_collapse_window(struct tcp_bbr *bbr)
Definition: bbr.c:8119
bbr_calc_thresh_rack
static uint32_t bbr_calc_thresh_rack(struct tcp_bbr *bbr, uint32_t srtt, uint32_t cts, struct bbr_sendmap *rsm)
Definition: bbr.c:4091
bbr_get_earliest_send_outstanding
static uint32_t bbr_get_earliest_send_outstanding(struct tcp_bbr *bbr, struct bbr_sendmap *u_rsm, uint32_t cts)
Definition: bbr.c:5326
bbr_stat_arry
counter_u64_t bbr_stat_arry[BBR_STAT_SIZE]
Definition: bbr.c:429
bbr_reorder_thresh
static int32_t bbr_reorder_thresh
Definition: bbr.c:336
bbr_log_enobuf_jmp
static void bbr_log_enobuf_jmp(struct tcp_bbr *bbr, uint32_t len, uint32_t cts, int32_t line, uint32_t o_len, uint32_t segcnt, uint32_t segsiz)
Definition: bbr.c:2317
bbr_persist_max
static int32_t bbr_persist_max
Definition: bbr.c:155
bbr_startup_loss_thresh
static int32_t bbr_startup_loss_thresh
Definition: bbr.c:185
bbr_type_log_hdwr_pacing
static void bbr_type_log_hdwr_pacing(struct tcp_bbr *bbr, const struct ifnet *ifp, uint64_t rate, uint64_t hw_rate, int line, uint32_t cts, int error)
Definition: bbr.c:2725
bbr_alloc_full_limit
static struct bbr_sendmap * bbr_alloc_full_limit(struct tcp_bbr *bbr)
Definition: bbr.c:3311
bbr_start_hpts_timer
static void bbr_start_hpts_timer(struct tcp_bbr *bbr, struct tcpcb *tp, uint32_t cts, int32_t frm, int32_t slot, uint32_t tot_len)
Definition: bbr.c:737
bbr_resends_use_tso
static int32_t bbr_resends_use_tso
Definition: bbr.c:421
DELAY_ACK
#define DELAY_ACK(tp, bbr, nsegs)
Definition: bbr.c:4032
bbr_update_bbr_info
static void bbr_update_bbr_info(struct tcp_bbr *bbr, struct bbr_sendmap *rsm, uint32_t rtt, uint32_t cts, uint32_t tsin, uint32_t uts, int32_t match, uint32_t rsm_send_time, int32_t ack_type, struct tcpopt *to)
Definition: bbr.c:6685
bbr_do_close_wait
static int bbr_do_close_wait(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos)
Definition: bbr.c:9342
get_min_cwnd
static uint32_t get_min_cwnd(struct tcp_bbr *bbr)
Definition: bbr.c:531
bbr_hptsi_segments_floor
static int32_t bbr_hptsi_segments_floor
Definition: bbr.c:400
bbr_sends_full_iwnd
int32_t bbr_sends_full_iwnd
Definition: bbr.c:291
bbr_target_is_bbunit
static int32_t bbr_target_is_bbunit
Definition: bbr.c:176
bbr_initial_bw_bps
static int32_t bbr_initial_bw_bps
Definition: bbr.c:278
bbr_window_update_needed
static int bbr_window_update_needed(struct tcpcb *tp, struct socket *so, uint32_t recwin, int32_t maxseg)
Definition: bbr.c:11858
bbr_cwnd_min_val_hs
static int32_t bbr_cwnd_min_val_hs
Definition: bbr.c:220
bbr_clone_rsm
static __inline void bbr_clone_rsm(struct tcp_bbr *bbr, struct bbr_sendmap *nrsm, struct bbr_sendmap *rsm, uint32_t start)
Definition: bbr.c:4407
bbr_fill_in_logging_data
static __inline void bbr_fill_in_logging_data(struct tcp_bbr *bbr, struct tcp_log_bbr *l, uint32_t cts)
Definition: bbr.c:1866
bbr_get_bw
static uint64_t bbr_get_bw(struct tcp_bbr *bbr)
Definition: bbr.c:3021
bbr_post_recovery
static void bbr_post_recovery(struct tcpcb *tp)
Definition: bbr.c:3777
bbr_hptsi_gain
static int16_t bbr_hptsi_gain[]
Definition: bbr.c:280
bbr_get_raw_target_cwnd
static uint32_t bbr_get_raw_target_cwnd(struct tcp_bbr *bbr, uint32_t gain, uint64_t bw)
Definition: bbr.c:3436
bbr_cwnd_limiting
static void bbr_cwnd_limiting(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t in_level)
Definition: bbr.c:11838
bbr_log_type_bbrrttprop
static void bbr_log_type_bbrrttprop(struct tcp_bbr *bbr, uint32_t t, uint32_t end, uint32_t tsconv, uint32_t cts, int32_t match, uint32_t seq, uint8_t flags)
Definition: bbr.c:2778
bbr_set_state_target
static void bbr_set_state_target(struct tcp_bbr *bbr, int line)
Definition: bbr.c:10683
bbr_ts_can_raise
static int32_t bbr_ts_can_raise
Definition: bbr.c:170
bbr_gain_gets_extra_too
static int32_t bbr_gain_gets_extra_too
Definition: bbr.c:223
bbr_log_hpts_diag
static void bbr_log_hpts_diag(struct tcp_bbr *bbr, uint32_t cts, struct hpts_diag *diag)
Definition: bbr.c:2410
bbr_pick_probebw_substate
static uint8_t bbr_pick_probebw_substate(struct tcp_bbr *bbr, uint32_t cts)
Definition: bbr.c:3102
bbr_max_net_error_cnt
static int32_t bbr_max_net_error_cnt
Definition: bbr.c:164
bbr_lt_bw_max_rtts
static uint32_t bbr_lt_bw_max_rtts
Definition: bbr.c:353
bbr_log_type_ltbw
static void bbr_log_type_ltbw(struct tcp_bbr *bbr, uint32_t cts, int32_t reason, uint32_t newbw, uint32_t obw, uint32_t diff, uint32_t tim)
Definition: bbr.c:2675
bbr_do_fin_wait_2
static int bbr_do_fin_wait_2(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos)
Definition: bbr.c:9815
bbr_pcb_zone
uma_zone_t bbr_pcb_zone
Definition: bbr.c:135
bbr_process_ack
static int bbr_process_ack(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp, struct tcpopt *to, uint32_t tiwin, int32_t tlen, int32_t *ofia, int32_t thflags, int32_t *ret_val)
Definition: bbr.c:7723
bbr_init
static int bbr_init(struct tcpcb *tp)
Definition: bbr.c:10008
bbr_state_change
static void bbr_state_change(struct tcp_bbr *bbr, uint32_t cts, int32_t epoch, int32_t pkt_epoch, uint32_t losses)
Definition: bbr.c:11114
bbr_no_pacing_until
static int32_t bbr_no_pacing_until
Definition: bbr.c:206
bbr_log_pkt_epoch
static void bbr_log_pkt_epoch(struct tcp_bbr *bbr, uint32_t cts, uint32_t line, uint32_t lost, uint32_t del)
Definition: bbr.c:2018
bbr_should_enter_probe_rtt
static int32_t bbr_should_enter_probe_rtt(struct tcp_bbr *bbr, uint32_t cts)
Definition: bbr.c:10931
bbr_log_type_rsmclear
static void bbr_log_type_rsmclear(struct tcp_bbr *bbr, uint32_t cts, struct bbr_sendmap *rsm, uint32_t flags, uint32_t line)
Definition: bbr.c:2616
bbr_state_lost
counter_u64_t bbr_state_lost[BBR_MAX_STAT]
Definition: bbr.c:427
bbr_stack_names
static const char * bbr_stack_names[]
Definition: bbr.c:14814
bbr_do_syn_sent
static int bbr_do_syn_sent(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos)
Definition: bbr.c:8780
bbr_hptsi_max_mul
static int32_t bbr_hptsi_max_mul
Definition: bbr.c:186
bbr_log_to_event
static void bbr_log_to_event(struct tcp_bbr *bbr, uint32_t cts, int32_t to_num)
Definition: bbr.c:2359
bbr_drain_floor
static int32_t bbr_drain_floor
Definition: bbr.c:235
bbr_lt_intvl_min_rtts
static uint32_t bbr_lt_intvl_min_rtts
Definition: bbr.c:355
bbr_collapsed_window
static void bbr_collapsed_window(struct tcp_bbr *bbr)
Definition: bbr.c:8006
bbr_strike_dupack
static void bbr_strike_dupack(struct tcp_bbr *bbr)
Definition: bbr.c:7704
bbr_get_pacing_length
static uint32_t bbr_get_pacing_length(struct tcp_bbr *bbr, uint16_t gain, uint32_t useconds_time, uint64_t bw)
Definition: bbr.c:3535
bbr_can_force_probertt
static int32_t bbr_can_force_probertt
Definition: bbr.c:241
bbr_sysctl_root
struct sysctl_oid * bbr_sysctl_root
Definition: bbr.c:138
bbr_log_flowend
static void bbr_log_flowend(struct tcp_bbr *bbr)
Definition: bbr.c:1996
bbr_timeout_delack
static int bbr_timeout_delack(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts)
Definition: bbr.c:4728
bbr_get_hardware_rate
static uint64_t bbr_get_hardware_rate(struct tcp_bbr *bbr)
Definition: bbr.c:5494
bbr_can_use_ts_for_rtt
static int32_t bbr_can_use_ts_for_rtt
Definition: bbr.c:244
bbr_isit_a_pkt_epoch
static void bbr_isit_a_pkt_epoch(struct tcp_bbr *bbr, uint32_t cts, struct bbr_sendmap *rsm, int32_t line, int32_t cum_acked)
Definition: bbr.c:2910
bbr_reset_lt_bw_interval
static void bbr_reset_lt_bw_interval(struct tcp_bbr *bbr, uint32_t cts)
Definition: bbr.c:3030
bbr_log_exit_gain
static void bbr_log_exit_gain(struct tcp_bbr *bbr, uint32_t cts, int32_t entry_method)
Definition: bbr.c:2801
bbr_rto_max_sec
static int32_t bbr_rto_max_sec
Definition: bbr.c:369
bbr_sack_mergable
static int bbr_sack_mergable(struct bbr_sendmap *at, uint32_t start, uint32_t end)
Definition: bbr.c:4453
bbr_flows_nohdwr_pacing
counter_u64_t bbr_flows_nohdwr_pacing
Definition: bbr.c:433
bbr_log_timer_var
static void bbr_log_timer_var(struct tcp_bbr *bbr, int mode, uint32_t cts, uint32_t time_since_sent, uint32_t srtt, uint32_t thresh, uint32_t to)
Definition: bbr.c:2446
bbr_cwnd_may_shrink
static int32_t bbr_cwnd_may_shrink
Definition: bbr.c:156
bbr_policer_detection_enabled
static int32_t bbr_policer_detection_enabled
Definition: bbr.c:192
bbr_use_lower_gain_in_startup
static int32_t bbr_use_lower_gain_in_startup
Definition: bbr.c:231
bbr_flows_whdwr_pacing
counter_u64_t bbr_flows_whdwr_pacing
Definition: bbr.c:432
sysctl_bbr_clear_lost
static int sysctl_bbr_clear_lost(SYSCTL_HANDLER_ARGS)
Definition: bbr.c:1132
bbr_log_type_enter_rec
static void bbr_log_type_enter_rec(struct tcp_bbr *bbr, uint32_t seq)
Definition: bbr.c:1959
bbr_update_rsm
static void bbr_update_rsm(struct tcpcb *tp, struct tcp_bbr *bbr, struct bbr_sendmap *rsm, uint32_t cts, uint32_t pacing_time)
Definition: bbr.c:5337
bbr_idle_restart_threshold
static int32_t bbr_idle_restart_threshold
Definition: bbr.c:412
bbr_check_probe_rtt_limits
static void bbr_check_probe_rtt_limits(struct tcp_bbr *bbr, uint32_t cts)
Definition: bbr.c:10800
bbr_set_probebw_gains
static void bbr_set_probebw_gains(struct tcp_bbr *bbr, uint32_t cts, uint32_t losses)
Definition: bbr.c:10533
bbr_persist_min
static int32_t bbr_persist_min
Definition: bbr.c:154
bbr_slam_cwnd_in_main_drain
static int32_t bbr_slam_cwnd_in_main_drain
Definition: bbr.c:249
bbr_setup_less_of_rate
static void bbr_setup_less_of_rate(struct tcp_bbr *bbr, uint32_t cts, uint64_t act_rate, uint64_t rate_wanted)
Definition: bbr.c:5505
bbr_hdwr_pace_adjust
static int32_t bbr_hdwr_pace_adjust
Definition: bbr.c:416
bbr_red_scale
static int32_t bbr_red_scale
Definition: bbr.c:172
bbr_output
static int bbr_output(struct tcpcb *tp)
Definition: bbr.c:14149
bbr_cross_over
static uint32_t bbr_cross_over
Definition: bbr.c:408
bbr_red_mul
static int32_t bbr_red_mul
Definition: bbr.c:173
bbr_opts_arry
counter_u64_t bbr_opts_arry[BBR_OPTS_SIZE]
Definition: bbr.c:430
bbr_cwnd_gain
static int32_t bbr_cwnd_gain
Definition: bbr.c:274
bbr_hptsi_segments_max
static int32_t bbr_hptsi_segments_max
Definition: bbr.c:399
bbr_lt_bw_sampling
static void bbr_lt_bw_sampling(struct tcp_bbr *bbr, uint32_t cts, int32_t loss_detected)
Definition: bbr.c:3130
bbr_gain_to_target
static int32_t bbr_gain_to_target
Definition: bbr.c:222
bbr_get_sockopt
static int bbr_get_sockopt(struct inpcb *inp, struct sockopt *sopt)
Definition: bbr.c:14646
bbr_get_rtt
static __inline uint32_t bbr_get_rtt(struct tcp_bbr *bbr, int32_t rtt_type)
Definition: bbr.c:4243
bbr_state_resend
counter_u64_t bbr_state_resend[BBR_MAX_STAT]
Definition: bbr.c:428
bbr_log_ack_clear
static void bbr_log_ack_clear(struct tcp_bbr *bbr, uint32_t cts)
Definition: bbr.c:2223
bbr_do_fastnewdata
static int bbr_do_fastnewdata(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen, uint32_t tiwin, int32_t nxt_pkt)
Definition: bbr.c:8435
bbr_ctloutput
static int bbr_ctloutput(struct inpcb *inp, struct sockopt *sopt)
Definition: bbr.c:14803
bbr_log_startup_event
static void bbr_log_startup_event(struct tcp_bbr *bbr, uint32_t cts, uint32_t flex1, uint32_t flex2, uint32_t flex3, uint8_t reason)
Definition: bbr.c:2387
bbr_hdwr_pace_floor
static int32_t bbr_hdwr_pace_floor
Definition: bbr.c:417
bbr_include_enet_oh
static int32_t bbr_include_enet_oh
Definition: bbr.c:386
bbr_verbose_logging
static int32_t bbr_verbose_logging
Definition: bbr.c:361
bbr_num_pktepo_for_del_limit
static int32_t bbr_num_pktepo_for_del_limit
Definition: bbr.c:158
bbr_sack_not_required
static int32_t bbr_sack_not_required
Definition: bbr.c:277
bbr_timeout_keepalive
static int bbr_timeout_keepalive(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts)
Definition: bbr.c:4828
bbr_mod_inited
static bool bbr_mod_inited
Definition: bbr.c:14821
DECLARE_MODULE
DECLARE_MODULE(MODNAME, tcp_bbr, SI_SUB_PROTO_DOMAIN, SI_ORDER_ANY)
MODULE_DEPEND
MODULE_DEPEND(MODNAME, tcphpts, 1, 1, 1)
bbr_lt_bw_ratio
static uint64_t bbr_lt_bw_ratio
Definition: bbr.c:352
bbr_target_cwnd_mult_limit
static int32_t bbr_target_cwnd_mult_limit
Definition: bbr.c:212
bbr_drain_drop_div
static int32_t bbr_drain_drop_div
Definition: bbr.c:239
bbr_set_pktepoch
static void bbr_set_pktepoch(struct tcp_bbr *bbr, uint32_t cts, int32_t line)
Definition: bbr.c:2860
bbr_log_rtt_sample
static void bbr_log_rtt_sample(struct tcp_bbr *bbr, uint32_t rtt, uint32_t tsin)
Definition: bbr.c:2178
bbr_log_sack_passed
static void bbr_log_sack_passed(struct tcpcb *tp, struct tcp_bbr *bbr, struct bbr_sendmap *rsm)
Definition: bbr.c:6952
TCPT_RANGESET_NOSLOP
#define TCPT_RANGESET_NOSLOP(tv, value, tvmin, tvmax)
Definition: bbr.c:140
bbr_timer_active
static int bbr_timer_active(struct tcpcb *tp, uint32_t timer_type)
Definition: bbr.c:5320
bbr_delayed_ack_time
static int32_t bbr_delayed_ack_time
Definition: bbr.c:344
bbr_hptsi_per_second
static int32_t bbr_hptsi_per_second
Definition: bbr.c:375
bbr_google_startup
static int32_t bbr_google_startup(struct tcp_bbr *bbr, uint32_t cts, int32_t pkt_epoch)
Definition: bbr.c:10948
bbr_include_ip_oh
static int32_t bbr_include_ip_oh
Definition: bbr.c:387
bbr_exit_startup_at_loss
static int32_t bbr_exit_startup_at_loss
Definition: bbr.c:345
bbr_filter_len_sec
static int32_t bbr_filter_len_sec
Definition: bbr.c:250
bbr_log_doseg_done
static void bbr_log_doseg_done(struct tcp_bbr *bbr, uint32_t cts, int32_t nxt_pkt, int32_t did_out)
Definition: bbr.c:2293
bbr_log_type_tsosize
static void bbr_log_type_tsosize(struct tcp_bbr *bbr, uint32_t cts, uint32_t tsosz, uint32_t tls, uint32_t old_val, uint32_t maxseg, int hdwr)
Definition: bbr.c:2588
bbr_init_sysctls
static void bbr_init_sysctls(void)
Definition: bbr.c:1172
bbr_google_measurement
static void bbr_google_measurement(struct tcp_bbr *bbr, struct bbr_sendmap *rsm, uint32_t rtt, uint32_t cts)
Definition: bbr.c:6602
bbr_is_lost
static int bbr_is_lost(struct tcp_bbr *bbr, struct bbr_sendmap *rsm, uint32_t cts)
Definition: bbr.c:4291
bbr_log_type_bw_reduce
static void bbr_log_type_bw_reduce(struct tcp_bbr *bbr, int reason)
Definition: bbr.c:1892
bbr_sysctl_ctx
struct sysctl_ctx_list bbr_sysctl_ctx
Definition: bbr.c:137
bbr_nohdwr_pacing_enobuf
counter_u64_t bbr_nohdwr_pacing_enobuf
Definition: bbr.c:435
bbr_calc_thresh_tlp
static uint32_t bbr_calc_thresh_tlp(struct tcpcb *tp, struct tcp_bbr *bbr, struct bbr_sendmap *rsm, uint32_t srtt, uint32_t cts)
Definition: bbr.c:4173
bbr_google_mode_on
static void bbr_google_mode_on(struct tcp_bbr *bbr)
Definition: bbr.c:9946
bbr_tlp_min
static int32_t bbr_tlp_min
Definition: bbr.c:343
bbr_lt_bw_diff
static uint64_t bbr_lt_bw_diff
Definition: bbr.c:351
tcp_bbr_tso_size_check
static void tcp_bbr_tso_size_check(struct tcp_bbr *bbr, uint32_t cts)
Definition: bbr.c:5713
bbr_reorder_fade
static int32_t bbr_reorder_fade
Definition: bbr.c:337
MODULE_VERSION
MODULE_VERSION(MODNAME, 1)
bbr_peer_reneges
static void bbr_peer_reneges(struct tcp_bbr *bbr, struct bbr_sendmap *rsm, tcp_seq th_ack)
Definition: bbr.c:7232
bbr_tlp_max_resend
static int32_t bbr_tlp_max_resend
Definition: bbr.c:422
bbr_sub_drain_app_limit
static int32_t bbr_sub_drain_app_limit
Definition: bbr.c:246
bbr_fastack
static int bbr_fastack(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen, uint32_t tiwin, int32_t nxt_pkt, uint8_t iptos)
Definition: bbr.c:8585
bbr_do_lastack
static int bbr_do_lastack(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos)
Definition: bbr.c:9702
bbr_log_to_processing
static void bbr_log_to_processing(struct tcp_bbr *bbr, uint32_t cts, int32_t ret, int32_t timers, uint8_t hpts_calling)
Definition: bbr.c:2337
bbr_set_probebw_google_gains
static void bbr_set_probebw_google_gains(struct tcp_bbr *bbr, uint32_t cts, uint32_t losses)
Definition: bbr.c:10495
bbr_state_drain_2_tar
static int32_t bbr_state_drain_2_tar
Definition: bbr.c:393
bbr_state_val
static uint8_t bbr_state_val(struct tcp_bbr *bbr)
Definition: bbr.c:525
bbr_hardware_pacing_limit
static int32_t bbr_hardware_pacing_limit
Definition: bbr.c:159
bbr_drain_drop_mul
static int32_t bbr_drain_drop_mul
Definition: bbr.c:238
__tcp_bbr
struct tcp_function_block __tcp_bbr
Definition: bbr.c:14210
bbr_log_type_rwnd_collapse
static void bbr_log_type_rwnd_collapse(struct tcp_bbr *bbr, int seq, int mode, uint32_t count)
Definition: bbr.c:1915
bbr_timer_cancel
static void bbr_timer_cancel(struct tcp_bbr *bbr, int32_t line, uint32_t cts)
Definition: bbr.c:5253
bbr_pkt_delay
static int32_t bbr_pkt_delay
Definition: bbr.c:339
bbr_log_ack_event
static void bbr_log_ack_event(struct tcp_bbr *bbr, struct tcphdr *th, struct tcpopt *to, uint32_t tlen, uint16_t nsegs, uint32_t cts, int32_t nxt_pkt, struct mbuf *m)
Definition: bbr.c:2243
bbr_gain_adjust
static uint16_t bbr_gain_adjust(struct tcp_bbr *bbr, uint16_t gain)
Definition: bbr.c:3496
bbr_log_to_cancel
static void bbr_log_to_cancel(struct tcp_bbr *bbr, int32_t line, uint32_t cts, uint8_t hpts_removed)
Definition: bbr.c:2543
bbr_lt_loss_thresh
static int32_t bbr_lt_loss_thresh
Definition: bbr.c:358
bbr_drain_rtt
static int32_t bbr_drain_rtt
Definition: bbr.c:234
bbr_ignore_data_after_close
static int32_t bbr_ignore_data_after_close
Definition: bbr.c:279
bbr_clear_lost
int32_t bbr_clear_lost
Definition: bbr.c:1093
bbr_set_epoch
static void bbr_set_epoch(struct tcp_bbr *bbr, uint32_t cts, int32_t line)
Definition: bbr.c:2898
bbr_mtu_chg
static void bbr_mtu_chg(struct tcpcb *tp)
Definition: bbr.c:14163
bbr_setup_red_bw
static void bbr_setup_red_bw(struct tcp_bbr *bbr, uint32_t cts)
Definition: bbr.c:3921
bbr_cwndtarget_rtt_touse
static int32_t bbr_cwndtarget_rtt_touse
Definition: bbr.c:157
__bbr_get_bw
static uint64_t __bbr_get_bw(struct tcp_bbr *bbr)
Definition: bbr.c:2923
bbr_get_target_cwnd
static uint32_t bbr_get_target_cwnd(struct tcp_bbr *bbr, uint64_t bw, uint32_t gain)
Definition: bbr.c:3460
bbr_log_msgsize_fail
static void bbr_log_msgsize_fail(struct tcp_bbr *bbr, struct tcpcb *tp, uint32_t len, uint32_t maxseg, uint32_t mtu, int32_t csum_flags, int32_t tso, uint32_t cts)
Definition: bbr.c:1977
bbr_nf_measurement
static void bbr_nf_measurement(struct tcp_bbr *bbr, struct bbr_sendmap *rsm, uint32_t rtt, uint32_t cts)
Definition: bbr.c:6488
bbr_set_reduced_rtt
static void bbr_set_reduced_rtt(struct tcp_bbr *bbr, uint32_t cts, uint32_t line)
Definition: bbr.c:6435
bbr_substate_change
static void bbr_substate_change(struct tcp_bbr *bbr, uint32_t cts, int line, int dolog)
__FBSDID
__FBSDID("$FreeBSD$")
bbr_make_timestamp_determination
static void bbr_make_timestamp_determination(struct tcp_bbr *bbr)
Definition: bbr.c:6153
bbr_lt_bw_samp_done
static void bbr_lt_bw_samp_done(struct tcp_bbr *bbr, uint64_t bw, uint32_t cts, uint32_t timin)
Definition: bbr.c:3048
bbr_enter_probe_rtt
static void bbr_enter_probe_rtt(struct tcp_bbr *bbr, uint32_t cts, int32_t line)
Definition: bbr.c:10726
bbr_red_growth_restrict
static int32_t bbr_red_growth_restrict
Definition: bbr.c:175
bbr_do_send_accounting
static void bbr_do_send_accounting(struct tcpcb *tp, struct tcp_bbr *bbr, struct bbr_sendmap *rsm, int32_t len, int32_t error)
Definition: bbr.c:11773
bbr_hdwr_pacing_enobuf
counter_u64_t bbr_hdwr_pacing_enobuf
Definition: bbr.c:436
bbr_no_retran
static int32_t bbr_no_retran
Definition: bbr.c:161
bbr_log_set_of_state_target
static void bbr_log_set_of_state_target(struct tcp_bbr *bbr, uint32_t new_tar, int line, int meth)
Definition: bbr.c:2062
bbr_sack_block_limit
static int32_t bbr_sack_block_limit
Definition: bbr.c:423
bbr_set_state
static void bbr_set_state(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t win)
Definition: bbr.c:10305
bbr_get_header_oh
static uint32_t bbr_get_header_oh(struct tcp_bbr *bbr)
Definition: bbr.c:3504
bbr_reset_lt_bw_sampling
static void bbr_reset_lt_bw_sampling(struct tcp_bbr *bbr, uint32_t cts)
Definition: bbr.c:3039
bbr_update_hardware_pacing_rate
static void bbr_update_hardware_pacing_rate(struct tcp_bbr *bbr, uint32_t cts)
Definition: bbr.c:5529
bbr_state_startup
static int32_t bbr_state_startup(struct tcp_bbr *bbr, uint32_t cts, int32_t epoch, int32_t pkt_epoch)
Definition: bbr.c:10974
bbr_log_progress_event
static void bbr_log_progress_event(struct tcp_bbr *bbr, struct tcpcb *tp, uint32_t tick, int event, int line)
Definition: bbr.c:2705
bbr_start_exit
static int32_t bbr_start_exit
Definition: bbr.c:183
bbr_all_get_min
static int32_t bbr_all_get_min
Definition: bbr.c:405
bbr_min_usec_delta
static int32_t bbr_min_usec_delta
Definition: bbr.c:208
bbr_rtt_probe_limit
static uint32_t bbr_rtt_probe_limit
Definition: bbr.c:252
bbr_min_to
static int32_t bbr_min_to
Definition: bbr.c:340
bbr_do_error_accounting
static void bbr_do_error_accounting(struct tcpcb *tp, struct tcp_bbr *bbr, struct bbr_sendmap *rsm, int32_t len, int32_t error)
Definition: bbr.c:11764
bbr_rand_ot
static int32_t bbr_rand_ot
Definition: bbr.c:240
bbr_google_discount
static int32_t bbr_google_discount
Definition: bbr.c:389
bbr_randomize_extra_state_time
static void bbr_randomize_extra_state_time(struct tcp_bbr *bbr)
Definition: bbr.c:3087
bbr_do_closing
static int bbr_do_closing(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos)
Definition: bbr.c:9589
bbr_fini
static void bbr_fini(struct tcpcb *tp, int32_t tcb_is_purged)
Definition: bbr.c:10253
bbr_adjust_for_hw_pacing
static void bbr_adjust_for_hw_pacing(struct tcp_bbr *bbr, uint32_t cts)
Definition: bbr.c:5591
google_consider_lost
static int32_t google_consider_lost
Definition: bbr.c:237
bbr_log_syn
static void bbr_log_syn(struct tcpcb *tp, struct tcpopt *to)
Definition: bbr.c:7287
bbr_is_ratio
static int32_t bbr_is_ratio
Definition: bbr.c:245
bbr_ts_limiting
static int32_t bbr_ts_limiting
Definition: bbr.c:169
BBR_MAX_STAT
#define BBR_MAX_STAT
Definition: bbr.c:425
bbr_proc_sack_blk
static uint32_t bbr_proc_sack_blk(struct tcpcb *tp, struct tcp_bbr *bbr, struct sackblk *sack, struct tcpopt *to, struct bbr_sendmap **prsm, uint32_t cts)
Definition: bbr.c:6993
bbr_get_pacing_delay
static uint32_t bbr_get_pacing_delay(struct tcp_bbr *bbr, uint16_t gain, int32_t len, uint32_t cts, int nolog)
Definition: bbr.c:3556
bbr_min_measurements_req
static int32_t bbr_min_measurements_req
Definition: bbr.c:193
bbr_free
static void bbr_free(struct tcp_bbr *bbr, struct bbr_sendmap *rsm)
Definition: bbr.c:3354
bbr_find_high_nonack
static struct bbr_sendmap * bbr_find_high_nonack(struct tcp_bbr *bbr, struct bbr_sendmap *rsm)
Definition: bbr.c:4066
bbr_low_start_exit
static int32_t bbr_low_start_exit
Definition: bbr.c:184
bbr_sub_drain_slam_cwnd
static int32_t bbr_sub_drain_slam_cwnd
Definition: bbr.c:248
bbr_log_rtt_shrinks
static void bbr_log_rtt_shrinks(struct tcp_bbr *bbr, uint32_t cts, uint32_t applied, uint32_t rtt, uint32_t line, uint8_t is_start, uint16_t set)
Definition: bbr.c:2113
bbr_out_size
counter_u64_t bbr_out_size[TCP_MSS_ACCT_SIZE]
Definition: bbr.c:431
bbr_do_red
static int32_t bbr_do_red
Definition: bbr.c:171
bbr_drop_limit
static int32_t bbr_drop_limit
Definition: bbr.c:177
bbr_state_is_pkt_epoch
static int32_t bbr_state_is_pkt_epoch
Definition: bbr.c:392
bbr_red_div
static int32_t bbr_red_div
Definition: bbr.c:174
bbr_high_gain
static int32_t bbr_high_gain
Definition: bbr.c:229
bbr_error_base_paceout
static int32_t bbr_error_base_paceout
Definition: bbr.c:163
bbr_hdwr_pacing_delay_cnt
static int32_t bbr_hdwr_pacing_delay_cnt
Definition: bbr.c:418
bbr_counter_destroy
static void bbr_counter_destroy(void)
Definition: bbr.c:1850
bbr_state_time
counter_u64_t bbr_state_time[BBR_MAX_STAT]
Definition: bbr.c:426
bbr_log_to_start
static void bbr_log_to_start(struct tcp_bbr *bbr, uint32_t cts, uint32_t to, int32_t slot, uint8_t which)
Definition: bbr.c:2496
bbr_rttprobe_gain
static int32_t bbr_rttprobe_gain
Definition: bbr.c:261
bbr_timer_audit
static void bbr_timer_audit(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts, struct sockbuf *sb)
Definition: bbr.c:959
bbr_use_google_algo
static int32_t bbr_use_google_algo
Definition: bbr.c:168
bbr_google_mode_off
static void bbr_google_mode_off(struct tcp_bbr *bbr)
Definition: bbr.c:9966
bbr_log_type_cwndupd
static void bbr_log_type_cwndupd(struct tcp_bbr *bbr, uint32_t bytes_this_ack, uint32_t chg, uint32_t prev_acked, int32_t meth, uint32_t target, uint32_t th_ack, int32_t line)
Definition: bbr.c:2155
bbr_lt_intvl_fp
static int32_t bbr_lt_intvl_fp
Definition: bbr.c:357
bbr_timer_start
static uint32_t bbr_timer_start(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts)
Definition: bbr.c:562
bbr_update_entry
static uint32_t bbr_update_entry(struct tcpcb *tp, struct tcp_bbr *bbr, struct bbr_sendmap *rsm, uint32_t cts, int32_t *lenp, uint32_t pacing_time)
Definition: bbr.c:5432
bbr_timeout_tlp
static int bbr_timeout_tlp(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts)
Definition: bbr.c:4560
bbr_can_adjust_probertt
static int32_t bbr_can_adjust_probertt
Definition: bbr.c:242
bbr_do_fin_wait_1
static int bbr_do_fin_wait_1(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos)
Definition: bbr.c:9462
bbr_do_segment_nounlock
static int bbr_do_segment_nounlock(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp, int32_t drop_hdrlen, int32_t tlen, uint8_t iptos, int32_t nxt_pkt, struct timeval *tv)
Definition: bbr.c:11345
bbr_zone
uma_zone_t bbr_zone
Definition: bbr.c:134
bbr_allow_hdwr_pacing
static int32_t bbr_allow_hdwr_pacing
Definition: bbr.c:415
bbr_restart_after_idle
static void bbr_restart_after_idle(struct tcp_bbr *bbr, uint32_t cts, uint32_t idle_time)
Definition: bbr.c:7916
bbr_timeout_rack
static int bbr_timeout_rack(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts)
Definition: bbr.c:4373
bbr_check_bbr_for_state
static void bbr_check_bbr_for_state(struct tcp_bbr *bbr, uint32_t cts, int32_t line, uint32_t losses)
Definition: bbr.c:11332
bbr_rtt_probe_cwndtarg
static uint32_t bbr_rtt_probe_cwndtarg
Definition: bbr.c:332
bbr_log_pacing_delay_calc
static void bbr_log_pacing_delay_calc(struct tcp_bbr *bbr, uint16_t gain, uint32_t len, uint32_t cts, uint32_t usecs, uint64_t bw, uint32_t override, int mod)
Definition: bbr.c:2469
bbr_log_type_pesist
static void bbr_log_type_pesist(struct tcp_bbr *bbr, uint32_t cts, uint32_t time_in, int32_t line, uint8_t enter_exit)
Definition: bbr.c:2206
bbr_marks_rxt_sack_passed
static int32_t bbr_marks_rxt_sack_passed
Definition: bbr.c:182
bbr_stop_all_timers
static void bbr_stop_all_timers(struct tcpcb *tp)
Definition: bbr.c:9927
bbr_exit_probe_rtt
static void bbr_exit_probe_rtt(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts)
Definition: bbr.c:10870
bbr_lt_fd_thresh
static int32_t bbr_lt_fd_thresh
Definition: bbr.c:359
bbr_log_type_statechange
static void bbr_log_type_statechange(struct tcp_bbr *bbr, uint32_t cts, int32_t line)
Definition: bbr.c:2086
bbr_log_thresh_choice
static void bbr_log_thresh_choice(struct tcp_bbr *bbr, uint32_t cts, uint32_t thresh, uint32_t lro, uint32_t srtt, struct bbr_sendmap *rsm, uint8_t frm)
Definition: bbr.c:2520
bbr_log_ack
static uint32_t bbr_log_ack(struct tcpcb *tp, struct tcpopt *to, struct tcphdr *th, uint32_t *prev_acked)
Definition: bbr.c:7327
bbr_log_type_exit_rec
static void bbr_log_type_exit_rec(struct tcp_bbr *bbr)
Definition: bbr.c:2137
bbr_timeout_persist
static int bbr_timeout_persist(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts)
Definition: bbr.c:4749
bbr_collapse_rtt
static void bbr_collapse_rtt(struct tcpcb *tp, struct tcp_bbr *bbr, int32_t rtt)
Definition: bbr.c:6133
bbr_do_segment
static void bbr_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp, int32_t drop_hdrlen, int32_t tlen, uint8_t iptos)
Definition: bbr.c:11694
tcp_addbbr
static int tcp_addbbr(module_t mod, int32_t type, void *data)
Definition: bbr.c:14824
bbr_update_rtt
static int bbr_update_rtt(struct tcpcb *tp, struct tcp_bbr *bbr, struct bbr_sendmap *rsm, struct tcpopt *to, uint32_t cts, int32_t ack_type, uint32_t th_ack)
Definition: bbr.c:6799
bbr_get_persists_timer_val
static uint32_t bbr_get_persists_timer_val(struct tcpcb *tp, struct tcp_bbr *bbr)
Definition: bbr.c:543
bbr_ack_received
static void bbr_ack_received(struct tcpcb *tp, struct tcp_bbr *bbr, struct tcphdr *th, uint32_t bytes_this_ack, uint32_t sack_changed, uint32_t prev_acked, int32_t line, uint32_t losses)
Definition: bbr.c:3610
bbr_probertt_sets_rtt
static int32_t bbr_probertt_sets_rtt
Definition: bbr.c:243
bbr_alloc
static struct bbr_sendmap * bbr_alloc(struct tcp_bbr *bbr)
Definition: bbr.c:3289
bbr_get_bw_delay_prod
static uint64_t bbr_get_bw_delay_prod(uint64_t rtt, uint64_t bw)
Definition: bbr.c:3386
bbr_get_full_bw
static uint64_t bbr_get_full_bw(struct tcp_bbr *bbr)
Definition: bbr.c:2850
bbr_log_output
static void bbr_log_output(struct tcp_bbr *bbr, struct tcpcb *tp, struct tcpopt *to, int32_t len, uint32_t seq_out, uint16_t th_flags, int32_t err, uint32_t cts, struct mbuf *mb, int32_t *abandon, struct bbr_sendmap *hintrsm, uint32_t delay_calc, struct sockbuf *sb)
Definition: bbr.c:5909
bbr_cong_signal
static void bbr_cong_signal(struct tcpcb *tp, struct tcphdr *th, uint32_t type, struct bbr_sendmap *rsm)
Definition: bbr.c:3933
bbr_def_init_win
static uint32_t bbr_def_init_win
Definition: bbr.c:153
bbr_log_settings_change
static void bbr_log_settings_change(struct tcp_bbr *bbr, int settings_desired)
Definition: bbr.c:2824
bbr_drain_gain
static int32_t bbr_drain_gain
Definition: bbr.c:260
bbr_log_time_epoch
static void bbr_log_time_epoch(struct tcp_bbr *bbr, uint32_t cts, uint32_t line, uint32_t epoch_time)
Definition: bbr.c:2043
tcp_bbr_commit_bw
static void tcp_bbr_commit_bw(struct tcp_bbr *bbr, uint32_t cts)
Definition: bbr.c:6451
bbr_pru_options
static int bbr_pru_options(struct tcpcb *tp, int flags)
Definition: bbr.c:14203
bbr_rto_min_ms
static int32_t bbr_rto_min_ms
Definition: bbr.c:368
bbr_log_type_bbrsnd
static void bbr_log_type_bbrsnd(struct tcp_bbr *bbr, uint32_t len, uint32_t slot, uint32_t del_by, uint32_t cts, uint32_t sloton, uint32_t prev_delay)
Definition: bbr.c:2755
bbr_rtt_gain_thresh
static int32_t bbr_rtt_gain_thresh
Definition: bbr.c:166
bbr_use_rack_resend_cheat
int32_t bbr_use_rack_resend_cheat
Definition: bbr.c:290
bbr_find_lowest_rsm
static struct bbr_sendmap * bbr_find_lowest_rsm(struct tcp_bbr *bbr)
Definition: bbr.c:4046
bbr_log_tstmp_validation
static void bbr_log_tstmp_validation(struct tcp_bbr *bbr, uint64_t peer_delta, uint64_t delta)
Definition: bbr.c:2566
bbr_minseg
static int32_t bbr_minseg(struct tcp_bbr *bbr)
Definition: bbr.c:731
bbr_delack_time
static int32_t bbr_delack_time
Definition: bbr.c:276
bbr_rtt_probe_time
static uint32_t bbr_rtt_probe_time
Definition: bbr.c:331
tcp_bbr_partialack
static void tcp_bbr_partialack(struct tcpcb *tp)
Definition: bbr.c:3763
bbr_initial_cwnd
static uint32_t bbr_initial_cwnd(struct tcp_bbr *bbr, struct tcpcb *tp)
Definition: bbr.c:3407
bbr_log_type_just_return
static void bbr_log_type_just_return(struct tcp_bbr *bbr, uint32_t cts, uint32_t tlen, uint8_t hpts_calling, uint8_t reason, uint32_t p_maxseg, int len)
Definition: bbr.c:1933
bbr_prtt_slam_cwnd
static int32_t bbr_prtt_slam_cwnd
Definition: bbr.c:247
bbr_check_recovery_mode
static struct bbr_sendmap * bbr_check_recovery_mode(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts)
Definition: bbr.c:4308
bbr_quanta
static int32_t bbr_quanta
Definition: bbr.c:160
bbr_incr_timers
static int32_t bbr_incr_timers
Definition: bbr.c:341
bbr_get_a_state_target
static uint32_t bbr_get_a_state_target(struct tcp_bbr *bbr, uint32_t gain)
Definition: bbr.c:10662
bbr_hptsi_bytes_min
static int32_t bbr_hptsi_bytes_min
Definition: bbr.c:404
bbr_merge_rsm
static struct bbr_sendmap * bbr_merge_rsm(struct tcp_bbr *bbr, struct bbr_sendmap *l_rsm, struct bbr_sendmap *r_rsm)
Definition: bbr.c:4505
bbr_ts_convert
static uint32_t bbr_ts_convert(uint32_t cts)
Definition: bbr.c:6786
bbr_cwnd_min_val
static int32_t bbr_cwnd_min_val
Definition: bbr.c:218
bbr_tlp_thresh
static int32_t bbr_tlp_thresh
Definition: bbr.c:335
bbr_policer_call_from_rack_to
static int32_t bbr_policer_call_from_rack_to
Definition: bbr.c:191
bbr_min_peer_delta
static int32_t bbr_min_peer_delta
Definition: bbr.c:209
bbr_tlp_type_to_use
static int32_t bbr_tlp_type_to_use
Definition: bbr.c:275
bbr_set_sockopt
static int bbr_set_sockopt(struct inpcb *inp, struct sockopt *sopt)
Definition: bbr.c:14237
bbr_output_wtime
static int bbr_output_wtime(struct tcpcb *tp, const struct timeval *tv)
Definition: bbr.c:11905
bbr_process_timers
static int bbr_process_timers(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts, uint8_t hpts_calling)
Definition: bbr.c:5193
bbr_remxt_tmr
static void bbr_remxt_tmr(struct tcpcb *tp)
Definition: bbr.c:4882
bbr_timeout_rxt
static int bbr_timeout_rxt(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts)
Definition: bbr.c:4973
bbr_hptsi_max_div
static int32_t bbr_hptsi_max_div
Definition: bbr.c:187
bbr_log_type_bbrupd
static void bbr_log_type_bbrupd(struct tcp_bbr *bbr, uint8_t flex8, uint32_t cts, uint32_t flex3, uint32_t flex2, uint32_t flex5, uint32_t flex6, uint32_t pkts_out, int flex7, uint32_t flex4, uint32_t flex1)
Definition: bbr.c:2642
bbr_delta_percent
static int32_t bbr_delta_percent
Definition: bbr.c:210
bbr_uses_idle_restart
static int32_t bbr_uses_idle_restart
Definition: bbr.c:411
bbr_check_data_after_close
static int bbr_check_data_after_close(struct mbuf *m, struct tcp_bbr *bbr, struct tcpcb *tp, int32_t *tlen, struct tcphdr *th, struct socket *so)
Definition: bbr.c:9432
bbr_startup_lower
static int32_t bbr_startup_lower
Definition: bbr.c:230
bbr_stopall
static int bbr_stopall(struct tcpcb *tp)
Definition: bbr.c:5305
bbr_min_req_free
static const int32_t bbr_min_req_free
Definition: bbr.c:333
bbr_do_established
static int bbr_do_established(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos)
Definition: bbr.c:9207
bbr_alloc_limit
static struct bbr_sendmap * bbr_alloc_limit(struct tcp_bbr *bbr, uint8_t limit_type)
Definition: bbr.c:3327
bbr_process_data
static int bbr_process_data(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp, int32_t drop_hdrlen, int32_t tlen, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt)
Definition: bbr.c:8143
bbr_exit_persist
static void bbr_exit_persist(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts, int32_t line)
Definition: bbr.c:7946
google_allow_early_out
static int32_t google_allow_early_out
Definition: bbr.c:236
bbr_enter_persist
static void bbr_enter_persist(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts, int32_t line)
Definition: bbr.c:7887
CC_NDUPACK
#define CC_NDUPACK
Definition: cc.h:137
CC_RTO_ERR
#define CC_RTO_ERR
Definition: cc.h:136
icmp_var.h
BANDLIM_UNLIMITED
#define BANDLIM_UNLIMITED
Definition: icmp_var.h:92
BANDLIM_RST_OPENPORT
#define BANDLIM_RST_OPENPORT
Definition: icmp_var.h:97
uint32_t
__uint32_t uint32_t
Definition: in.h:62
uint16_t
__uint16_t uint16_t
Definition: in.h:57
uint8_t
__uint8_t uint8_t
Definition: in.h:52
IPPROTO_TCP
#define IPPROTO_TCP
Definition: in.h:45
IPPROTO_UDP
#define IPPROTO_UDP
Definition: in.h:46
IPPROTO_IP
#define IPPROTO_IP
Definition: in.h:43
in_pseudo
u_short in_pseudo(u_int32_t a, u_int32_t b, u_int32_t c)
Definition: in_cksum.c:197
in_kdtrace.h
TCP_PROBE5
#define TCP_PROBE5(probe, arg0, arg1, arg2, arg3, arg4)
Definition: in_kdtrace.h:47
in_losing
void in_losing(struct inpcb *inp)
Definition: in_pcb.c:2577
INP_FREED
#define INP_FREED
Definition: in_pcb.c:118
in_pcb.h
INP_WLOCK
#define INP_WLOCK(inp)
Definition: in_pcb.h:518
INP_DONT_SACK_QUEUE
#define INP_DONT_SACK_QUEUE
Definition: in_pcb.h:679
INP_MBUF_QUEUE_READY
#define INP_MBUF_QUEUE_READY
Definition: in_pcb.h:678
INP_WLOCK_ASSERT
#define INP_WLOCK_ASSERT(inp)
Definition: in_pcb.h:529
INP_SUPPORTS_MBUFQ
#define INP_SUPPORTS_MBUFQ
Definition: in_pcb.h:677
INP_TIMEWAIT
#define INP_TIMEWAIT
Definition: in_pcb.h:644
INP_DROPPED
#define INP_DROPPED
Definition: in_pcb.h:646
INP_WUNLOCK
#define INP_WUNLOCK(inp)
Definition: in_pcb.h:522
INP_CANNOT_DO_ECN
#define INP_CANNOT_DO_ECN
Definition: in_pcb.h:675
INP_IPV6
#define INP_IPV6
Definition: in_pcb.h:614
IP_MAXPACKET
#define IP_MAXPACKET
Definition: ip.h:74
IPVERSION
#define IPVERSION
Definition: ip.h:46
IP_DF
#define IP_DF
Definition: ip.h:13
count
u_int16_t count
Definition: ip_fw.h:18
ip_icmp.h
ip_output
int ip_output(struct mbuf *m, struct mbuf *opt, struct route *ro, int flags, struct ip_moptions *imo, struct inpcb *inp)
Definition: ip_output.c:320
ip_var.h
IP_NO_SND_TAG_RL
#define IP_NO_SND_TAG_RL
Definition: ip_var.h:172
ctf_progress_timeout_check
int32_t ctf_progress_timeout_check(struct tcpcb *tp, bool log)
Definition: rack_bbr_common.c:1045
ctf_do_queued_segments
int ctf_do_queued_segments(struct socket *so, struct tcpcb *tp, int have_pkt)
Definition: rack_bbr_common.c:527
ctf_log_sack_filter
void ctf_log_sack_filter(struct tcpcb *tp, int num_sack_blks, struct sackblk *sack_blocks)
Definition: rack_bbr_common.c:985
ctf_do_dropwithreset
void ctf_do_dropwithreset(struct mbuf *m, struct tcpcb *tp, struct tcphdr *th, int32_t rstreason, int32_t tlen)
Definition: rack_bbr_common.c:568
ctf_do_drop
void ctf_do_drop(struct mbuf *m, struct tcpcb *tp)
Definition: rack_bbr_common.c:754
ctf_challenge_ack
void ctf_challenge_ack(struct mbuf *m, struct tcphdr *th, struct tcpcb *tp, int32_t *ret_val)
Definition: rack_bbr_common.c:863
ctf_outstanding
uint32_t ctf_outstanding(struct tcpcb *tp)
Definition: rack_bbr_common.c:545
ctf_flight_size
uint32_t ctf_flight_size(struct tcpcb *tp, uint32_t rc_sacked)
Definition: rack_bbr_common.c:558
ctf_ts_check
int ctf_ts_check(struct mbuf *m, struct tcphdr *th, struct tcpcb *tp, int32_t tlen, int32_t thflags, int32_t *ret_val)
Definition: rack_bbr_common.c:894
ctf_calc_rwin
void ctf_calc_rwin(struct socket *so, struct tcpcb *tp)
Definition: rack_bbr_common.c:952
ctf_do_dropwithreset_conn
void ctf_do_dropwithreset_conn(struct mbuf *m, struct tcpcb *tp, struct tcphdr *th, int32_t rstreason, int32_t tlen)
Definition: rack_bbr_common.c:968
rack_bbr_common.h
TWENTY_THREE_MBPS
#define TWENTY_THREE_MBPS
Definition: rack_bbr_common.h:83
ctf_drop_checks
#define ctf_drop_checks(a, b, c, d, e, f, g, h)
Definition: rack_bbr_common.h:106
TCP_MSS_SMALL_SIZE_OFF
#define TCP_MSS_SMALL_SIZE_OFF
Definition: rack_bbr_common.h:38
PROGRESS_UPDATE
#define PROGRESS_UPDATE
Definition: rack_bbr_common.h:46
MS_IN_USEC
#define MS_IN_USEC
Definition: rack_bbr_common.h:76
ONE_POINT_TWO_MEG
#define ONE_POINT_TWO_MEG
Definition: rack_bbr_common.h:85
ctf_process_rst
#define ctf_process_rst(m, t, s, p)
Definition: rack_bbr_common.h:124
BBR_JR_RWND_LIMITED
#define BBR_JR_RWND_LIMITED
Definition: rack_bbr_common.h:62
TCP_MSS_ACCT_SIZE
#define TCP_MSS_ACCT_SIZE
Definition: rack_bbr_common.h:39
BBR_JR_SENT_DATA
#define BBR_JR_SENT_DATA
Definition: rack_bbr_common.h:60
TCP_MSS_ACCT_INPACE
#define TCP_MSS_ACCT_INPACE
Definition: rack_bbr_common.h:36
TCP_MSS_ACCT_JUSTRET
#define TCP_MSS_ACCT_JUSTRET
Definition: rack_bbr_common.h:32
FIVETWELVE_MBPS
#define FIVETWELVE_MBPS
Definition: rack_bbr_common.h:84
BBR_JR_CWND_LIMITED
#define BBR_JR_CWND_LIMITED
Definition: rack_bbr_common.h:61
PROGRESS_DROP
#define PROGRESS_DROP
Definition: rack_bbr_common.h:45
ctf_do_dropafterack
#define ctf_do_dropafterack(a, b, c, d, e, f)
Definition: rack_bbr_common.h:113
TCP_MSS_ACCT_LATE
#define TCP_MSS_ACCT_LATE
Definition: rack_bbr_common.h:37
TCP_MSS_ACCT_SNDACK
#define TCP_MSS_ACCT_SNDACK
Definition: rack_bbr_common.h:33
TCP_TS_OVERHEAD
#define TCP_TS_OVERHEAD
Definition: rack_bbr_common.h:78
BBR_JR_APP_LIMITED
#define BBR_JR_APP_LIMITED
Definition: rack_bbr_common.h:63
PROGRESS_CLEAR
#define PROGRESS_CLEAR
Definition: rack_bbr_common.h:47
PROGRESS_START
#define PROGRESS_START
Definition: rack_bbr_common.h:48
TCP_MSS_ACCT_ATIMER
#define TCP_MSS_ACCT_ATIMER
Definition: rack_bbr_common.h:35
TCP_MSS_SMALL_MAX_SIZE_DIV
#define TCP_MSS_SMALL_MAX_SIZE_DIV
Definition: rack_bbr_common.h:40
PACE_MAX_IP_BYTES
#define PACE_MAX_IP_BYTES
Definition: rack_bbr_common.h:73
DUP_ACK_THRESHOLD
#define DUP_ACK_THRESHOLD
Definition: rack_bbr_common.h:42
sack_filter_blks
int sack_filter_blks(struct sack_filter *sf, struct sackblk *in, int numblks, tcp_seq th_ack)
Definition: sack_filter.c:499
sack_filter_reject
void sack_filter_reject(struct sack_filter *sf, struct sackblk *in)
Definition: sack_filter.c:561
sack_filter_clear
void sack_filter_clear(struct sack_filter *sf, tcp_seq seq)
Definition: sack_filter.c:101
sack_filter.h
bbr_control::crte
const struct tcp_hwrate_limit_table * crte
Definition: tcp_bbr.h:663
bbr_control::rc_last_tlp_seq
uint32_t rc_last_tlp_seq
Definition: tcp_bbr.h:557
bbr_control::rc_num_small_maps_alloced
uint16_t rc_num_small_maps_alloced
Definition: tcp_bbr.h:705
bbr_control::rc_rtt_epoch
uint32_t rc_rtt_epoch
Definition: tcp_bbr.h:618
bbr_control::last_startup_measure
uint32_t last_startup_measure
Definition: tcp_bbr.h:673
bbr_control::rc_exta_time_gd
uint32_t rc_exta_time_gd
Definition: tcp_bbr.h:647
bbr_control::rc_del_time
uint32_t rc_del_time
Definition: tcp_bbr.h:530
bbr_control::rc_flight_at_input
uint32_t rc_flight_at_input
Definition: tcp_bbr.h:518
bbr_control::rc_lt_epoch
uint32_t rc_lt_epoch
Definition: tcp_bbr.h:623
bbr_control::rc_lt_lost
uint32_t rc_lt_lost
Definition: tcp_bbr.h:649
bbr_control::rc_bbr_lastbtlbw
uint64_t rc_bbr_lastbtlbw
Definition: tcp_bbr.h:637
bbr_control::bbr_hptsi_bytes_min
uint16_t bbr_hptsi_bytes_min
Definition: tcp_bbr.h:707
bbr_control::rc_init_rwnd
uint32_t rc_init_rwnd
Definition: tcp_bbr.h:570
bbr_control::rc_pkt_epoch_time
uint32_t rc_pkt_epoch_time
Definition: tcp_bbr.h:615
bbr_control::rc_startup_pg
uint16_t rc_startup_pg
Definition: tcp_bbr.h:535
bbr_control::bbr_utter_max
uint16_t bbr_utter_max
Definition: tcp_bbr.h:710
bbr_control::rc_bbr_cwnd_gain
uint32_t rc_bbr_cwnd_gain
Definition: tcp_bbr.h:640
bbr_control::rc_lt_time
uint32_t rc_lt_time
Definition: tcp_bbr.h:631
bbr_control::rc_min_rto_ms
uint32_t rc_min_rto_ms
Definition: tcp_bbr.h:670
bbr_control::rc_saved_cwnd
uint32_t rc_saved_cwnd
Definition: tcp_bbr.h:643
bbr_control::rc_initial_hptsi_bw
uint32_t rc_initial_hptsi_bw
Definition: tcp_bbr.h:654
bbr_control::gain_epoch
uint32_t gain_epoch
Definition: tcp_bbr.h:669
bbr_control::bbr_hptsi_segments_max
int32_t bbr_hptsi_segments_max
Definition: tcp_bbr.h:678
bbr_control::rc_rtt_shrinks
uint32_t rc_rtt_shrinks
Definition: tcp_bbr.h:544
bbr_control::rc_pace_max_segs
uint32_t rc_pace_max_segs
Definition: tcp_bbr.h:541
bbr_control::cur_rtt
uint32_t cur_rtt
Definition: tcp_bbr.h:537
bbr_control::rc_bbr_cur_del_rate
uint64_t rc_bbr_cur_del_rate
Definition: tcp_bbr.h:601
bbr_control::r_app_limited_until
uint32_t r_app_limited_until
Definition: tcp_bbr.h:545
bbr_control::rc_lost_at_pktepoch
uint32_t rc_lost_at_pktepoch
Definition: tcp_bbr.h:553
bbr_control::rc_last_rtt
uint32_t rc_last_rtt
Definition: tcp_bbr.h:577
bbr_control::bbr_hdwr_cnt_noset_snt
uint32_t bbr_hdwr_cnt_noset_snt
Definition: tcp_bbr.h:689
bbr_control::rc_lost
uint32_t rc_lost
Definition: tcp_bbr.h:646
bbr_control::rc_inc_tcp_oh
uint32_t rc_inc_tcp_oh
Definition: tcp_bbr.h:591
bbr_control::rc_pkt_delay
uint16_t rc_pkt_delay
Definition: tcp_bbr.h:559
bbr_control::rc_num_maps_alloced
uint32_t rc_num_maps_alloced
Definition: tcp_bbr.h:703
bbr_control::rc_num_split_allocs
uint32_t rc_num_split_allocs
Definition: tcp_bbr.h:704
bbr_control::last_inbound_ts
uint32_t last_inbound_ts
Definition: tcp_bbr.h:589
bbr_control::rc_tlp_rxt_last_time
uint32_t rc_tlp_rxt_last_time
Definition: tcp_bbr.h:687
bbr_control::restrict_growth
uint32_t restrict_growth
Definition: tcp_bbr.h:595
bbr_control::rc_lost_bytes
uint32_t rc_lost_bytes
Definition: tcp_bbr.h:519
bbr_control::red_bw
uint64_t red_bw
Definition: tcp_bbr.h:664
bbr_control::rc_delivered
uint32_t rc_delivered
Definition: tcp_bbr.h:515
bbr_control::rc_red_cwnd_pe
uint32_t rc_red_cwnd_pe
Definition: tcp_bbr.h:662
bbr_control::rc_target_at_state
uint32_t rc_target_at_state
Definition: tcp_bbr.h:531
bbr_control::rc_agg_early
uint32_t rc_agg_early
Definition: tcp_bbr.h:609
bbr_control::last_in_probertt
uint32_t last_in_probertt
Definition: tcp_bbr.h:695
bbr_control::rc_delrate
struct time_filter rc_delrate
Definition: tcp_bbr.h:523
bbr_control::startup_last_srtt
uint32_t startup_last_srtt
Definition: tcp_bbr.h:690
bbr_control::rc_bbr_enters_probertt
uint32_t rc_bbr_enters_probertt
Definition: tcp_bbr.h:629
bbr_control::rc_recovery_start
uint32_t rc_recovery_start
Definition: tcp_bbr.h:598
bbr_control::rc_bbr_last_startup_epoch
uint32_t rc_bbr_last_startup_epoch
Definition: tcp_bbr.h:627
bbr_control::rc_rcvtime
uint32_t rc_rcvtime
Definition: tcp_bbr.h:611
bbr_control::rc_high_rwnd
uint32_t rc_high_rwnd
Definition: tcp_bbr.h:574
bbr_control::rc_lt_bw
uint64_t rc_lt_bw
Definition: tcp_bbr.h:636
bbr_control::bbr_smallest_srtt_this_state
uint32_t bbr_smallest_srtt_this_state
Definition: tcp_bbr.h:621
bbr_control::rc_last_delay_val
uint32_t rc_last_delay_val
Definition: tcp_bbr.h:511
bbr_control::bbr_hptsi_segments_delay_tar
int32_t bbr_hptsi_segments_delay_tar
Definition: tcp_bbr.h:676
bbr_control::rc_went_idle_time
uint32_t rc_went_idle_time
Definition: tcp_bbr.h:539
bbr_control::rc_lt_epoch_use
uint32_t rc_lt_epoch_use
Definition: tcp_bbr.h:596
bbr_control::bbr_sf
struct sack_filter bbr_sf
Definition: tcp_bbr.h:583
bbr_control::rc_probertt_int
uint32_t rc_probertt_int
Definition: tcp_bbr.h:666
bbr_control::highest_hdwr_delay
uint32_t highest_hdwr_delay
Definition: tcp_bbr.h:692
bbr_control::rc_reorder_fade
uint32_t rc_reorder_fade
Definition: tcp_bbr.h:672
bbr_control::rc_min_to
uint32_t rc_min_to
Definition: tcp_bbr.h:653
bbr_control::rc_rcv_epoch_start
uint32_t rc_rcv_epoch_start
Definition: tcp_bbr.h:547
bbr_control::substate_pe
uint32_t substate_pe
Definition: tcp_bbr.h:644
bbr_control::rc_tlp_send
struct bbr_sendmap * rc_tlp_send
Definition: tcp_bbr.h:528
bbr_control::rc_bbr_state_time
uint32_t rc_bbr_state_time
Definition: tcp_bbr.h:651
bbr_control::rc_pkt_epoch_rtt
uint32_t rc_pkt_epoch_rtt
Definition: tcp_bbr.h:617
bbr_control::rc_incr_tmrs
uint32_t rc_incr_tmrs
Definition: tcp_bbr.h:594
bbr_control::rc_bbr_hptsi_gain
uint32_t rc_bbr_hptsi_gain
Definition: tcp_bbr.h:512
bbr_control::bbr_lost_at_state
uint32_t bbr_lost_at_state
Definition: tcp_bbr.h:656
bbr_control::rc_cwnd_on_ent
uint32_t rc_cwnd_on_ent
Definition: tcp_bbr.h:607
bbr_control::rc_lowest_rtt
uint32_t rc_lowest_rtt
Definition: tcp_bbr.h:575
bbr_control::rc_inc_enet_oh
uint32_t rc_inc_enet_oh
Definition: tcp_bbr.h:593
bbr_control::bbr_ts_check_our_cts
uint32_t bbr_ts_check_our_cts
Definition: tcp_bbr.h:686
bbr_control::r_measurement_count
uint32_t r_measurement_count
Definition: tcp_bbr.h:555
bbr_control::rc_next
struct bbr_sendmap * rc_next
Definition: tcp_bbr.h:563
bbr_control::rc_bbr_state_atflight
uint32_t rc_bbr_state_atflight
Definition: tcp_bbr.h:626
bbr_control::cur_rtt_send_time
uint32_t cur_rtt_send_time
Definition: tcp_bbr.h:683
bbr_control::bbr_rttprobe_gain_val
uint32_t bbr_rttprobe_gain_val
Definition: tcp_bbr.h:679
bbr_control::rc_ack_hdwr_delay
uint32_t rc_ack_hdwr_delay
Definition: tcp_bbr.h:691
bbr_control::rc_sacklast
struct bbr_sendmap * rc_sacklast
Definition: tcp_bbr.h:561
bbr_control::bbr_peer_tsratio
uint32_t bbr_peer_tsratio
Definition: tcp_bbr.h:684
bbr_control::rc_drain_pg
uint16_t rc_drain_pg
Definition: tcp_bbr.h:702
bbr_control::rc_hptsi_agg_delay
uint32_t rc_hptsi_agg_delay
Definition: tcp_bbr.h:516
bbr_control::flightsize_at_drain
uint32_t flightsize_at_drain
Definition: tcp_bbr.h:696
bbr_control::rc_pace_min_segs
uint32_t rc_pace_min_segs
Definition: tcp_bbr.h:542
bbr_control::rc_sacked
uint32_t rc_sacked
Definition: tcp_bbr.h:566
bbr_control::rc_free_cnt
uint16_t rc_free_cnt
Definition: tcp_bbr.h:533
bbr_control::bbr_ts_check_tstmp
uint32_t bbr_ts_check_tstmp
Definition: tcp_bbr.h:685
bbr_control::rc_resend
struct bbr_sendmap * rc_resend
Definition: tcp_bbr.h:509
bbr_control::rc_lt_del
uint32_t rc_lt_del
Definition: tcp_bbr.h:599
bbr_control::rc_holes_rxt
uint32_t rc_holes_rxt
Definition: tcp_bbr.h:567
bbr_control::bbr_smallest_srtt_state2
uint32_t bbr_smallest_srtt_state2
Definition: tcp_bbr.h:688
bbr_control::rc_rttprop
struct time_filter_small rc_rttprop
Definition: tcp_bbr.h:588
bbr_control::rc_inc_ip_oh
uint32_t rc_inc_ip_oh
Definition: tcp_bbr.h:592
bbr_control::rc_hpts_flags
uint32_t rc_hpts_flags
Definition: tcp_bbr.h:513
bbr_control::rc_reorder_shift
uint16_t rc_reorder_shift
Definition: tcp_bbr.h:558
bbr_control::rc_free
struct bbr_head rc_free
Definition: tcp_bbr.h:527
bbr_control::bbr_google_discount
uint16_t bbr_google_discount
Definition: tcp_bbr.h:711
bbr_control::bbr_hptsi_per_second
int32_t bbr_hptsi_per_second
Definition: tcp_bbr.h:675
bbr_control::rc_lost_at_startup
uint32_t rc_lost_at_startup
Definition: tcp_bbr.h:624
bbr_control::ts_in
uint32_t ts_in
Definition: tcp_bbr.h:698
bbr_control::rc_tmap
struct bbr_head rc_tmap
Definition: tcp_bbr.h:508
bbr_control::bbr_hptsi_segments_floor
uint16_t bbr_hptsi_segments_floor
Definition: tcp_bbr.h:709
bbr_control::rc_pkt_epoch_loss_rate
uint32_t rc_pkt_epoch_loss_rate
Definition: tcp_bbr.h:641
bbr_control::rc_pkt_epoch_del
uint32_t rc_pkt_epoch_del
Definition: tcp_bbr.h:612
bbr_control::rc_level_state_extra
uint32_t rc_level_state_extra
Definition: tcp_bbr.h:661
bbr_control::bbr_cross_over
uint32_t bbr_cross_over
Definition: tcp_bbr.h:578
bbr_control::rc_timer_exp
uint32_t rc_timer_exp
Definition: tcp_bbr.h:546
bbr_control::rc_reorder_ts
uint32_t rc_reorder_ts
Definition: tcp_bbr.h:569
bbr_control::rc_map
struct bbr_head rc_map
Definition: tcp_bbr.h:507
bbr_control::rc_pkt_epoch
uint32_t rc_pkt_epoch
Definition: tcp_bbr.h:614
bbr_control::rc_probertt_srttchktim
uint32_t rc_probertt_srttchktim
Definition: tcp_bbr.h:667
bbr_control::rc_tlp_seg_send_cnt
uint16_t rc_tlp_seg_send_cnt
Definition: tcp_bbr.h:700
bbr_control::rc_pe_of_prtt
uint32_t rc_pe_of_prtt
Definition: tcp_bbr.h:697
bbr_control::recovery_lr
uint32_t recovery_lr
Definition: tcp_bbr.h:694
bbr_sendmap
Definition: tcp_bbr.h:58
bbr_sendmap::r_first_sent_time
uint32_t r_first_sent_time
Definition: tcp_bbr.h:91
bbr_sendmap::r_rtt_not_allowed
uint8_t r_rtt_not_allowed
Definition: tcp_bbr.h:70
bbr_sendmap::r_start
uint32_t r_start
Definition: tcp_bbr.h:61
bbr_sendmap::r_delivered
uint32_t r_delivered
Definition: tcp_bbr.h:65
bbr_sendmap::r_is_smallmap
uint8_t r_is_smallmap
Definition: tcp_bbr.h:76
bbr_sendmap::r_is_drain
uint8_t r_is_drain
Definition: tcp_bbr.h:71
bbr_sendmap::r_in_tmap
uint8_t r_in_tmap
Definition: tcp_bbr.h:75
bbr_sendmap::r_pacing_delay
uint32_t r_pacing_delay
Definition: tcp_bbr.h:92
bbr_sendmap::r_tim_lastsent
uint32_t r_tim_lastsent[BBR_NUM_OF_RETRANS]
Definition: tcp_bbr.h:85
bbr_sendmap::r_is_gain
uint8_t r_is_gain
Definition: tcp_bbr.h:77
bbr_sendmap::r_flight_at_send
uint32_t r_flight_at_send
Definition: tcp_bbr.h:93
bbr_sendmap::r_del_time
uint32_t r_del_time
Definition: tcp_bbr.h:67
bbr_sendmap::r_limit_type
uint8_t r_limit_type
Definition: tcp_bbr.h:79
bbr_sendmap::r_dupack
uint8_t r_dupack
Definition: tcp_bbr.h:74
bbr_sendmap::r_rtr_bytes
uint32_t r_rtr_bytes
Definition: tcp_bbr.h:64
bbr_sendmap::r_end
uint32_t r_end
Definition: tcp_bbr.h:62
bbr_sendmap::r_rtr_cnt
uint8_t r_rtr_cnt
Definition: tcp_bbr.h:68
bbr_sendmap::r_ts_valid
uint8_t r_ts_valid
Definition: tcp_bbr.h:73
bbr_sendmap::r_flags
uint16_t r_flags
Definition: tcp_bbr.h:81
bbr_sendmap::r_del_ack_ts
uint32_t r_del_ack_ts
Definition: tcp_bbr.h:83
bbr_sendmap::r_app_limited
uint8_t r_app_limited
Definition: tcp_bbr.h:72
bbr_sendmap::r_bbr_state
uint8_t r_bbr_state
Definition: tcp_bbr.h:78
hpts_diag
Definition: tcp_hpts.h:39
hpts_diag::p_hpts_active
uint32_t p_hpts_active
Definition: tcp_hpts.h:40
hpts_diag::slot_req
uint32_t slot_req
Definition: tcp_hpts.h:45
hpts_diag::p_nxt_slot
uint32_t p_nxt_slot
Definition: tcp_hpts.h:41
hpts_diag::yet_to_sleep
uint32_t yet_to_sleep
Definition: tcp_hpts.h:50
hpts_diag::p_prev_slot
uint32_t p_prev_slot
Definition: tcp_hpts.h:43
hpts_diag::wheel_cts
uint32_t wheel_cts
Definition: tcp_hpts.h:54
hpts_diag::p_on_min_sleep
uint8_t p_on_min_sleep
Definition: tcp_hpts.h:58
hpts_diag::p_curtick
uint32_t p_curtick
Definition: tcp_hpts.h:56
hpts_diag::slot_remaining
uint32_t slot_remaining
Definition: tcp_hpts.h:47
hpts_diag::co_ret
int32_t co_ret
Definition: tcp_hpts.h:55
hpts_diag::wheel_slot
uint32_t wheel_slot
Definition: tcp_hpts.h:52
hpts_diag::inp_hptsslot
uint32_t inp_hptsslot
Definition: tcp_hpts.h:46
hpts_diag::p_lasttick
uint32_t p_lasttick
Definition: tcp_hpts.h:57
hpts_diag::p_runningslot
uint32_t p_runningslot
Definition: tcp_hpts.h:44
hpts_diag::hpts_sleep_time
uint32_t hpts_sleep_time
Definition: tcp_hpts.h:49
hpts_diag::maxslots
uint32_t maxslots
Definition: tcp_hpts.h:53
hpts_diag::need_new_to
uint32_t need_new_to
Definition: tcp_hpts.h:51
hpts_diag::have_slept
uint32_t have_slept
Definition: tcp_hpts.h:48
hpts_diag::p_cur_slot
uint32_t p_cur_slot
Definition: tcp_hpts.h:42
in_addr::s_addr
in_addr_t s_addr
Definition: in.h:84
inpcb
Definition: in_pcb.h:217
inpcb::inp_socket
struct socket * inp_socket
Definition: in_pcb.h:254
inpcb::in6p_outputopts
struct ip6_pktopts * in6p_outputopts
Definition: in_pcb.h:286
inpcb::inp_route
struct route inp_route
Definition: in_pcb.h:301
inpcb::inp_hpts_calls
uint8_t inp_hpts_calls
Definition: in_pcb.h:249
inpcb::inp_flags
int inp_flags
Definition: in_pcb.h:246
inpcb::inp_route6
struct route_in6 inp_route6
Definition: in_pcb.h:302
inpcb::inp_flags2
int inp_flags2
Definition: in_pcb.h:247
inpcb::inp_vflag
u_char inp_vflag
Definition: in_pcb.h:260
inpcb::inp_options
struct mbuf * inp_options
Definition: in_pcb.h:279
inpcb::inp_snd_tag
struct m_snd_tag * inp_snd_tag
Definition: in_pcb.h:265
inpcb::inp_hptsslot
int32_t inp_hptsslot
Definition: in_pcb.h:255
inpcb::inp_inc
struct in_conninfo inp_inc
Definition: in_pcb.h:270
ip6_hdr
Definition: ip6.h:74
ip
Definition: ip.h:51
ip::ip_dst
struct in_addr ip_src ip_dst
Definition: ip.h:71
ip::ip_len
u_short ip_len
Definition: ip.h:61
ip::ip_v
u_char ip_v
Definition: ip.h:54
ip::ip_ttl
u_char ip_ttl
Definition: ip.h:68
ip::ip_off
u_short ip_off
Definition: ip.h:63
ipovly
Definition: ip_var.h:47
ipovly::ih_len
u_short ih_len
Definition: ip_var.h:50
sackblk::end
tcp_seq end
Definition: tcp_var.h:98
sackblk::start
tcp_seq start
Definition: tcp_var.h:97
tcp_bbr
Definition: tcp_bbr.h:716
tcp_bbr::hw_pacing_set
uint16_t hw_pacing_set
Definition: tcp_bbr.h:731
tcp_bbr::rc_use_ts_limit
uint8_t rc_use_ts_limit
Definition: tcp_bbr.h:785
tcp_bbr::ts_can_raise
uint16_t ts_can_raise
Definition: tcp_bbr.h:726
tcp_bbr::rc_allow_data_af_clo
uint8_t rc_allow_data_af_clo
Definition: tcp_bbr.h:771
tcp_bbr::r_substate
int32_t(* r_substate)(struct mbuf *, struct tcphdr *, struct socket *, struct tcpcb *, struct tcpopt *, int32_t, int32_t, uint32_t, int32_t, int32_t, uint8_t)
Definition: tcp_bbr.h:718
tcp_bbr::rc_filled_pipe
uint16_t rc_filled_pipe
Definition: tcp_bbr.h:755
tcp_bbr::bbr_prev_in_rec
uint8_t bbr_prev_in_rec
Definition: tcp_bbr.h:742
tcp_bbr::rc_max_rto_sec
uint8_t rc_max_rto_sec
Definition: tcp_bbr.h:781
tcp_bbr::rc_resends_use_tso
uint16_t rc_resends_use_tso
Definition: tcp_bbr.h:750
tcp_bbr::rc_tlp_rtx_out
uint8_t rc_tlp_rtx_out
Definition: tcp_bbr.h:772
tcp_bbr::rc_tlp_in_progress
uint8_t rc_tlp_in_progress
Definition: tcp_bbr.h:773
tcp_bbr::bbr_hdrw_pacing
uint8_t bbr_hdrw_pacing
Definition: tcp_bbr.h:740
tcp_bbr::bbr_use_rack_cheat
uint8_t bbr_use_rack_cheat
Definition: tcp_bbr.h:737
tcp_bbr::rc_ts_clock_set
uint8_t rc_ts_clock_set
Definition: tcp_bbr.h:787
tcp_bbr::rc_no_pacing
uint8_t rc_no_pacing
Definition: tcp_bbr.h:790
tcp_bbr::rc_inp
struct inpcb * rc_inp
Definition: tcp_bbr.h:722
tcp_bbr::output_error_seen
uint16_t output_error_seen
Definition: tcp_bbr.h:729
tcp_bbr::r_init_rtt
uint8_t r_init_rtt
Definition: tcp_bbr.h:765
tcp_bbr::r_ctl
struct bbr_control r_ctl
Definition: tcp_bbr.h:794
tcp_bbr::rc_last_options
uint8_t rc_last_options
Definition: tcp_bbr.h:779
tcp_bbr::r_state
uint8_t r_state
Definition: tcp_bbr.h:763
tcp_bbr::gain_is_limited
uint16_t gain_is_limited
Definition: tcp_bbr.h:728
tcp_bbr::r_use_policer
uint8_t r_use_policer
Definition: tcp_bbr.h:766
tcp_bbr::rc_ts_data_set
uint8_t rc_ts_data_set
Definition: tcp_bbr.h:786
tcp_bbr::rc_output_starts_timer
uint16_t rc_output_starts_timer
Definition: tcp_bbr.h:749
tcp_bbr::rc_in_persist
uint8_t rc_in_persist
Definition: tcp_bbr.h:769
tcp_bbr::rc_loss_exit
uint16_t rc_loss_exit
Definition: tcp_bbr.h:752
tcp_bbr::r_wanted_output
uint16_t r_wanted_output
Definition: tcp_bbr.h:746
tcp_bbr::r_is_v6
uint8_t r_is_v6
Definition: tcp_bbr.h:777
tcp_bbr::rc_init_win
uint8_t rc_init_win
Definition: tcp_bbr.h:792
tcp_bbr::no_pacing_until
uint16_t no_pacing_until
Definition: tcp_bbr.h:725
tcp_bbr::rc_all_timers_stopped
uint16_t rc_all_timers_stopped
Definition: tcp_bbr.h:751
tcp_bbr::rc_tp
struct tcpcb * rc_tp
Definition: tcp_bbr.h:721
tcp_bbr::rc_lt_use_bw
uint8_t rc_lt_use_bw
Definition: tcp_bbr.h:770
tcp_bbr::rc_tlp_new_data
uint16_t rc_tlp_new_data
Definition: tcp_bbr.h:756
tcp_bbr::rc_lt_is_sampling
uint16_t rc_lt_is_sampling
Definition: tcp_bbr.h:754
tcp_bbr::rc_ack_was_delayed
uint16_t rc_ack_was_delayed
Definition: tcp_bbr.h:753
tcp_bbr::r_agg_early_set
uint8_t r_agg_early_set
Definition: tcp_bbr.h:764
tcp_bbr::rc_pacer_started
uint32_t rc_pacer_started
Definition: tcp_bbr.h:724
tcp_bbr::rc_hit_state_1
uint16_t rc_hit_state_1
Definition: tcp_bbr.h:757
tcp_bbr::r_timer_override
uint8_t r_timer_override
Definition: tcp_bbr.h:768
tcp_bbr::bbr_segs_rcvd
uint16_t bbr_segs_rcvd
Definition: tcp_bbr.h:734
tcp_bbr::bbr_timer_src
uint8_t bbr_timer_src
Definition: tcp_bbr.h:736
tcp_bbr::bbr_init_win_cheat
uint8_t bbr_init_win_cheat
Definition: tcp_bbr.h:738
tcp_bbr::rc_past_init_win
uint8_t rc_past_init_win
Definition: tcp_bbr.h:778
tcp_bbr::rc_use_google
uint8_t rc_use_google
Definition: tcp_bbr.h:784
tcp_bbr::rc_tlp_threshold
uint8_t rc_tlp_threshold
Definition: tcp_bbr.h:780
tcp_bbr::rc_use_idle_restart
uint8_t rc_use_idle_restart
Definition: tcp_bbr.h:774
tcp_bbr::bbr_hdw_pace_ena
uint8_t bbr_hdw_pace_ena
Definition: tcp_bbr.h:741
tcp_bbr::rc_ack_is_cumack
uint8_t rc_ack_is_cumack
Definition: tcp_bbr.h:789
tcp_bbr::oerror_cnt
uint16_t oerror_cnt
Definition: tcp_bbr.h:730
tcp_bbr::rc_cwnd_limited
uint8_t rc_cwnd_limited
Definition: tcp_bbr.h:782
tcp_bbr::rc_bbr_substate
uint8_t rc_bbr_substate
Definition: tcp_bbr.h:776
tcp_bbr::rc_tmr_stopped
uint8_t rc_tmr_stopped
Definition: tcp_bbr.h:783
tcp_bbr::skip_gain
uint16_t skip_gain
Definition: tcp_bbr.h:727
tcp_bbr::alloc_limit_reported
uint8_t alloc_limit_reported
Definition: tcp_bbr.h:791
tcp_bbr::rc_bbr_state
uint8_t rc_bbr_state
Definition: tcp_bbr.h:775
tcp_bbr::rc_is_pkt_epoch_now
uint16_t rc_is_pkt_epoch_now
Definition: tcp_bbr.h:760
tcp_bbr::rc_tv
struct timeval rc_tv
Definition: tcp_bbr.h:723
tcp_bbr::rc_prtt_set_ts
uint16_t rc_prtt_set_ts
Definition: tcp_bbr.h:759
tcp_bbr::r_recovery_bw
uint8_t r_recovery_bw
Definition: tcp_bbr.h:767
tcp_bbr::rc_has_collapsed
uint16_t rc_has_collapsed
Definition: tcp_bbr.h:761
tcp_bbr::rc_timer_first
uint16_t rc_timer_first
Definition: tcp_bbr.h:748
tcp_bbr::rc_ts_cant_be_used
uint8_t rc_ts_cant_be_used
Definition: tcp_bbr.h:788
tcp_bbr::rtt_valid
uint16_t rtt_valid
Definition: tcp_bbr.h:747
tcp_bbr::bbr_attempt_hdwr_pace
uint8_t bbr_attempt_hdwr_pace
Definition: tcp_bbr.h:739
tcp_bbr::rc_ts_valid
uint16_t rc_ts_valid
Definition: tcp_bbr.h:758
tcp_bbr::pkt_conservation
uint8_t pkt_conservation
Definition: tcp_bbr.h:743
tcp_function_block
Definition: tcp_var.h:347
tcp_function_block::tfb_tcp_block_name
char tfb_tcp_block_name[TCP_FUNCTION_NAME_LEN_MAX]
Definition: tcp_var.h:348
tcp_function_block::tfb_tcp_do_segment
void(* tfb_tcp_do_segment)(struct mbuf *, struct tcphdr *, struct socket *, struct tcpcb *, int, int, uint8_t)
Definition: tcp_var.h:351
tcp_hwrate_limit_table
Definition: tcp_ratelimit.h:43
tcp_hwrate_limit_table::time_between
uint32_t time_between
Definition: tcp_ratelimit.h:49
tcp_hwrate_limit_table::rate
long rate
Definition: tcp_ratelimit.h:46
tcp_hwrate_limit_table::ptbl
const struct tcp_rate_set * ptbl
Definition: tcp_ratelimit.h:44
tcp_log_bbr
Definition: tcp_log_buf.h:73
tcp_log_bbr::delRate
uint64_t delRate
Definition: tcp_log_buf.h:75
tcp_log_bbr::bw_inuse
uint64_t bw_inuse
Definition: tcp_log_buf.h:77
tcp_log_bbr::lost
uint32_t lost
Definition: tcp_log_buf.h:91
tcp_log_bbr::bbr_substate
uint8_t bbr_substate
Definition: tcp_log_buf.h:96
tcp_log_bbr::inflight
uint32_t inflight
Definition: tcp_log_buf.h:78
tcp_log_bbr::use_lt_bw
uint8_t use_lt_bw
Definition: tcp_log_buf.h:99
tcp_log_bbr::flex3
uint32_t flex3
Definition: tcp_log_buf.h:87
tcp_log_bbr::cwnd_gain
uint16_t cwnd_gain
Definition: tcp_log_buf.h:93
tcp_log_bbr::timeStamp
uint32_t timeStamp
Definition: tcp_log_buf.h:81
tcp_log_bbr::flex5
uint32_t flex5
Definition: tcp_log_buf.h:89
tcp_log_bbr::lt_epoch
uint32_t lt_epoch
Definition: tcp_log_buf.h:83
tcp_log_bbr::pacing_gain
uint16_t pacing_gain
Definition: tcp_log_buf.h:92
tcp_log_bbr::flex2
uint32_t flex2
Definition: tcp_log_buf.h:86
tcp_log_bbr::bbr_state
uint8_t bbr_state
Definition: tcp_log_buf.h:95
tcp_log_bbr::rttProp
uint64_t rttProp
Definition: tcp_log_buf.h:76
tcp_log_bbr::pkt_epoch
uint32_t pkt_epoch
Definition: tcp_log_buf.h:101
tcp_log_bbr::flex4
uint32_t flex4
Definition: tcp_log_buf.h:88
tcp_log_bbr::flex1
uint32_t flex1
Definition: tcp_log_buf.h:85
tcp_log_bbr::inhpts
uint8_t inhpts
Definition: tcp_log_buf.h:97
tcp_log_bbr::applimited
uint32_t applimited
Definition: tcp_log_buf.h:79
tcp_log_bbr::flex7
uint16_t flex7
Definition: tcp_log_buf.h:94
tcp_log_bbr::flex6
uint32_t flex6
Definition: tcp_log_buf.h:90
tcp_log_bbr::pkts_out
uint32_t pkts_out
Definition: tcp_log_buf.h:84
tcp_log_bbr::delivered
uint32_t delivered
Definition: tcp_log_buf.h:80
tcp_log_bbr::flex8
uint8_t flex8
Definition: tcp_log_buf.h:100
tcp_log_bbr::epoch
uint32_t epoch
Definition: tcp_log_buf.h:82
tcp_log_bbr::cur_del_rate
uint64_t cur_del_rate
Definition: tcp_log_buf.h:74
tcp_log_buffer
Definition: tcp_log_buf.h:112
tcp_log_buffer::tlb_errno
int tlb_errno
Definition: tcp_log_buf.h:125
tcp_rate_set::rs_min_seg
int rs_min_seg
Definition: tcp_ratelimit.h:71
tcp_rate_set::rs_ifp
struct ifnet * rs_ifp
Definition: tcp_ratelimit.h:65
tcpcb
Definition: tcp_var.h:132
tcpcb::last_ack_sent
tcp_seq last_ack_sent
Definition: tcp_var.h:174
tcpcb::iss
tcp_seq iss
Definition: tcp_var.h:194
tcpcb::t_logstate
uint32_t t_logstate
Definition: tcp_var.h:138
tcpcb::snd_recover
tcp_seq snd_recover
Definition: tcp_var.h:198
tcpcb::t_rcvtime
u_int t_rcvtime
Definition: tcp_var.h:175
tcpcb::rcv_up
tcp_seq rcv_up
Definition: tcp_var.h:177
tcpcb::snd_nxt
tcp_seq snd_nxt
Definition: tcp_var.h:151
tcpcb::t_softerror
int t_softerror
Definition: tcp_var.h:219
tcpcb::t_starttime
u_int t_starttime
Definition: tcp_var.h:208
tcpcb::server
uint64_t server
Definition: tcp_var.h:279
tcpcb::t_sndrexmitpack
int t_sndrexmitpack
Definition: tcp_var.h:237
tcpcb::t_tsomaxsegsize
u_int t_tsomaxsegsize
Definition: tcp_var.h:162
tcpcb::t_tfo_pending
unsigned int * t_tfo_pending
Definition: tcp_var.h:276
tcpcb::gput_seq
tcp_seq gput_seq
Definition: tcp_var.h:262
tcpcb::sackblks
struct sackblk sackblks[MAX_SACK_BLKS]
Definition: tcp_var.h:232
tcpcb::snd_scale
u_char snd_scale
Definition: tcp_var.h:170
tcpcb::t_badrxtwin
u_int t_badrxtwin
Definition: tcp_var.h:228
tcpcb::t_tsomax
u_int t_tsomax
Definition: tcp_var.h:160
tcpcb::request_r_scale
u_char request_r_scale
Definition: tcp_var.h:173
tcpcb::snd_wnd
uint32_t snd_wnd
Definition: tcp_var.h:153
tcpcb::snd_up
tcp_seq snd_up
Definition: tcp_var.h:152
tcpcb::snd_max
tcp_seq snd_max
Definition: tcp_var.h:148
tcpcb::snd_cwnd_prev
uint32_t snd_cwnd_prev
Definition: tcp_var.h:222
tcpcb::gput_ack
tcp_seq gput_ack
Definition: tcp_var.h:263
tcpcb::t_tfo_cookie
union tcpcb::@55 t_tfo_cookie
tcpcb::snd_una
tcp_seq snd_una
Definition: tcp_var.h:147
tcpcb::t_state
uint32_t t_state
Definition: tcp_var.h:140
tcpcb::gput_ts
uint32_t gput_ts
Definition: tcp_var.h:261
tcpcb::t_maxunacktime
u_int t_maxunacktime
Definition: tcp_var.h:244
tcpcb::rfbuf_ts
u_int32_t rfbuf_ts
Definition: tcp_var.h:158
tcpcb::t_rttbest
u_int t_rttbest
Definition: tcp_var.h:217
tcpcb::rcv_scale
u_char rcv_scale
Definition: tcp_var.h:171
tcpcb::t_rttupdated
u_long t_rttupdated
Definition: tcp_var.h:226
tcpcb::t_stats_gput_prev
int32_t t_stats_gput_prev
Definition: tcp_var.h:264
tcpcb::t_rttvar
int t_rttvar
Definition: tcp_var.h:168
tcpcb::snd_recover_prev
tcp_seq snd_recover_prev
Definition: tcp_var.h:224
tcpcb::t_acktime
u_int t_acktime
Definition: tcp_var.h:195
tcpcb::t_fbyte_in
u_int t_fbyte_in
Definition: tcp_var.h:209
tcpcb::snd_ssthresh
uint32_t snd_ssthresh
Definition: tcp_var.h:185
tcpcb::t_pmtud_saved_maxseg
u_int t_pmtud_saved_maxseg
Definition: tcp_var.h:212
tcpcb::t_stats
struct statsblob * t_stats
Definition: tcp_var.h:259
tcpcb::snd_wl2
tcp_seq snd_wl2
Definition: tcp_var.h:191
tcpcb::rcv_wnd
uint32_t rcv_wnd
Definition: tcp_var.h:165
tcpcb::t_fin_is_rst
uint32_t t_fin_is_rst
Definition: tcp_var.h:143
tcpcb::ts_recent
u_int32_t ts_recent
Definition: tcp_var.h:169
tcpcb::t_tfo_client_cookie_len
uint8_t t_tfo_client_cookie_len
Definition: tcp_var.h:274
tcpcb::t_srtt
int t_srtt
Definition: tcp_var.h:167
tcpcb::ts_recent_age
u_int ts_recent_age
Definition: tcp_var.h:197
tcpcb::t_delayed_ack
uint32_t t_delayed_ack
Definition: tcp_var.h:142
tcpcb::snd_ssthresh_prev
uint32_t snd_ssthresh_prev
Definition: tcp_var.h:223
tcpcb::ts_offset
u_int32_t ts_offset
Definition: tcp_var.h:157
tcpcb::snd_wl1
tcp_seq snd_wl1
Definition: tcp_var.h:189
tcpcb::t_tsomaxsegcount
u_int t_tsomaxsegcount
Definition: tcp_var.h:161
tcpcb::irs
tcp_seq irs
Definition: tcp_var.h:193
tcpcb::t_fbyte_out
u_int t_fbyte_out
Definition: tcp_var.h:210
tcpcb::t_rxtcur
int t_rxtcur
Definition: tcp_var.h:202
tcpcb::t_rxtshift
int t_rxtshift
Definition: tcp_var.h:204
tcpcb::t_flags2
u_int t_flags2
Definition: tcp_var.h:166
tcpcb::rcv_nxt
tcp_seq rcv_nxt
Definition: tcp_var.h:163
tcpcb::t_flags
u_int t_flags
Definition: tcp_var.h:146
tcpcb::t_rtttime
u_int t_rtttime
Definition: tcp_var.h:205
tcpcb::t_port
uint32_t t_port
Definition: tcp_var.h:139
tcpcb::t_maxpeakrate
uint32_t t_maxpeakrate
Definition: tcp_var.h:265
tcpcb::t_maxseg
uint32_t t_maxseg
Definition: tcp_var.h:137
tcpcb::max_sndwnd
uint32_t max_sndwnd
Definition: tcp_var.h:220
tcpcb::snd_cwnd
uint32_t snd_cwnd
Definition: tcp_var.h:154
tcpcb::t_peakrate_thr
uint32_t t_peakrate_thr
Definition: tcp_var.h:155
tcpcb::t_in_pkt
struct mbuf * t_in_pkt
Definition: tcp_var.h:181
tcpcb::client
uint8_t client[TCP_FASTOPEN_MAX_COOKIE_LEN]
Definition: tcp_var.h:278
tcpcb::t_sndzerowin
int t_sndzerowin
Definition: tcp_var.h:225
tcpcb::t_fb_ptr
void * t_fb_ptr
Definition: tcp_var.h:136
tcpcb::t_fb
struct tcp_function_block * t_fb
Definition: tcp_var.h:135
tcpcb::rcv_numsacks
int rcv_numsacks
Definition: tcp_var.h:159
tcpcb::t_inpcb
struct inpcb * t_inpcb
Definition: tcp_var.h:134
tcpcb::rcv_adv
tcp_seq rcv_adv
Definition: tcp_var.h:164
tcpiphdr
Definition: tcpip.h:41
tcpopt::to_tfo_len
u_int8_t to_tfo_len
Definition: tcp_var.h:595
tcpopt::to_signature
u_char * to_signature
Definition: tcp_var.h:590
tcpopt::to_tsval
u_int32_t to_tsval
Definition: tcp_var.h:587
tcpopt::to_nsacks
u_int8_t to_nsacks
Definition: tcp_var.h:594
tcpopt::to_sacks
u_char * to_sacks
Definition: tcp_var.h:589
tcpopt::to_mss
u_int16_t to_mss
Definition: tcp_var.h:592
tcpopt::to_flags
u_int32_t to_flags
Definition: tcp_var.h:578
tcpopt::to_wscale
u_int8_t to_wscale
Definition: tcp_var.h:593
tcpopt::to_tfo_cookie
u_int8_t * to_tfo_cookie
Definition: tcp_var.h:591
tcpopt::to_tsecr
u_int32_t to_tsecr
Definition: tcp_var.h:588
tcptemp::tt_ipgen
u_char tt_ipgen[40]
Definition: tcp_var.h:294
tcptemp::tt_t
struct tcphdr tt_t
Definition: tcp_var.h:295
udphdr
Definition: udp.h:45
udphdr::uh_ulen
u_short uh_ulen
Definition: udp.h:48
udphdr::uh_sport
u_short uh_sport
Definition: udp.h:46
udphdr::uh_sum
u_short uh_sum
Definition: udp.h:49
udphdr::uh_dport
u_short uh_dport
Definition: udp.h:47
VOI_TCP_FRWIN
#define VOI_TCP_FRWIN
Definition: tcp.h:430
VOI_TCP_RETXPB
#define VOI_TCP_RETXPB
Definition: tcp.h:429
VOI_TCP_GPUT_ND
#define VOI_TCP_GPUT_ND
Definition: tcp.h:436
VOI_TCP_CSIG
#define VOI_TCP_CSIG
Definition: tcp.h:433
VOI_TCP_TXPB
#define VOI_TCP_TXPB
Definition: tcp.h:428
VOI_TCP_GPUT
#define VOI_TCP_GPUT
Definition: tcp.h:434
VOI_TCP_RTT
#define VOI_TCP_RTT
Definition: tcp.h:432
tcp_bbr.h
BBR_STATE_STARTUP
#define BBR_STATE_STARTUP
Definition: tcp_bbr.h:335
BBR_MARKED_LOST
#define BBR_MARKED_LOST
Definition: tcp_bbr.h:42
BBR_HIGHSPEED_NUM_MSS
#define BBR_HIGHSPEED_NUM_MSS
Definition: tcp_bbr.h:148
BBR_STARTUP_EPOCHS
#define BBR_STARTUP_EPOCHS
Definition: tcp_bbr.h:321
BBR_RTTS_SHRINK_PG_FINAL
#define BBR_RTTS_SHRINK_PG_FINAL
Definition: tcp_bbr.h:191
BBR_OPTS_SIZE
#define BBR_OPTS_SIZE
Definition: tcp_bbr.h:489
r_start
uint32_t r_start
Definition: tcp_bbr.h:2
BBR_STATE_PROBE_BW
#define BBR_STATE_PROBE_BW
Definition: tcp_bbr.h:337
BBR_NUM_RTTS_FOR_DEL_LIMIT
#define BBR_NUM_RTTS_FOR_DEL_LIMIT
Definition: tcp_bbr.h:315
BBR_RTTS_PERSIST
#define BBR_RTTS_PERSIST
Definition: tcp_bbr.h:187
BBR_MIN_SEG
#define BBR_MIN_SEG
Definition: tcp_bbr.h:105
BBR_RTTS_NEW_TARGET
#define BBR_RTTS_NEW_TARGET
Definition: tcp_bbr.h:192
BBR_RWND_COLLAPSED
#define BBR_RWND_COLLAPSED
Definition: tcp_bbr.h:43
BBR_SACKED
#define BBR_SACKED
Definition: tcp_bbr.h:143
r_ctl
struct bbr_control r_ctl
Definition: tcp_bbr.h:77
BBR_RTTS_INIT
#define BBR_RTTS_INIT
Definition: tcp_bbr.h:183
BBR_RTTS_RTTPROBE
#define BBR_RTTS_RTTPROBE
Definition: tcp_bbr.h:185
BBR_RTT_RACK
#define BBR_RTT_RACK
Definition: tcp_bbr.h:139
BBR_TLP
#define BBR_TLP
Definition: tcp_bbr.h:40
BBR_RTTS_NEWRTT
#define BBR_RTTS_NEWRTT
Definition: tcp_bbr.h:184
BBR_RTT_BY_THIS_RETRAN
#define BBR_RTT_BY_THIS_RETRAN
Definition: tcp_bbr.h:126
BBR_TO_FRM_TLP
#define BBR_TO_FRM_TLP
Definition: tcp_bbr.h:269
USECS_IN_MSEC
#define USECS_IN_MSEC
Definition: tcp_bbr.h:322
BBR_RTT_PROP
#define BBR_RTT_PROP
Definition: tcp_bbr.h:138
BBR_SUB_LEVEL1
#define BBR_SUB_LEVEL1
Definition: tcp_bbr.h:344
BBR_RECOVERY_LOWRTT
#define BBR_RECOVERY_LOWRTT
Definition: tcp_bbr.h:498
BBR_NUM_OF_RETRANS
#define BBR_NUM_OF_RETRANS
Definition: tcp_bbr.h:44
BBR_SUB_DRAIN
#define BBR_SUB_DRAIN
Definition: tcp_bbr.h:343
r_rtt_not_allowed
uint8_t r_rtt_not_allowed
Definition: tcp_bbr.h:11
BBR_RTTS_RESETS_VALUES
#define BBR_RTTS_RESETS_VALUES
Definition: tcp_bbr.h:194
BBR_STAT_SIZE
#define BBR_STAT_SIZE
Definition: tcp_bbr.h:485
BBR_OPTS_INC
#define BBR_OPTS_INC(name)
Definition: tcp_bbr.h:492
BBR_INCL_IP_OH
#define BBR_INCL_IP_OH
Definition: tcp_bbr.h:48
BBR_RED_BW_USELRBW
#define BBR_RED_BW_USELRBW
Definition: tcp_bbr.h:113
BBR_RTT_PKTRTT
#define BBR_RTT_PKTRTT
Definition: tcp_bbr.h:140
BBR_SRTT
#define BBR_SRTT
Definition: tcp_bbr.h:141
BBR_UNIT
#define BBR_UNIT
Definition: tcp_bbr.h:313
BBR_CUM_ACKED
#define BBR_CUM_ACKED
Definition: tcp_bbr.h:144
BBR_OVERMAX
#define BBR_OVERMAX
Definition: tcp_bbr.h:36
BBR_ACKED
#define BBR_ACKED
Definition: tcp_bbr.h:33
BBR_TO_FRM_DELACK
#define BBR_TO_FRM_DELACK
Definition: tcp_bbr.h:273
BBR_RTT_BY_EXACTMATCH
#define BBR_RTT_BY_EXACTMATCH
Definition: tcp_bbr.h:124
BBR_TO_FRM_RACK
#define BBR_TO_FRM_RACK
Definition: tcp_bbr.h:270
BBR_HAS_SYN
#define BBR_HAS_SYN
Definition: tcp_bbr.h:41
BBR_INCL_ENET_OH
#define BBR_INCL_ENET_OH
Definition: tcp_bbr.h:47
BBR_WAS_RENEGED
#define BBR_WAS_RENEGED
Definition: tcp_bbr.h:34
BBR_RTTS_SHRINK_PG
#define BBR_RTTS_SHRINK_PG
Definition: tcp_bbr.h:190
BBR_SUB_LEVEL6
#define BBR_SUB_LEVEL6
Definition: tcp_bbr.h:349
BBR_HIGH_SPEED
#define BBR_HIGH_SPEED
Definition: tcp_bbr.h:147
BBR_TIMER_FUDGE
#define BBR_TIMER_FUDGE
Definition: tcp_bbr.h:108
BBR_INITIAL_RTO
#define BBR_INITIAL_RTO
Definition: tcp_bbr.h:31
BBR_STAT_ADD
#define BBR_STAT_ADD(name, amm)
Definition: tcp_bbr.h:487
BBR_RTT_BY_SOME_RETRAN
#define BBR_RTT_BY_SOME_RETRAN
Definition: tcp_bbr.h:127
BBR_HAS_FIN
#define BBR_HAS_FIN
Definition: tcp_bbr.h:39
BBR_RTT_BY_EARLIER_RET
#define BBR_RTT_BY_EARLIER_RET
Definition: tcp_bbr.h:125
BBR_SUB_GAIN
#define BBR_SUB_GAIN
Definition: tcp_bbr.h:342
BBR_PROBERTT_NUM_MSS
#define BBR_PROBERTT_NUM_MSS
Definition: tcp_bbr.h:320
BBR_RTTS_REACHTAR
#define BBR_RTTS_REACHTAR
Definition: tcp_bbr.h:188
BBR_STATE_PROBE_RTT
#define BBR_STATE_PROBE_RTT
Definition: tcp_bbr.h:338
BBR_INCL_TCP_OH
#define BBR_INCL_TCP_OH
Definition: tcp_bbr.h:49
BBR_WAS_SACKPASS
#define BBR_WAS_SACKPASS
Definition: tcp_bbr.h:38
BBR_RTTS_LEAVE_DRAIN
#define BBR_RTTS_LEAVE_DRAIN
Definition: tcp_bbr.h:193
BBR_TO_FRM_TMR
#define BBR_TO_FRM_TMR
Definition: tcp_bbr.h:268
BBR_STATE_DRAIN
#define BBR_STATE_DRAIN
Definition: tcp_bbr.h:336
BBR_TO_FRM_KEEP
#define BBR_TO_FRM_KEEP
Definition: tcp_bbr.h:271
BBR_SACK_PASSED
#define BBR_SACK_PASSED
Definition: tcp_bbr.h:37
BBR_RTT_BY_TIMESTAMP
#define BBR_RTT_BY_TIMESTAMP
Definition: tcp_bbr.h:123
BBR_RTTS_WASIDLE
#define BBR_RTTS_WASIDLE
Definition: tcp_bbr.h:186
BBR_LIMIT_TYPE_SPLIT
#define BBR_LIMIT_TYPE_SPLIT
Definition: tcp_bbr.h:99
BBR_TO_FRM_PERSIST
#define BBR_TO_FRM_PERSIST
Definition: tcp_bbr.h:272
BBR_SUBSTATE_COUNT
#define BBR_SUBSTATE_COUNT
Definition: tcp_bbr.h:350
BBR_NUM_RTTS_FOR_GOOG_DEL_LIMIT
#define BBR_NUM_RTTS_FOR_GOOG_DEL_LIMIT
Definition: tcp_bbr.h:316
BBR_STAT_INC
#define BBR_STAT_INC(name)
Definition: tcp_bbr.h:488
BBR_STATE_IDLE_EXIT
#define BBR_STATE_IDLE_EXIT
Definition: tcp_bbr.h:339
BBR_RXT_CLEARED
#define BBR_RXT_CLEARED
Definition: tcp_bbr.h:35
BBR_RTT_BY_TSMATCHING
#define BBR_RTT_BY_TSMATCHING
Definition: tcp_bbr.h:128
BBR_RTTS_ENTERPROBE
#define BBR_RTTS_ENTERPROBE
Definition: tcp_bbr.h:189
BBR_MAX_GAIN_VALUE
#define BBR_MAX_GAIN_VALUE
Definition: tcp_bbr.h:106
tcp_trace
void tcp_trace(short act, short ostate, struct tcpcb *tp, void *ipgen, struct tcphdr *th, int req)
Definition: tcp_debug.c:99
tcp_debug.h
IP6_HDR_LEN
#define IP6_HDR_LEN
Definition: tcp_debug.h:50
TA_OUTPUT
#define TA_OUTPUT
Definition: tcp_debug.h:64
TA_INPUT
#define TA_INPUT
Definition: tcp_debug.h:63
tcp_fastopen_disable_path
void tcp_fastopen_disable_path(struct tcpcb *tp)
Definition: tcp_fastopen.c:956
tcp_fastopen_decrement_counter
void tcp_fastopen_decrement_counter(unsigned int *counter)
Definition: tcp_fastopen.c:479
tcp_fastopen_update_cache
void tcp_fastopen_update_cache(struct tcpcb *tp, uint16_t mss, uint8_t cookie_len, uint8_t *cookie)
Definition: tcp_fastopen.c:981
tcp_fastopen.h
TCP_FASTOPEN_COOKIE_LEN
#define TCP_FASTOPEN_COOKIE_LEN
Definition: tcp_fastopen.h:36
tcp_fsm.h
TCPS_CLOSING
#define TCPS_CLOSING
Definition: tcp_fsm.h:56
TCPS_HAVERCVDSYN
#define TCPS_HAVERCVDSYN(s)
Definition: tcp_fsm.h:62
TCPS_FIN_WAIT_1
#define TCPS_FIN_WAIT_1
Definition: tcp_fsm.h:55
TCPS_TIME_WAIT
#define TCPS_TIME_WAIT
Definition: tcp_fsm.h:60
TCPS_HAVERCVDFIN
#define TCPS_HAVERCVDFIN(s)
Definition: tcp_fsm.h:64
TCPS_ESTABLISHED
#define TCPS_ESTABLISHED
Definition: tcp_fsm.h:52
TCPS_SYN_SENT
#define TCPS_SYN_SENT
Definition: tcp_fsm.h:49
TCPS_SYN_RECEIVED
#define TCPS_SYN_RECEIVED
Definition: tcp_fsm.h:50
TCPS_LAST_ACK
#define TCPS_LAST_ACK
Definition: tcp_fsm.h:57
TCPS_CLOSE_WAIT
#define TCPS_CLOSE_WAIT
Definition: tcp_fsm.h:53
TCPS_LISTEN
#define TCPS_LISTEN
Definition: tcp_fsm.h:48
TCPS_FIN_WAIT_2
#define TCPS_FIN_WAIT_2
Definition: tcp_fsm.h:59
TCPS_HAVEESTABLISHED
#define TCPS_HAVEESTABLISHED(s)
Definition: tcp_fsm.h:63
TCPS_CLOSED
#define TCPS_CLOSED
Definition: tcp_fsm.h:47
msec
#define msec(u)
tcp_hpts_remove
void tcp_hpts_remove(struct inpcb *inp)
Definition: tcp_hpts.c:563
tcp_min_hptsi_time
int32_t tcp_min_hptsi_time
Definition: tcp_hpts.c:253
tcp_in_hpts
bool tcp_in_hpts(struct inpcb *inp)
Definition: tcp_hpts.c:610
tcp_hpts_insert_diag
uint32_t tcp_hpts_insert_diag(struct inpcb *inp, uint32_t slot, int32_t line, struct hpts_diag *diag)
Definition: tcp_hpts.c:815
tcp_hpts.h
HPTS_USEC_TO_SLOTS
#define HPTS_USEC_TO_SLOTS(x)
Definition: tcp_hpts.h:34
PACE_TMR_PERSIT
#define PACE_TMR_PERSIT
Definition: tcp_hpts.h:66
tcp_tv_to_mssectick
static __inline uint32_t tcp_tv_to_mssectick(const struct timeval *sv)
Definition: tcp_hpts.h:174
PACE_TMR_RACK
#define PACE_TMR_RACK
Definition: tcp_hpts.h:63
PACE_TMR_MASK
#define PACE_TMR_MASK
Definition: tcp_hpts.h:69
PACE_TMR_DELACK
#define PACE_TMR_DELACK
Definition: tcp_hpts.h:62
tcp_tv_to_usectick
static __inline uint32_t tcp_tv_to_usectick(const struct timeval *sv)
Definition: tcp_hpts.h:168
tcp_set_hpts
#define tcp_set_hpts(a)
Definition: tcp_hpts.h:145
PACE_PKT_OUTPUT
#define PACE_PKT_OUTPUT
Definition: tcp_hpts.h:68
PACE_TMR_TLP
#define PACE_TMR_TLP
Definition: tcp_hpts.h:64
tcp_get_usecs
static __inline uint32_t tcp_get_usecs(struct timeval *tv)
Definition: tcp_hpts.h:198
PACE_TMR_KEEP
#define PACE_TMR_KEEP
Definition: tcp_hpts.h:67
tcp_hpts_insert
#define tcp_hpts_insert(inp, slot)
Definition: tcp_hpts.h:141
PACE_TMR_RXT
#define PACE_TMR_RXT
Definition: tcp_hpts.h:65
tcp_autorcvbuf
int tcp_autorcvbuf(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp, int tlen)
Definition: tcp_input.c:1465
tcp_dooptions
void tcp_dooptions(struct tcpopt *to, u_char *cp, int cnt, int flags)
Definition: tcp_input.c:3422
cc_conn_init
void cc_conn_init(struct tcpcb *tp)
Definition: tcp_input.c:368
tcp_mssopt
int tcp_mssopt(struct in_conninfo *inc)
Definition: tcp_input.c:3891
tcp_mss
void tcp_mss(struct tcpcb *tp, int offer)
Definition: tcp_input.c:3814
tcp_mss_update
void tcp_mss_update(struct tcpcb *tp, int offer, int mtuoffer, struct hc_metrics_lite *metricptr, struct tcp_ifcap *cap)
Definition: tcp_input.c:3665
tcp_log_event_
struct tcp_log_buffer * tcp_log_event_(struct tcpcb *tp, struct tcphdr *th, struct sockbuf *rxbuf, struct sockbuf *txbuf, uint8_t eventid, int errornum, uint32_t len, union tcp_log_stackspecific *stackinfo, int th_hostorder, const char *output_caller, const char *func, int line, const struct timeval *itv)
Definition: tcp_log_buf.c:1520
tcp_log_buf.h
TCP_LOG_STATE_OFF
@ TCP_LOG_STATE_OFF
Definition: tcp_log_buf.h:244
ERRNO_UNK
#define ERRNO_UNK
Definition: tcp_log_buf.h:257
TCP_LOG_EVENTP
#define TCP_LOG_EVENTP(tp, th, rxbuf, txbuf, eventid, errornum, len, stackinfo, th_hostorder, tv)
Definition: tcp_log_buf.h:346
BBR_LOG_ENOBUF_JMP
@ BBR_LOG_ENOBUF_JMP
Definition: tcp_log_buf.h:215
BBR_LOG_BBRUPD
@ BBR_LOG_BBRUPD
Definition: tcp_log_buf.h:182
TCP_LOG_FLOWEND
@ TCP_LOG_FLOWEND
Definition: tcp_log_buf.h:198
TCP_LOG_IN
@ TCP_LOG_IN
Definition: tcp_log_buf.h:174
BBR_LOG_TSTMP_VAL
@ BBR_LOG_TSTMP_VAL
Definition: tcp_log_buf.h:226
BBR_LOG_HPTSDIAG
@ BBR_LOG_HPTSDIAG
Definition: tcp_log_buf.h:210
BBR_LOG_MSGSIZE
@ BBR_LOG_MSGSIZE
Definition: tcp_log_buf.h:192
BBR_LOG_STATE_TARGET
@ BBR_LOG_STATE_TARGET
Definition: tcp_log_buf.h:206
BBR_LOG_TIMERSTAR
@ BBR_LOG_TIMERSTAR
Definition: tcp_log_buf.h:186
BBR_LOG_CWND
@ BBR_LOG_CWND
Definition: tcp_log_buf.h:190
BBR_LOG_TO_PROCESS
@ BBR_LOG_TO_PROCESS
Definition: tcp_log_buf.h:208
BBR_LOG_LOWGAIN
@ BBR_LOG_LOWGAIN
Definition: tcp_log_buf.h:211
BBR_LOG_STATE
@ BBR_LOG_STATE
Definition: tcp_log_buf.h:195
BBR_LOG_TIMERPREP
@ BBR_LOG_TIMERPREP
Definition: tcp_log_buf.h:214
BBR_LOG_RTO
@ BBR_LOG_RTO
Definition: tcp_log_buf.h:199
BBR_LOG_PERSIST
@ BBR_LOG_PERSIST
Definition: tcp_log_buf.h:197
BBR_LOG_DOSEG_DONE
@ BBR_LOG_DOSEG_DONE
Definition: tcp_log_buf.h:200
BBR_LOG_PROGRESS
@ BBR_LOG_PROGRESS
Definition: tcp_log_buf.h:212
BBR_LOG_EXIT_GAIN
@ BBR_LOG_EXIT_GAIN
Definition: tcp_log_buf.h:201
BBR_LOG_EXITREC
@ BBR_LOG_EXITREC
Definition: tcp_log_buf.h:189
BBR_LOG_ENTREC
@ BBR_LOG_ENTREC
Definition: tcp_log_buf.h:188
BBR_LOG_REDUCE
@ BBR_LOG_REDUCE
Definition: tcp_log_buf.h:219
BBR_LOG_TIME_EPOCH
@ BBR_LOG_TIME_EPOCH
Definition: tcp_log_buf.h:207
BBR_LOG_RTT_SHRINKS
@ BBR_LOG_RTT_SHRINKS
Definition: tcp_log_buf.h:217
BBR_LOG_PKT_EPOCH
@ BBR_LOG_PKT_EPOCH
Definition: tcp_log_buf.h:196
TCP_LOG_RTT
@ TCP_LOG_RTT
Definition: tcp_log_buf.h:220
BBR_LOG_BWSAMP
@ BBR_LOG_BWSAMP
Definition: tcp_log_buf.h:191
BBR_LOG_TIMERCANC
@ BBR_LOG_TIMERCANC
Definition: tcp_log_buf.h:187
BBR_LOG_JUSTRET
@ BBR_LOG_JUSTRET
Definition: tcp_log_buf.h:194
TCP_LOG_OUT
@ TCP_LOG_OUT
Definition: tcp_log_buf.h:175
BBR_LOG_HPTSI_CALC
@ BBR_LOG_HPTSI_CALC
Definition: tcp_log_buf.h:216
BBR_RSM_CLEARED
@ BBR_RSM_CLEARED
Definition: tcp_log_buf.h:205
BBR_LOG_ACKCLEAR
@ BBR_LOG_ACKCLEAR
Definition: tcp_log_buf.h:184
BBR_LOG_BW_RED_EV
@ BBR_LOG_BW_RED_EV
Definition: tcp_log_buf.h:218
BBR_LOG_BBRRTT
@ BBR_LOG_BBRRTT
Definition: tcp_log_buf.h:193
BBR_LOG_SETTINGS_CHG
@ BBR_LOG_SETTINGS_CHG
Definition: tcp_log_buf.h:221
BBR_LOG_BBRTSO
@ BBR_LOG_BBRTSO
Definition: tcp_log_buf.h:209
BBR_LOG_BBRSND
@ BBR_LOG_BBRSND
Definition: tcp_log_buf.h:183
BBR_LOG_HDWR_PACE
@ BBR_LOG_HDWR_PACE
Definition: tcp_log_buf.h:225
BBR_LOG_THRESH_CALC
@ BBR_LOG_THRESH_CALC
Definition: tcp_log_buf.h:202
tcp_lro_reg_mbufq
void tcp_lro_reg_mbufq(void)
Definition: tcp_lro.c:136
tcp_lro_dereg_mbufq
void tcp_lro_dereg_mbufq(void)
Definition: tcp_lro.c:142
tcp_lro.h
tcp_offload_output
int tcp_offload_output(struct tcpcb *tp)
Definition: tcp_offload.c:143
tcp_offload.h
tcp_addoptions
int tcp_addoptions(struct tcpopt *to, u_char *optp)
Definition: tcp_output.c:1790
tcp_sndbuf_autoscale
void tcp_sndbuf_autoscale(struct tcpcb *tp, struct socket *so, uint32_t sendwin)
Definition: tcp_output.c:2119
tcp_m_copym
struct mbuf * tcp_m_copym(struct mbuf *m, int32_t off0, int32_t *plen, int32_t seglimit, int32_t segsize, struct sockbuf *sb, bool hw_tls)
Definition: tcp_output.c:1954
delayed_ack
uint8_t delayed_ack
Definition: tcp_rack.h:99
USECS_IN_SECOND
#define USECS_IN_SECOND
Definition: tcp_ratelimit.c:64
tcp_ratelimit.h
tcp_chg_pacing_rate
static const struct tcp_hwrate_limit_table * tcp_chg_pacing_rate(const struct tcp_hwrate_limit_table *crte, struct tcpcb *tp, struct ifnet *ifp, uint64_t bytes_per_sec, int flags, int *error, uint64_t *lower_rate)
Definition: tcp_ratelimit.h:131
tcp_set_pacing_rate
static const struct tcp_hwrate_limit_table * tcp_set_pacing_rate(struct tcpcb *tp, struct ifnet *ifp, uint64_t bytes_per_sec, int flags, int *error, uint64_t *lower_rate)
Definition: tcp_ratelimit.h:122
RS_PACING_SUB_OK
#define RS_PACING_SUB_OK
Definition: tcp_ratelimit.h:86
tcp_rel_pacing_rate
static void tcp_rel_pacing_rate(const struct tcp_hwrate_limit_table *crte, struct tcpcb *tp)
Definition: tcp_ratelimit.h:141
RS_PACING_GEQ
#define RS_PACING_GEQ
Definition: tcp_ratelimit.h:84
tcp_reass
int tcp_reass(struct tcpcb *tp, struct tcphdr *th, tcp_seq *seq_start, int *tlenp, struct mbuf *m)
Definition: tcp_reass.c:526
tcp_update_dsack_list
void tcp_update_dsack_list(struct tcpcb *tp, tcp_seq rcv_start, tcp_seq rcv_end)
Definition: tcp_sack.c:176
tcp_clean_sackreport
void tcp_clean_sackreport(struct tcpcb *tp)
Definition: tcp_sack.c:451
tcp_update_sack_list
void tcp_update_sack_list(struct tcpcb *tp, tcp_seq rcv_start, tcp_seq rcv_end)
Definition: tcp_sack.c:272
tcp_clean_dsack_blocks
void tcp_clean_dsack_blocks(struct tcpcb *tp)
Definition: tcp_sack.c:409
tcp_seq.h
SEQ_GEQ
#define SEQ_GEQ(a, b)
Definition: tcp_seq.h:45
TSTMP_GEQ
#define TSTMP_GEQ(a, b)
Definition: tcp_seq.h:61
SEQ_GT
#define SEQ_GT(a, b)
Definition: tcp_seq.h:44
TSTMP_LT
#define TSTMP_LT(a, b)
Definition: tcp_seq.h:59
SEQ_LEQ
#define SEQ_LEQ(a, b)
Definition: tcp_seq.h:43
tcp_rcvseqinit
#define tcp_rcvseqinit(tp)
Definition: tcp_seq.h:68
TSTMP_GT
#define TSTMP_GT(a, b)
Definition: tcp_seq.h:60
SEQ_LT
#define SEQ_LT(a, b)
Definition: tcp_seq.h:42
tcp_drop
struct tcpcb * tcp_drop(struct tcpcb *tp, int errno)
Definition: tcp_subr.c:2283
deregister_tcp_functions
int deregister_tcp_functions(struct tcp_function_block *blk, bool quiesce, bool force)
Definition: tcp_subr.c:1338
register_tcp_functions_as_names
int register_tcp_functions_as_names(struct tcp_function_block *blk, int wait, const char *names[], int *num_names)
Definition: tcp_subr.c:1189
tcp_switch_back_to_default
void tcp_switch_back_to_default(struct tcpcb *tp)
Definition: tcp_subr.c:523
tcpip_fillheaders
void tcpip_fillheaders(struct inpcb *inp, uint16_t port, void *ip_ptr, void *tcp_ptr)
Definition: tcp_subr.c:1637
tcp_state_change
void tcp_state_change(struct tcpcb *tp, int newstate)
Definition: tcp_subr.c:3999
tcp_close
struct tcpcb * tcp_close(struct tcpcb *tp)
Definition: tcp_subr.c:2471
tcp_respond
void tcp_respond(struct tcpcb *tp, void *ipgen, struct tcphdr *th, struct mbuf *m, tcp_seq ack, tcp_seq seq, int flags)
Definition: tcp_subr.c:1728
tcp_log_end_status
void tcp_log_end_status(struct tcpcb *tp, uint8_t status)
Definition: tcp_subr.c:4081
tcpip_maketemplate
struct tcptemp * tcpip_maketemplate(struct inpcb *inp)
Definition: tcp_subr.c:1702
tcp_record_dsack
void tcp_record_dsack(struct tcpcb *tp, tcp_seq start, tcp_seq end, int tlp)
Definition: tcp_subr.c:397
tcp_maxpersistidle
int tcp_maxpersistidle
Definition: tcp_timer.c:165
tcp_totbackoff
int tcp_totbackoff
Definition: tcp_timer.c:257
tcp_timer_active
int tcp_timer_active(struct tcpcb *tp, uint32_t timer_type)
Definition: tcp_timer.c:905
tcp_backoff
int tcp_backoff[TCP_MAXRXTSHIFT+1]
Definition: tcp_timer.c:254
tcp_timer_suspend
int tcp_timer_suspend(struct tcpcb *tp, uint32_t timer_type)
Definition: tcp_timer.c:943
tcp_rexmit_drop_options
int tcp_rexmit_drop_options
Definition: tcp_timer.c:167
tcp_timer_activate
void tcp_timer_activate(struct tcpcb *tp, uint32_t timer_type, u_int delta)
Definition: tcp_timer.c:854
tcp_finwait2_timeout
int tcp_finwait2_timeout
Definition: tcp_timer.c:154
tcp_fast_finwait2_recycle
int tcp_fast_finwait2_recycle
Definition: tcp_timer.c:149
tcp_timer.h
TCP_MAXRXTSHIFT
#define TCP_MAXRXTSHIFT
Definition: tcp_timer.h:117
TP_MAXIDLE
#define TP_MAXIDLE(tp)
Definition: tcp_timer.h:184
TCPTV_PERSMAX
#define TCPTV_PERSMAX
Definition: tcp_timer.h:82
TP_KEEPINIT
#define TP_KEEPINIT(tp)
Definition: tcp_timer.h:180
V_tcp_always_keepalive
#define V_tcp_always_keepalive
Definition: tcp_timer.h:206
V_tcp_pmtud_blackhole_detect
#define V_tcp_pmtud_blackhole_detect
Definition: tcp_timer.h:208
TP_KEEPIDLE
#define TP_KEEPIDLE(tp)
Definition: tcp_timer.h:181
V_tcp_pmtud_blackhole_mss
#define V_tcp_pmtud_blackhole_mss
Definition: tcp_timer.h:210
TT_KEEP
#define TT_KEEP
Definition: tcp_timer.h:164
TCPT_RANGESET
#define TCPT_RANGESET(tv, value, tvmin, tvmax)
Definition: tcp_timer.h:136
V_tcp_v6pmtud_blackhole_mss
#define V_tcp_v6pmtud_blackhole_mss
Definition: tcp_timer.h:212
TT_2MSL
#define TT_2MSL
Definition: tcp_timer.h:165
TT_DELACK
#define TT_DELACK
Definition: tcp_timer.h:161
TT_PERSIST
#define TT_PERSIST
Definition: tcp_timer.h:163
TT_REXMT
#define TT_REXMT
Definition: tcp_timer.h:162
tcp_twstart
void tcp_twstart(struct tcpcb *tp)
Definition: tcp_timewait.c:236
tcp_default_ctloutput
int tcp_default_ctloutput(struct inpcb *inp, struct sockopt *sopt)
Definition: tcp_usrreq.c:2120
tcp_var.h
TCP_EI_STATUS_SERVER_FIN
#define TCP_EI_STATUS_SERVER_FIN
Definition: tcp_var.h:53
TOF_MSS
#define TOF_MSS
Definition: tcp_var.h:579
TF_SIGNATURE
#define TF_SIGNATURE
Definition: tcp_var.h:519
TCP_EI_STATUS_SERVER_RST
#define TCP_EI_STATUS_SERVER_RST
Definition: tcp_var.h:54
TF_ACKNOW
#define TF_ACKNOW
Definition: tcp_var.h:497
TCP_EI_STATUS_RST_IN_FRONT
#define TCP_EI_STATUS_RST_IN_FRONT
Definition: tcp_var.h:60
TOF_SIGNATURE
#define TOF_SIGNATURE
Definition: tcp_var.h:583
tcp_get_flags
static uint16_t tcp_get_flags(const struct tcphdr *th)
Definition: tcp_var.h:1265
TF2_PLPMTU_MAXSEGSNT
#define TF2_PLPMTU_MAXSEGSNT
Definition: tcp_var.h:561
SEGQ_EMPTY
#define SEGQ_EMPTY(tp)
Definition: tcp_var.h:123
TOF_FASTOPEN
#define TOF_FASTOPEN
Definition: tcp_var.h:585
TCP_EI_STATUS_KEEP_MAX
#define TCP_EI_STATUS_KEEP_MAX
Definition: tcp_var.h:58
TCP_DELTA_SHIFT
#define TCP_DELTA_SHIFT
Definition: tcp_var.h:662
tcp_set_flags
static void tcp_set_flags(struct tcphdr *th, uint16_t flags)
Definition: tcp_var.h:1271
TF_RCVD_SCALE
#define TF_RCVD_SCALE
Definition: tcp_var.h:503
TCP_FUNC_OUTPUT_CANDROP
#define TCP_FUNC_OUTPUT_CANDROP
Definition: tcp_var.h:322
V_tcp_udp_tunneling_port
#define V_tcp_udp_tunneling_port
Definition: tcp_var.h:1067
V_tcp_udp_tunneling_overhead
#define V_tcp_udp_tunneling_overhead
Definition: tcp_var.h:1066
TF_NEEDFIN
#define TF_NEEDFIN
Definition: tcp_var.h:508
TOF_SACKPERM
#define TOF_SACKPERM
Definition: tcp_var.h:581
TF_NOPUSH
#define TF_NOPUSH
Definition: tcp_var.h:509
TF_WAKESOR
#define TF_WAKESOR
Definition: tcp_var.h:511
intotcpcb
#define intotcpcb(ip)
Definition: tcp_var.h:645
TO_SYN
#define TO_SYN
Definition: tcp_var.h:602
TF_TSO
#define TF_TSO
Definition: tcp_var.h:521
V_tcp_mssdflt
#define V_tcp_mssdflt
Definition: tcp_var.h:1055
TF2_FBYTES_COMPLETE
#define TF2_FBYTES_COMPLETE
Definition: tcp_var.h:568
V_tcp_do_tso
#define V_tcp_do_tso
Definition: tcp_var.h:1047
TF_REQ_SCALE
#define TF_REQ_SCALE
Definition: tcp_var.h:502
TF_NOOPT
#define TF_NOOPT
Definition: tcp_var.h:500
TCP_REXMTVAL
#define TCP_REXMTVAL(tp)
Definition: tcp_var.h:680
V_tcp_map_entries_limit
#define V_tcp_map_entries_limit
Definition: tcp_var.h:1052
TF_DELACK
#define TF_DELACK
Definition: tcp_var.h:498
TF_SENTFIN
#define TF_SENTFIN
Definition: tcp_var.h:501
TF_PREVVALID
#define TF_PREVVALID
Definition: tcp_var.h:510
TF2_PLPMTU_PMTUD
#define TF2_PLPMTU_PMTUD
Definition: tcp_var.h:560
TOF_SCALE
#define TOF_SCALE
Definition: tcp_var.h:580
V_tcp_delack_enabled
#define V_tcp_delack_enabled
Definition: tcp_var.h:1035
TF2_DROP_AF_DATA
#define TF2_DROP_AF_DATA
Definition: tcp_var.h:563
V_tcp_initcwnd_segments
#define V_tcp_initcwnd_segments
Definition: tcp_var.h:1049
TF_TOE
#define TF_TOE
Definition: tcp_var.h:522
TF2_PLPMTU_BLACKHOLE
#define TF2_PLPMTU_BLACKHOLE
Definition: tcp_var.h:559
TF_GPUTINPROG
#define TF_GPUTINPROG
Definition: tcp_var.h:512
TF_RCVD_TSTMP
#define TF_RCVD_TSTMP
Definition: tcp_var.h:505
TCP_EI_STATUS_DATA_A_CLOSE
#define TCP_EI_STATUS_DATA_A_CLOSE
Definition: tcp_var.h:59
TOF_SACK
#define TOF_SACK
Definition: tcp_var.h:584
TF_RXWIN0SENT
#define TF_RXWIN0SENT
Definition: tcp_var.h:516
IN_RECOVERY
#define IN_RECOVERY(t_flags)
Definition: tcp_var.h:538
TCP_EI_STATUS_CLIENT_FIN
#define TCP_EI_STATUS_CLIENT_FIN
Definition: tcp_var.h:51
V_tcp_tolerate_missing_ts
#define V_tcp_tolerate_missing_ts
Definition: tcp_var.h:1040
BYTES_THIS_ACK
#define BYTES_THIS_ACK(tp, th)
Definition: tcp_var.h:548
ENTER_RECOVERY
#define ENTER_RECOVERY(t_flags)
Definition: tcp_var.h:539
V_tcp_map_split_limit
#define V_tcp_map_split_limit
Definition: tcp_var.h:1053
TF_NODELAY
#define TF_NODELAY
Definition: tcp_var.h:499
TCP_EI_STATUS_RETRAN
#define TCP_EI_STATUS_RETRAN
Definition: tcp_var.h:55
KMOD_TCPSTAT_INC
#define KMOD_TCPSTAT_INC(name)
Definition: tcp_var.h:850
IS_FASTOPEN
#define IS_FASTOPEN(t_flags)
Definition: tcp_var.h:543
TCP_RTTVAR_SHIFT
#define TCP_RTTVAR_SHIFT
Definition: tcp_var.h:661
TF_NEEDSYN
#define TF_NEEDSYN
Definition: tcp_var.h:507
TF_SACK_PERMIT
#define TF_SACK_PERMIT
Definition: tcp_var.h:506
V_path_mtu_discovery
#define V_path_mtu_discovery
Definition: tcp_var.h:1029
V_tcp_minmss
#define V_tcp_minmss
Definition: tcp_var.h:1054
KMOD_TCPSTAT_ADD
#define KMOD_TCPSTAT_ADD(name, val)
Definition: tcp_var.h:848
TF_REQ_TSTMP
#define TF_REQ_TSTMP
Definition: tcp_var.h:504
TF_MORETOCOME
#define TF_MORETOCOME
Definition: tcp_var.h:513
V_tcp_do_rfc3390
#define V_tcp_do_rfc3390
Definition: tcp_var.h:1043
TOF_TS
#define TOF_TS
Definition: tcp_var.h:582
EXIT_RECOVERY
#define EXIT_RECOVERY(t_flags)
Definition: tcp_var.h:540
TCP_EI_STATUS_PERSIST_MAX
#define TCP_EI_STATUS_PERSIST_MAX
Definition: tcp_var.h:57
tcp_account_for_send
static void tcp_account_for_send(struct tcpcb *tp, uint32_t len, uint8_t is_rxt, uint8_t is_tlp, int hw_tls)
Definition: tcp_var.h:1278
V_drop_synfin
#define V_drop_synfin
Definition: tcp_var.h:1028
TCP_RTT_SHIFT
#define TCP_RTT_SHIFT
Definition: tcp_var.h:659
udp_var.h
UDPSTAT_INC
#define UDPSTAT_INC(name)
Definition: udp_var.h:118
tcp_log_stackspecific
Definition: tcp_log_buf.h:106
tcp_log_stackspecific::u_bbr
struct tcp_log_bbr u_bbr
Definition: tcp_log_buf.h:108