39#include "opt_tcpdebug.h"
46#include <sys/kernel.h>
47#include <sys/eventhandler.h>
48#include <sys/malloc.h>
50#include <sys/socket.h>
51#include <sys/socketvar.h>
52#include <sys/sysctl.h>
53#include <sys/syslog.h>
59#include <net/if_var.h>
71#include <netinet6/in6_pcb.h>
72#include <netinet6/ip6_var.h>
73#include <netinet6/nd6.h>
79#ifdef TCP_REASS_LOGGING
83#include <netinet6/tcp6_var.h>
89#define TCP_R_LOG_ADD 1
90#define TCP_R_LOG_LIMIT_REACHED 2
91#define TCP_R_LOG_APPEND 3
92#define TCP_R_LOG_PREPEND 4
93#define TCP_R_LOG_REPLACE 5
94#define TCP_R_LOG_MERGE_INTO 6
95#define TCP_R_LOG_NEW_ENTRY 7
96#define TCP_R_LOG_READ 8
97#define TCP_R_LOG_ZERO 9
98#define TCP_R_LOG_DUMP 10
99#define TCP_R_LOG_TRIM 11
102 CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
103 "TCP Segment Reassembly Queue");
106 CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
107 "TCP Segment Reassembly stats");
110SYSCTL_INT(_net_inet_tcp_reass, OID_AUTO, maxsegments, CTLFLAG_RDTUN,
112 "Global maximum number of TCP Segments in Reassembly Queue");
117 "Global number of TCP Segments currently in Reassembly Queue");
120SYSCTL_UINT(_net_inet_tcp_reass, OID_AUTO, maxqueuelen, CTLFLAG_RWTUN,
122 "Maximum number of TCP Segments per Reassembly Queue");
125SYSCTL_INT(_net_inet_tcp_reass, OID_AUTO, new_limit, CTLFLAG_RWTUN,
127 "Do we use the new limit method we are discussing?");
130SYSCTL_UINT(_net_inet_tcp_reass, OID_AUTO, queueguard, CTLFLAG_RWTUN,
132 "Number of TCP Segments in Reassembly Queue where we flip over to guard mode");
134#ifdef TCP_REASS_COUNTERS
136counter_u64_t reass_entry;
138 &reass_entry,
"A segment entered reassembly ");
140counter_u64_t reass_path1;
142 &reass_path1,
"Took path 1");
144counter_u64_t reass_path2;
146 &reass_path2,
"Took path 2");
148counter_u64_t reass_path3;
150 &reass_path3,
"Took path 3");
152counter_u64_t reass_path4;
154 &reass_path4,
"Took path 4");
156counter_u64_t reass_path5;
158 &reass_path5,
"Took path 5");
160counter_u64_t reass_path6;
162 &reass_path6,
"Took path 6");
164counter_u64_t reass_path7;
166 &reass_path7,
"Took path 7");
168counter_u64_t reass_fullwalk;
170 &reass_fullwalk,
"Took a full walk ");
172counter_u64_t reass_nospace;
174 &reass_nospace,
"Had no mbuf capacity ");
176counter_u64_t merge_fwd;
178 &merge_fwd,
"Ran merge fwd");
180counter_u64_t merge_into;
182 &merge_into,
"Ran merge into");
184counter_u64_t tcp_zero_input;
186 &tcp_zero_input,
"The reassembly buffer saw a zero len segment etc");
201#ifdef TCP_REASS_LOGGING
205 tcp_seq seq,
int len,
uint8_t action,
int instance)
213 memset(&log, 0,
sizeof(log));
215 log.u_bbr.flex1 = seq;
216 log.u_bbr.cur_del_rate = (uint64_t)q;
217 log.u_bbr.delRate = (uint64_t)p;
226 log.u_bbr.pkts_out = p->
tqe_len;
231 log.u_bbr.flex7 = instance;
232 log.u_bbr.flex8 = action;
233 log.u_bbr.timeStamp = cts;
238 len, &log,
false, &tv);
243tcp_reass_log_dump(
struct tcpcb *tp)
248 TAILQ_FOREACH(q, &tp->
t_segq, tqe_q) {
255tcp_reass_log_new_in(
struct tcpcb *tp, tcp_seq seq,
int len,
struct mbuf *m,
267 tcp_log_reassm(tp, q, NULL, seq, len, logval, cnt);
277 TUNABLE_INT_FETCH(
"net.inet.tcp.reass.maxsegments",
280 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
284#ifdef TCP_REASS_COUNTERS
285 reass_path1 = counter_u64_alloc(M_WAITOK);
286 reass_path2 = counter_u64_alloc(M_WAITOK);
287 reass_path3 = counter_u64_alloc(M_WAITOK);
288 reass_path4 = counter_u64_alloc(M_WAITOK);
289 reass_path5 = counter_u64_alloc(M_WAITOK);
290 reass_path6 = counter_u64_alloc(M_WAITOK);
291 reass_path7 = counter_u64_alloc(M_WAITOK);
292 reass_fullwalk = counter_u64_alloc(M_WAITOK);
293 reass_nospace = counter_u64_alloc(M_WAITOK);
294 reass_entry = counter_u64_alloc(M_WAITOK);
295 merge_fwd = counter_u64_alloc(M_WAITOK);
296 merge_into = counter_u64_alloc(M_WAITOK);
297 tcp_zero_input = counter_u64_alloc(M_WAITOK);
299 EVENTHANDLER_REGISTER(nmbclusters_change,
311 while ((qe = TAILQ_FIRST(&tp->
t_segq)) != NULL) {
312 TAILQ_REMOVE(&tp->
t_segq, qe, tqe_q);
319 (
"TCP reass queue %p segment count is %d instead of 0 after flush.",
325 struct mbuf *m,
struct tcphdr *th,
int tlen,
326 struct mbuf *mlast,
int lenofoh)
329#ifdef TCP_REASS_LOGGING
333 last->
tqe_m->m_pkthdr.len += tlen;
342#ifdef TCP_REASS_LOGGING
351 int tlen,
struct mbuf *mlast,
int lenofoh)
355#ifdef TCP_REASS_LOGGING
360 i = (th->th_seq + tlen) - first->
tqe_start;
361#ifdef TCP_REASS_LOGGING
364 m_adj(first->
tqe_m, i);
369 mlast->m_next = first->
tqe_m;
378#ifdef TCP_REASS_LOGGING
387 tcp_seq seq,
int len,
struct mbuf *mlast,
int mbufoh,
uint16_t flags)
395#ifdef TCP_REASS_LOGGING
400 (
"Tp:%p seg queue goes negative", tp));
427#ifdef TCP_REASS_LOGGING
430#ifdef TCP_REASS_COUNTERS
431 counter_u64_add(merge_into, 1);
439 TAILQ_REMOVE(&tp->
t_segq, q, tqe_q);
457 q = TAILQ_NEXT(ent, tqe_q);
462 TAILQ_FOREACH_FROM_SAFE(q, &tp->
t_segq, tqe_q, qtmp) {
472 TAILQ_REMOVE(&tp->
t_segq, q, tqe_q);
483#ifdef TCP_REASS_LOGGING
492#ifdef TCP_REASS_COUNTERS
493 counter_u64_add(merge_fwd, 1);
502 if (m->m_flags & M_EXT)
503 len += m->m_ext.ext_size;
504 while (m->m_next != NULL) {
507 if (m->m_flags & M_EXT)
508 len += m->m_ext.ext_size;
527 int *tlenp,
struct mbuf *m)
533 struct mbuf *mlast = NULL;
537 int flags, i, lenofoh;
545 KASSERT(th == NULL || (seq_start != NULL && tlenp != NULL),
546 (
"tcp_reass called with illegal parameter combination "
547 "(tp=%p, th=%p, seq_start=%p, tlenp=%p, m=%p)",
548 tp, th, seq_start, tlenp, m));
556 (
"Attempt to add old entry to reassembly queue (th=%p, tp=%p)",
558#ifdef TCP_REASS_LOGGING
559 tcp_reass_log_new_in(tp, th->th_seq, *tlenp, m,
TCP_R_LOG_ADD, NULL);
561#ifdef TCP_REASS_COUNTERS
562 counter_u64_add(reass_entry, 1);
575#ifdef TCP_REASS_COUNTERS
576 counter_u64_add(tcp_zero_input, 1);
579#ifdef TCP_REASS_LOGGING
580 tcp_reass_log_dump(tp);
583 }
else if ((*tlenp == 0) &&
603 (sb->sb_mbcnt + tp->
t_segqmbuflen + lenofoh) > sb->sb_mbmax) {
606#ifdef TCP_REASS_COUNTERS
607 counter_u64_add(reass_nospace, 1);
609#ifdef TCP_REASS_LOGGING
613 log(LOG_DEBUG,
"%s; %s: mbuf count limit reached, "
614 "segment dropped\n", s, __func__);
619#ifdef TCP_REASS_LOGGING
620 tcp_reass_log_dump(tp);
650#ifdef TCP_REASS_COUNTERS
651 counter_u64_add(reass_path1, 1);
656#ifdef TCP_REASS_LOGGING
705#ifdef TCP_REASS_COUNTERS
706 counter_u64_add(reass_path2, 1);
715 first = TAILQ_FIRST(&tp->
t_segq);
736 struct mbuf *firstmbuf;
739#ifdef TCP_REASS_COUNTERS
740 counter_u64_add(reass_path3, 1);
750 panic(
"th->th_seq:%u rcv_nxt:%u tp:%p not pre-trimmed",
754#ifdef TCP_REASS_LOGGING
763 firstmbuf = first->
tqe_m;
767 if (firstmbuf == first->
tqe_m) {
768 panic(
"First stayed same m:%p foobar:%p first->tqe_m:%p tp:%p first:%p",
769 m, firstmbuf, first->
tqe_m, tp, first);
770 }
else if (first->
tqe_m != m) {
771 panic(
"First did not change to m:%p foobar:%p first->tqe_m:%p tp:%p first:%p",
772 m, firstmbuf, first->
tqe_m, tp, first);
793#ifdef TCP_REASS_COUNTERS
794 counter_u64_add(reass_fullwalk, 1);
796 TAILQ_FOREACH(q, &tp->
t_segq, tqe_q) {
800 p = TAILQ_PREV(q, tsegqe_head, tqe_q);
828#ifdef TCP_REASS_COUNTERS
829 counter_u64_add(reass_path4, 1);
861#ifdef TCP_REASS_COUNTERS
862 counter_u64_add(reass_path5, 1);
864#ifdef TCP_REASS_LOGGING
888 panic(
"Impossible cut th_seq:%u p->seq:%u(%d) p:%p tp:%p",
904#ifdef TCP_REASS_COUNTERS
905 counter_u64_add(reass_path6, 1);
915#ifdef TCP_REASS_COUNTERS
916 counter_u64_add(reass_path7, 1);
970 SOCKBUF_LOCK(&so->so_rcv);
971 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
974 sbappendstream_locked(&so->so_rcv, m, 0);
992 log(LOG_DEBUG,
"%s; %s: queue limit reached, "
993 "segment dropped\n", s, __func__);
997#ifdef TCP_REASS_LOGGING
998 tcp_reass_log_dump(tp);
1008 log(LOG_DEBUG,
"%s; %s: queue limit reached, "
1009 "segment dropped\n", s, __func__);
1013#ifdef TCP_REASS_LOGGING
1014 tcp_reass_log_dump(tp);
1030 log(LOG_DEBUG,
"%s; %s: global zone limit "
1031 "reached, segment dropped\n", s, __func__);
1049 TAILQ_INSERT_HEAD(&tp->
t_segq, te, tqe_q);
1051 TAILQ_INSERT_AFTER(&tp->
t_segq, p, te, tqe_q);
1053#ifdef TCP_REASS_LOGGING
1063 q = TAILQ_FIRST(&tp->
t_segq);
1065 (
"Reassembly queue for %p has stale entry at head", tp));
1067#ifdef TCP_REASS_LOGGING
1068 tcp_reass_log_dump(tp);
1072 SOCKBUF_LOCK(&so->so_rcv);
1076 nq = TAILQ_NEXT(q, tqe_q);
1077 TAILQ_REMOVE(&tp->
t_segq, q, tqe_q);
1078 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
1081#ifdef TCP_REASS_LOGGING
1084 tcp_log_reassm(tp, q, NULL, th->th_seq, *tlenp,
TCP_R_LOG_READ, 1);
1089 sbappendstream_locked(&so->so_rcv, q->
tqe_m, 0);
1091#ifdef TCP_REASS_LOGGING
1093 tcp_log_reassm(tp, q, NULL, th->th_seq, *tlenp,
TCP_R_LOG_READ, 2);
1099 (
"tp:%p seg queue goes negative", tp));
1105 if (TAILQ_EMPTY(&tp->
t_segq) &&
1108 panic(
"tp:%p segq:%p len:%d queue empty",
1111#ifdef TCP_REASS_LOGGING
1113 tcp_log_reassm(tp, NULL, NULL, th->th_seq, *tlenp,
TCP_R_LOG_ZERO, 0);
1121#ifdef TCP_REASS_LOGGING
1122 tcp_reass_log_dump(tp);
#define INP_WLOCK_ASSERT(inp)
struct socket * inp_socket
struct in_conninfo inp_inc
struct tsegqe_head t_segq
#define TCPS_HAVEESTABLISHED(s)
SYSCTL_COUNTER_U64(_net_inet_tcp_hpts_stats, OID_AUTO, hopeless, CTLFLAG_RD, &hpts_hopelessly_behind, "Number of times hpts could not catch up and was behind hopelessly")
static __inline uint32_t tcp_get_usecs(struct timeval *tv)
#define TCP_LOG_EVENTP(tp, th, rxbuf, txbuf, eventid, errornum, len, stackinfo, th_hostorder, tv)
static uma_zone_t tcp_reass_zone
#define TCP_R_LOG_LIMIT_REACHED
static void tcp_reass_zone_change(void *tag)
#define TCP_R_LOG_PREPEND
static int tcp_reass_maxseg
static void tcp_reass_merge_forward(struct tcpcb *tp, struct tseg_qent *ent)
static void tcp_reass_prepend(struct tcpcb *tp, struct tseg_qent *first, struct mbuf *m, struct tcphdr *th, int tlen, struct mbuf *mlast, int lenofoh)
void tcp_reass_global_init(void)
int tcp_reass(struct tcpcb *tp, struct tcphdr *th, tcp_seq *seq_start, int *tlenp, struct mbuf *m)
static SYSCTL_NODE(_net_inet_tcp, OID_AUTO, reass, CTLFLAG_RW|CTLFLAG_MPSAFE, 0, "TCP Segment Reassembly Queue")
#define TCP_R_LOG_NEW_ENTRY
static void tcp_reass_replace(struct tcpcb *tp, struct tseg_qent *q, struct mbuf *m, tcp_seq seq, int len, struct mbuf *mlast, int mbufoh, uint16_t flags)
SYSCTL_UINT(_net_inet_tcp_reass, OID_AUTO, maxqueuelen, CTLFLAG_RWTUN, &tcp_reass_maxqueuelen, 0, "Maximum number of TCP Segments per Reassembly Queue")
void tcp_reass_flush(struct tcpcb *tp)
SYSCTL_INT(_net_inet_tcp_reass, OID_AUTO, maxsegments, CTLFLAG_RDTUN, &tcp_reass_maxseg, 0, "Global maximum number of TCP Segments in Reassembly Queue")
static u_int tcp_reass_queue_guard
static void tcp_reass_append(struct tcpcb *tp, struct tseg_qent *last, struct mbuf *m, struct tcphdr *th, int tlen, struct mbuf *mlast, int lenofoh)
static int tcp_new_limits
#define TCP_R_LOG_MERGE_INTO
static void tcp_reass_merge_into(struct tcpcb *tp, struct tseg_qent *ent, struct tseg_qent *q)
static int tcp_reass_overhead_of_chain(struct mbuf *m, struct mbuf **mlast)
SYSCTL_UMA_CUR(_net_inet_tcp_reass, OID_AUTO, cursegments, 0, &tcp_reass_zone, "Global number of TCP Segments currently in Reassembly Queue")
static u_int tcp_reass_maxqueuelen
#define TCP_R_LOG_REPLACE
char * tcp_log_addrs(struct in_conninfo *inc, struct tcphdr *th, void *ip4hdr, const void *ip6hdr)
static uint16_t tcp_get_flags(const struct tcphdr *th)
#define TCPSTAT_ADD(name, val)
#define TCPSTAT_INC(name)