80#include "opt_tcpdebug.h"
84#include <sys/kernel.h>
85#include <sys/sysctl.h>
86#include <sys/malloc.h>
89#include <sys/protosw.h>
90#include <sys/socket.h>
91#include <sys/socketvar.h>
92#include <sys/syslog.h>
95#include <machine/cpu.h>
100#include <net/if_var.h>
101#include <net/route.h>
112#include <netinet6/nd6.h>
113#include <netinet6/ip6_var.h>
114#include <netinet6/in6_pcb.h>
120#include <netinet6/tcp6_var.h>
127#include <machine/in_cksum.h>
130#define V_sack_hole_zone VNET(sack_hole_zone)
132SYSCTL_NODE(_net_inet_tcp, OID_AUTO, sack, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
136SYSCTL_INT(_net_inet_tcp_sack, OID_AUTO, enable, CTLFLAG_VNET | CTLFLAG_RW,
137 &VNET_NAME(tcp_do_sack), 0,
138 "Enable/Disable TCP SACK support");
141SYSCTL_INT(_net_inet_tcp_sack, OID_AUTO, revised, CTLFLAG_VNET | CTLFLAG_RW,
142 &VNET_NAME(tcp_do_newsack), 0,
143 "Use revised SACK loss recovery per RFC 6675");
146SYSCTL_INT(_net_inet_tcp_sack, OID_AUTO, maxholes, CTLFLAG_VNET | CTLFLAG_RW,
147 &VNET_NAME(tcp_sack_maxholes), 0,
148 "Maximum number of TCP SACK holes allowed per connection");
151SYSCTL_INT(_net_inet_tcp_sack, OID_AUTO, globalmaxholes, CTLFLAG_VNET | CTLFLAG_RW,
152 &VNET_NAME(tcp_sack_globalmaxholes), 0,
153 "Global maximum number of TCP SACK holes");
156SYSCTL_INT(_net_inet_tcp_sack, OID_AUTO, globalholes, CTLFLAG_VNET | CTLFLAG_RD,
157 &VNET_NAME(tcp_sack_globalholes), 0,
158 "Global number of TCP SACK holes currently allocated");
178 struct sackblk head_blk,mid_blk,saved_blks[MAX_SACK_BLKS];
179 int i, j, n, identical;
184 KASSERT(
SEQ_LT(rcv_start, rcv_end), (
"rcv_start < rcv_end"));
190 saved_blks[0].
start = rcv_start;
191 saved_blks[0].
end = rcv_end;
193 saved_blks[0].
start = saved_blks[0].
end = 0;
197 mid_blk.
start = rcv_start;
198 mid_blk.
end = rcv_end;
215 (head_blk.
start == head_blk.
end))) {
221 if (((head_blk.
start == head_blk.
end) ||
246 (mid_blk.
end == saved_blks[0].
end))) ||
249 saved_blks[n++].
end = mid_blk.
end;
251 for (j = 0; (j < tp->
rcv_numsacks) && (n < MAX_SACK_BLKS); j++) {
258 for (i = 0; i < n; i++) {
280 struct sackblk head_blk, saved_blks[MAX_SACK_BLKS];
281 int num_head, num_saved, i;
286 KASSERT(
SEQ_LEQ(rcv_start, rcv_end), (
"rcv_start <= rcv_end"));
288 if ((rcv_start == rcv_end) &&
295 head_blk.
start = rcv_start;
296 head_blk.
end = rcv_end;
337 tcp_seq temp =
start;
339 head_blk.
start = temp;
348 if ((num_saved >= 1) &&
353 saved_blks[num_saved].
end =
end;
361 if ((num_saved >= 1) &&
369 saved_blks[num_saved].
end =
end;
378 if (
SEQ_LT(rcv_start, rcv_end)) {
389 if (num_saved >= MAX_SACK_BLKS)
392 if ((rcv_start == rcv_end) &&
400 bcopy(saved_blks, &tp->
sackblks[num_head],
401 sizeof(
struct sackblk) * num_saved);
411 struct sackblk saved_blks[MAX_SACK_BLKS];
434 saved_blks[num_saved].
end =
end;
442 sizeof(
struct sackblk) * num_saved);
457 for (i = 0; i < MAX_SACK_BLKS; i++)
501 KASSERT(tp->
snd_numholes >= 0, (
"tp->snd_numholes >= 0"));
521 TAILQ_INSERT_AFTER(&tp->snd_holes, after, hole, scblink);
523 TAILQ_INSERT_TAIL(&tp->snd_holes, hole, scblink);
544 TAILQ_REMOVE(&tp->snd_holes, hole, scblink);
561 struct sackblk sack, sack_blocks[TCP_MAX_SACK + 1], *sblkp;
562 int i, j, num_sack_blks, sack_changed;
563 int delivered_data, left_edge_delta;
576 if (
SEQ_LT(tp->
snd_una, th_ack) && !TAILQ_EMPTY(&tp->snd_holes)) {
577 left_edge_delta = th_ack - tp->
snd_una;
579 sack_blocks[num_sack_blks++].
end = th_ack;
585 delivered_data += th_ack - tp->
snd_una;
596 bcopy((to->
to_sacks + i * TCPOLEN_SACK),
597 &sack,
sizeof(sack));
599 sack.
end = ntohl(sack.
end);
606 sack_blocks[num_sack_blks++] = sack;
620 if (num_sack_blks == 0)
621 return (sack_changed);
628 for (i = 0; i < num_sack_blks; i++) {
629 for (j = i + 1; j < num_sack_blks; j++) {
631 sack = sack_blocks[i];
632 sack_blocks[i] = sack_blocks[j];
633 sack_blocks[j] = sack;
637 if (TAILQ_EMPTY(&tp->snd_holes)) {
660 sblkp = &sack_blocks[num_sack_blks - 1];
669 if (((temp = TAILQ_LAST(&tp->snd_holes, sackhole_head)) != NULL) &&
674 delivered_data += sblkp->
end - sblkp->
start;
687 delivered_data += sblkp->
end - sblkp->
start;
701 while (sblkp >= sack_blocks &&
704 if (sblkp >= sack_blocks &&
718 cur = TAILQ_LAST(&tp->snd_holes, sackhole_head);
723 while (sblkp >= sack_blocks && cur != NULL) {
737 cur = TAILQ_PREV(cur, sackhole_head, scblink);
743 (
"sackhint bytes rtx >= 0"));
749 delivered_data += (cur->
end - cur->
start);
751 cur = TAILQ_PREV(cur, sackhole_head, scblink);
761 delivered_data += (sblkp->
end - cur->
start);
769 delivered_data += (cur->
end - sblkp->
start);
793 delivered_data += (sblkp->
end - sblkp->
start);
805 cur = TAILQ_PREV(cur, sackhole_head, scblink);
821 KASSERT((delivered_data >= 0), (
"delivered_data < 0"));
823 return (sack_changed);
835 while ((q = TAILQ_FIRST(&tp->snd_holes)) != NULL)
839 KASSERT(tp->
snd_numholes == 0, (
"tp->snd_numholes == 0"));
841 (
"tp->sackhint.nexthole == NULL"));
888 TAILQ_EMPTY(&tp->snd_holes) &&
896 tcp_seq highdata = tp->
snd_max;
899 if (th->th_ack != highdata) {
902 highdata - maxseg), highdata, NULL);
905 (void) tcp_output(tp);
914tcp_sack_output_debug(
struct tcpcb *tp,
int *sack_bytes_rexmt)
919 *sack_bytes_rexmt = 0;
920 TAILQ_FOREACH(p, &tp->snd_holes, scblink) {
961 while ((hole = TAILQ_NEXT(hole, scblink)) != NULL) {
979 struct sackhole *p, *cur = TAILQ_FIRST(&tp->snd_holes);
991 while ((p = TAILQ_NEXT(cur, scblink)) != NULL) {
1019 ((temp = TAILQ_FIRST(&tp->snd_holes)) != NULL) &&
1029 TAILQ_FOREACH(temp, &tp->snd_holes, scblink) {
#define INP_WLOCK_ASSERT(inp)
int32_t sack_bytes_rexmit
struct sackhole * nexthole
struct sackblk sackblks[MAX_SACK_BLKS]
void tcp_update_dsack_list(struct tcpcb *tp, tcp_seq rcv_start, tcp_seq rcv_end)
SYSCTL_NODE(_net_inet_tcp, OID_AUTO, sack, CTLFLAG_RW|CTLFLAG_MPSAFE, 0, "TCP SACK")
void tcp_clean_sackreport(struct tcpcb *tp)
void tcp_sack_partialack(struct tcpcb *tp, struct tcphdr *th)
VNET_DEFINE(int, tcp_do_sack)
static void tcp_sackhole_free(struct tcpcb *tp, struct sackhole *hole)
static void tcp_sackhole_remove(struct tcpcb *tp, struct sackhole *hole)
void tcp_free_sackholes(struct tcpcb *tp)
int tcp_sack_doack(struct tcpcb *tp, struct tcpopt *to, tcp_seq th_ack)
VNET_DECLARE(struct uma_zone *, sack_hole_zone)
struct sackhole * tcp_sack_output(struct tcpcb *tp, int *sack_bytes_rexmt)
static struct sackhole * tcp_sackhole_insert(struct tcpcb *tp, tcp_seq start, tcp_seq end, struct sackhole *after)
SYSCTL_INT(_net_inet_tcp_sack, OID_AUTO, enable, CTLFLAG_VNET|CTLFLAG_RW, &VNET_NAME(tcp_do_sack), 0, "Enable/Disable TCP SACK support")
static struct sackhole * tcp_sackhole_alloc(struct tcpcb *tp, tcp_seq start, tcp_seq end)
void tcp_update_sack_list(struct tcpcb *tp, tcp_seq rcv_start, tcp_seq rcv_end)
void tcp_sack_lost_retransmission(struct tcpcb *tp, struct tcphdr *th)
int tcp_dsack_block_exists(struct tcpcb *tp)
void tcp_sack_adjust(struct tcpcb *tp)
void tcp_clean_dsack_blocks(struct tcpcb *tp)
u_int tcp_maxseg(const struct tcpcb *tp)
void tcp_record_dsack(struct tcpcb *tp, tcp_seq start, tcp_seq end, int tlp)
void tcp_timer_activate(struct tcpcb *tp, uint32_t timer_type, u_int delta)
#define V_tcp_sack_globalmaxholes
#define IN_RECOVERY(t_flags)
#define BYTES_THIS_ACK(tp, th)
#define TCPSTAT_INC(name)
#define V_tcp_sack_maxholes
#define V_tcp_sack_globalholes
#define EXIT_RECOVERY(t_flags)