37#include "opt_kern_tls.h"
42#include <sys/kernel.h>
45#include <sys/malloc.h>
49#include <sys/protosw.h>
50#include <sys/resourcevar.h>
51#include <sys/signalvar.h>
52#include <sys/socket.h>
53#include <sys/socketvar.h>
55#include <sys/sysctl.h>
61void (*
aio_swake)(
struct socket *,
struct sockbuf *);
69 (quad_t)SB_MAX * MCLBYTES / (MSIZE + MCLBYTES);
74static void sbcompress_ktls_rx(
struct sockbuf *sb,
struct mbuf *m,
89 if (
flags & PRUS_NOTREADY)
112 SOCKBUF_LOCK_ASSERT(sb);
114 if ((sb->sb_flags & SB_NOCOALESCE) != 0)
117 for (m = m0; m != end; m = m->m_next) {
118 MPASS((m->m_flags & M_NOTREADY) == 0);
130 if ((n != NULL) && (n != end) && (m->m_flags & M_EOR) == 0 &&
131 (m->m_flags & M_EXTPG) &&
132 (n->m_flags & M_EXTPG) &&
133 !mbuf_has_tls_session(m) &&
134 !mbuf_has_tls_session(n)) {
135 int hdr_len, trail_len;
137 hdr_len = n->m_epg_hdrlen;
138 trail_len = m->m_epg_trllen;
139 if (trail_len != 0 && hdr_len != 0 &&
140 trail_len + hdr_len <= MBUF_PEXT_TRAIL_LEN) {
142 memcpy(&m->m_epg_trail[trail_len],
143 n->m_epg_hdr, hdr_len);
144 m->m_epg_trllen += hdr_len;
153 if ((m->m_flags & M_EXTPG) && m->m_len <= MLEN &&
154 !mbuf_has_tls_session(m)) {
155 ext_size = m->m_ext.ext_size;
157 sb->sb_mbcnt -= ext_size;
162 while ((n != NULL) && (n != end) && (m->m_flags & M_EOR) == 0 &&
164 (m->m_flags & M_EXTPG) == 0 &&
165 !mbuf_has_tls_session(n) &&
166 !mbuf_has_tls_session(m) &&
167 n->m_len <= MCLBYTES / 4 &&
168 n->m_len <= M_TRAILINGSPACE(m) &&
169 m->m_type == n->m_type) {
170 KASSERT(sb->sb_lastrecord != n,
171 (
"%s: merging start of record (%p) into previous mbuf (%p)",
173 m_copydata(n, 0, n->m_len, mtodo(m, m->m_len));
174 m->m_len += n->m_len;
175 m->m_next = n->m_next;
176 m->m_flags |= n->m_flags & M_EOR;
177 if (sb->sb_mbtail == n)
180 sb->sb_mbcnt -= MSIZE;
182 if (n->m_flags & M_EXT) {
183 sb->sb_mbcnt -= n->m_ext.ext_size;
205 SOCKBUF_LOCK_ASSERT(sb);
206 KASSERT(sb->sb_fnrdy != NULL, (
"%s: sb %p NULL fnrdy", __func__, sb));
207 KASSERT(
count > 0, (
"%s: invalid count %d", __func__,
count));
210 blocker = (sb->sb_fnrdy == m) ? M_BLOCKED : 0;
213 KASSERT(m->m_flags & M_NOTREADY,
214 (
"%s: m %p !M_NOTREADY", __func__, m));
215 if ((m->m_flags & M_EXTPG) != 0 && m->m_epg_npgs != 0) {
216 if (count < m->m_epg_nrdy) {
217 m->m_epg_nrdy -=
count;
221 count -= m->m_epg_nrdy;
226 m->m_flags &= ~(M_NOTREADY | blocker);
228 sb->sb_acc += m->m_len;
238 MPASS(m->m_flags & M_NOTREADY);
239 return (EINPROGRESS);
244 return (EINPROGRESS);
248 for (; m && (m->m_flags & M_NOTREADY) == 0; m = m->m_next) {
249 KASSERT(m->m_flags & M_BLOCKED,
250 (
"%s: m %p !M_BLOCKED", __func__, m));
251 m->m_flags &= ~M_BLOCKED;
252 sb->sb_acc += m->m_len;
268 SOCKBUF_LOCK_ASSERT(sb);
270 sb->sb_ccc += m->m_len;
272 if (sb->sb_fnrdy == NULL) {
273 if (m->m_flags & M_NOTREADY)
276 sb->sb_acc += m->m_len;
278 m->m_flags |= M_BLOCKED;
280 if (m->m_type != MT_DATA && m->m_type != MT_OOBDATA)
281 sb->sb_ctl += m->m_len;
283 sb->sb_mbcnt += MSIZE;
286 if (m->m_flags & M_EXT) {
287 sb->sb_mbcnt += m->m_ext.ext_size;
296sbfree(
struct sockbuf *sb,
struct mbuf *m)
300 SOCKBUF_LOCK_ASSERT(sb);
303 sb->sb_ccc -= m->m_len;
305 if (!(m->m_flags & M_NOTAVAIL))
306 sb->sb_acc -= m->m_len;
308 if (m == sb->sb_fnrdy) {
311 KASSERT(m->m_flags & M_NOTREADY,
312 (
"%s: m %p !M_NOTREADY", __func__, m));
315 while (n != NULL && !(n->m_flags & M_NOTREADY)) {
316 n->m_flags &= ~M_BLOCKED;
317 sb->sb_acc += n->m_len;
323 if (m->m_type != MT_DATA && m->m_type != MT_OOBDATA)
324 sb->sb_ctl -= m->m_len;
326 sb->sb_mbcnt -= MSIZE;
328 if (m->m_flags & M_EXT) {
329 sb->sb_mbcnt -= m->m_ext.ext_size;
333 if (sb->sb_sndptr == m) {
334 sb->sb_sndptr = NULL;
335 sb->sb_sndptroff = 0;
337 if (sb->sb_sndptroff != 0)
338 sb->sb_sndptroff -= m->m_len;
348sballoc_ktls_rx(
struct sockbuf *sb,
struct mbuf *m)
351 SOCKBUF_LOCK_ASSERT(sb);
353 sb->sb_ccc += m->m_len;
354 sb->sb_tlscc += m->m_len;
356 sb->sb_mbcnt += MSIZE;
359 if (m->m_flags & M_EXT) {
360 sb->sb_mbcnt += m->m_ext.ext_size;
366sbfree_ktls_rx(
struct sockbuf *sb,
struct mbuf *m)
370 SOCKBUF_LOCK_ASSERT(sb);
373 sb->sb_ccc -= m->m_len;
374 sb->sb_tlscc -= m->m_len;
376 sb->sb_mbcnt -= MSIZE;
379 if (m->m_flags & M_EXT) {
380 sb->sb_mbcnt -= m->m_ext.ext_size;
399 SOCKBUF_LOCK_ASSERT(&so->so_snd);
401 so->so_snd.sb_state |= SBS_CANTSENDMORE;
402 sowwakeup_locked(so);
403 mtx_assert(SOCKBUF_MTX(&so->so_snd), MA_NOTOWNED);
410 SOCKBUF_LOCK(&so->so_snd);
412 mtx_assert(SOCKBUF_MTX(&so->so_snd), MA_NOTOWNED);
419 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
421 so->so_rcv.sb_state |= SBS_CANTRCVMORE;
423 if (so->so_rcv.sb_flags & SB_TLS_RX)
426 sorwakeup_locked(so);
427 mtx_assert(SOCKBUF_MTX(&so->so_rcv), MA_NOTOWNED);
434 SOCKBUF_LOCK(&so->so_rcv);
436 mtx_assert(SOCKBUF_MTX(&so->so_rcv), MA_NOTOWNED);
443 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
445 if (so->so_options & SO_RERROR) {
446 so->so_rerror = ENOBUFS;
447 sorwakeup_locked(so);
449 SOCKBUF_UNLOCK(&so->so_rcv);
451 mtx_assert(SOCKBUF_MTX(&so->so_rcv), MA_NOTOWNED);
458 SOCKBUF_LOCK(&so->so_rcv);
460 mtx_assert(SOCKBUF_MTX(&so->so_rcv), MA_NOTOWNED);
470 SOCKBUF_LOCK_ASSERT(sb);
472 sb->sb_flags |= SB_WAIT;
473 return (msleep_sbt(&sb->sb_acc, SOCKBUF_MTX(sb),
474 (sb->sb_flags & SB_NOINTR) ? PSOCK : PSOCK | PCATCH,
"sbwait",
475 sb->sb_timeo, 0, 0));
495 SOCKBUF_LOCK_ASSERT(sb);
498 if (!SEL_WAITING(sb->sb_sel))
499 sb->sb_flags &= ~SB_SEL;
500 if (sb->sb_flags & SB_WAIT) {
501 sb->sb_flags &= ~SB_WAIT;
504 KNOTE_LOCKED(&sb->sb_sel->si_note, 0);
505 if (sb->sb_upcall != NULL) {
506 ret = sb->sb_upcall(so, sb->sb_upcallarg, M_NOWAIT);
507 if (ret == SU_ISCONNECTED) {
508 KASSERT(sb == &so->so_rcv,
509 (
"SO_SND upcall returned SU_ISCONNECTED"));
514 if (sb->sb_flags & SB_AIO)
517 if (ret == SU_ISCONNECTED)
519 if ((so->so_state & SS_ASYNC) && so->so_sigio != NULL)
520 pgsigio(&so->so_sigio, SIGIO, 0);
521 mtx_assert(SOCKBUF_MTX(sb), MA_NOTOWNED);
558 struct thread *td = curthread;
560 SOCKBUF_LOCK(&so->so_snd);
561 SOCKBUF_LOCK(&so->so_rcv);
566 if (so->so_rcv.sb_lowat == 0)
567 so->so_rcv.sb_lowat = 1;
568 if (so->so_snd.sb_lowat == 0)
569 so->so_snd.sb_lowat = MCLBYTES;
570 if (so->so_snd.sb_lowat > so->so_snd.sb_hiwat)
571 so->so_snd.sb_lowat = so->so_snd.sb_hiwat;
572 SOCKBUF_UNLOCK(&so->so_rcv);
573 SOCKBUF_UNLOCK(&so->so_snd);
578 SOCKBUF_UNLOCK(&so->so_rcv);
579 SOCKBUF_UNLOCK(&so->so_snd);
587 u_long tmp_sb_max =
sb_max;
590 if (error || !req->newptr)
592 if (tmp_sb_max < MSIZE + MCLBYTES)
609 SOCKBUF_LOCK_ASSERT(sb);
621 sbsize_limit =
lim_cur(td, RLIMIT_SBSIZE);
623 sbsize_limit = RLIM_INFINITY;
624 if (!
chgsbsize(so->so_cred->cr_uidinfo, &sb->sb_hiwat, cc,
628 if (sb->sb_lowat > sb->sb_hiwat)
629 sb->sb_lowat = sb->sb_hiwat;
638 u_int *hiwat, *lowat;
643 if (SOLISTENING(so)) {
647 lowat = &so->sol_sbsnd_lowat;
648 hiwat = &so->sol_sbsnd_hiwat;
649 flags = &so->sol_sbsnd_flags;
653 lowat = &so->sol_sbrcv_lowat;
654 hiwat = &so->sol_sbrcv_hiwat;
655 flags = &so->sol_sbrcv_flags;
669 flags = &sb->sb_flags;
670 hiwat = &sb->sb_hiwat;
671 lowat = &sb->sb_lowat;
679 if (SOLISTENING(so)) {
692 *
flags &= ~SB_AUTOSIZE;
700 *lowat = (cc > *hiwat) ? *hiwat : cc;
704 if (!SOLISTENING(so))
718 (void)
chgsbsize(so->so_cred->cr_uidinfo, &sb->sb_hiwat, 0,
727 SOCKBUF_LOCK_ASSERT(sb);
747 if (sb->sb_tls_info != NULL)
748 ktls_free(sb->sb_tls_info);
749 sb->sb_tls_info = NULL;
777sblastrecordchk(
struct sockbuf *sb,
const char *file,
int line)
779 struct mbuf *m = sb->sb_mb;
781 SOCKBUF_LOCK_ASSERT(sb);
783 while (m && m->m_nextpkt)
786 if (m != sb->sb_lastrecord) {
787 printf(
"%s: sb_mb %p sb_lastrecord %p last %p\n",
788 __func__, sb->sb_mb, sb->sb_lastrecord, m);
789 printf(
"packet chain:\n");
790 for (m = sb->sb_mb; m != NULL; m = m->m_nextpkt)
792 panic(
"%s from %s:%u", __func__, file, line);
797sblastmbufchk(
struct sockbuf *sb,
const char *file,
int line)
799 struct mbuf *m = sb->sb_mb;
802 SOCKBUF_LOCK_ASSERT(sb);
804 while (m && m->m_nextpkt)
807 while (m && m->m_next)
810 if (m != sb->sb_mbtail) {
811 printf(
"%s: sb_mb %p sb_mbtail %p last %p\n",
812 __func__, sb->sb_mb, sb->sb_mbtail, m);
814 for (m = sb->sb_mb; m != NULL; m = m->m_nextpkt) {
816 for (n = m; n != NULL; n = n->m_next)
820 panic(
"%s from %s:%u", __func__, file, line);
825 while (m && m->m_next)
828 if (m != sb->sb_mtlstail) {
829 printf(
"%s: sb_mtls %p sb_mtlstail %p last %p\n",
830 __func__, sb->sb_mtls, sb->sb_mtlstail, m);
831 printf(
"TLS packet tree:\n");
833 for (m = sb->sb_mtls; m != NULL; m = m->m_next) {
837 panic(
"%s from %s:%u", __func__, file, line);
843#define SBLINKRECORD(sb, m0) do { \
844 SOCKBUF_LOCK_ASSERT(sb); \
845 if ((sb)->sb_lastrecord != NULL) \
846 (sb)->sb_lastrecord->m_nextpkt = (m0); \
848 (sb)->sb_mb = (m0); \
849 (sb)->sb_lastrecord = (m0); \
862 SOCKBUF_LOCK_ASSERT(sb);
873 if (n->m_flags & M_EOR) {
877 }
while (n->m_next && (n = n->m_next));
884 if ((n = sb->sb_lastrecord) != NULL) {
886 if (n->m_flags & M_EOR) {
890 }
while (n->m_next && (n = n->m_next));
896 sb->sb_lastrecord = m;
924sbappend_ktls_rx(
struct sockbuf *sb,
struct mbuf *m)
933 for (n = m; n != NULL; n = n->m_next)
934 n->m_flags |= M_NOTREADY;
935 sbcompress_ktls_rx(sb, m, sb->sb_mtlstail);
948 SOCKBUF_LOCK_ASSERT(sb);
950 KASSERT(m->m_nextpkt == NULL,(
"sbappendstream 0"));
958 if (sb->sb_flags & SB_TLS_RX) {
959 sbappend_ktls_rx(sb, m);
964 KASSERT(sb->sb_mb == sb->sb_lastrecord,(
"sbappendstream 1"));
969 if (sb->sb_tls_info != NULL)
978 sb->sb_lastrecord = sb->sb_mb;
998sbcheck(
struct sockbuf *sb,
const char *file,
int line)
1000 struct mbuf *m, *n, *fnrdy;
1001 u_long acc, ccc, mbcnt;
1006 SOCKBUF_LOCK_ASSERT(sb);
1008 acc = ccc = mbcnt = 0;
1011 for (m = sb->sb_mb; m; m = n) {
1013 for (; m; m = m->m_next) {
1014 if (m->m_len == 0) {
1015 printf(
"sb %p empty mbuf %p\n", sb, m);
1018 if ((m->m_flags & M_NOTREADY) && fnrdy == NULL) {
1019 if (m != sb->sb_fnrdy) {
1020 printf(
"sb %p: fnrdy %p != m %p\n",
1021 sb, sb->sb_fnrdy, m);
1027 if (!(m->m_flags & M_NOTAVAIL)) {
1028 printf(
"sb %p: fnrdy %p, m %p is avail\n",
1029 sb, sb->sb_fnrdy, m);
1036 if (m->m_flags & M_EXT)
1037 mbcnt += m->m_ext.ext_size;
1048 ccc += sb->sb_tlsdcc;
1051 for (m = sb->sb_mtls; m; m = m->m_next) {
1052 if (m->m_nextpkt != NULL) {
1053 printf(
"sb %p TLS mbuf %p with nextpkt\n", sb, m);
1056 if ((m->m_flags & M_NOTREADY) == 0) {
1057 printf(
"sb %p TLS mbuf %p ready\n", sb, m);
1063 if (m->m_flags & M_EXT)
1064 mbcnt += m->m_ext.ext_size;
1067 if (sb->sb_tlscc != tlscc) {
1068 printf(
"tlscc %ld/%u dcc %u\n", tlscc, sb->sb_tlscc,
1073 if (acc != sb->sb_acc || ccc != sb->sb_ccc || mbcnt != sb->sb_mbcnt) {
1074 printf(
"acc %ld/%u ccc %ld/%u mbcnt %ld/%u\n",
1075 acc, sb->sb_acc, ccc, sb->sb_ccc, mbcnt, sb->sb_mbcnt);
1077 printf(
"tlscc %ld/%u dcc %u\n", tlscc, sb->sb_tlscc,
1084 panic(
"%s from %s:%u", __func__, file, line);
1096 SOCKBUF_LOCK_ASSERT(sb);
1100 m_clrprotoflags(m0);
1106 SBLASTRECORDCHK(sb);
1111 if (m && (m0->m_flags & M_EOR)) {
1112 m0->m_flags &= ~M_EOR;
1113 m->m_flags |= M_EOR;
1134 struct mbuf *m0,
struct mbuf *control,
struct mbuf *ctrl_last)
1136 struct mbuf *m, *n, *nlast;
1138 if (asa->sa_len > MLEN)
1141 m = m_get(M_NOWAIT, MT_SONAME);
1144 m->m_len = asa->sa_len;
1145 bcopy(asa, mtod(m, caddr_t), asa->sa_len);
1147 M_ASSERT_NO_SND_TAG(m0);
1148 m_clrprotoflags(m0);
1155 m0->m_pkthdr.rcvif = NULL;
1156 m0->m_pkthdr.flowid = 0;
1157 m0->m_pkthdr.csum_flags = 0;
1158 m0->m_pkthdr.fibnum = 0;
1159 m0->m_pkthdr.rsstype = 0;
1162 ctrl_last->m_next = m0;
1165 m->m_next = control;
1166 for (n = m; n->m_next != NULL; n = n->m_next)
1172 sb->sb_mbtail = nlast;
1175 SBLASTRECORDCHK(sb);
1187 struct mbuf *m0,
struct mbuf *control)
1189 struct mbuf *ctrl_last;
1190 int space = asa->sa_len;
1192 SOCKBUF_LOCK_ASSERT(sb);
1194 if (m0 && (m0->m_flags & M_PKTHDR) == 0)
1195 panic(
"sbappendaddr_locked");
1197 space += m0->m_pkthdr.len;
1198 space +=
m_length(control, &ctrl_last);
1200 if (space > sbspace(sb))
1213 struct mbuf *m0,
struct mbuf *control)
1215 struct mbuf *ctrl_last;
1217 SOCKBUF_LOCK_ASSERT(sb);
1219 ctrl_last = (control == NULL) ? NULL : m_last(control);
1231 struct mbuf *m0,
struct mbuf *control)
1243 struct mbuf *control,
int flags)
1245 struct mbuf *m, *mlast;
1248 m_last(control)->m_next = m0;
1250 SBLASTRECORDCHK(sb);
1252 for (m = control; m->m_next; m = m->m_next)
1258 sb->sb_mbtail = mlast;
1261 SBLASTRECORDCHK(sb);
1300 SOCKBUF_LOCK_ASSERT(sb);
1303 eor |= m->m_flags & M_EOR;
1304 if (m->m_len == 0 &&
1306 (((o = m->m_next) || (o = n)) &&
1307 o->m_type == m->m_type))) {
1308 if (sb->sb_lastrecord == m)
1309 sb->sb_lastrecord = m->m_next;
1313 if (n && (n->m_flags & M_EOR) == 0 &&
1315 ((sb->sb_flags & SB_NOCOALESCE) == 0) &&
1316 !(m->m_flags & M_NOTREADY) &&
1317 !(n->m_flags & (M_NOTREADY | M_EXTPG)) &&
1318 !mbuf_has_tls_session(m) &&
1319 !mbuf_has_tls_session(n) &&
1320 m->m_len <= MCLBYTES / 4 &&
1321 m->m_len <= M_TRAILINGSPACE(n) &&
1322 n->m_type == m->m_type) {
1323 m_copydata(m, 0, m->m_len, mtodo(n, n->m_len));
1324 n->m_len += m->m_len;
1325 sb->sb_ccc += m->m_len;
1326 if (sb->sb_fnrdy == NULL)
1327 sb->sb_acc += m->m_len;
1328 if (m->m_type != MT_DATA && m->m_type != MT_OOBDATA)
1330 sb->sb_ctl += m->m_len;
1334 if (m->m_len <= MLEN && (m->m_flags & M_EXTPG) &&
1335 (m->m_flags & M_NOTREADY) == 0 &&
1336 !mbuf_has_tls_session(m))
1345 m->m_flags &= ~M_EOR;
1350 KASSERT(n != NULL, (
"sbcompress: eor && n == NULL"));
1363sbcompress_ktls_rx(
struct sockbuf *sb,
struct mbuf *m,
struct mbuf *n)
1366 SOCKBUF_LOCK_ASSERT(sb);
1369 KASSERT((m->m_flags & M_EOR) == 0,
1370 (
"TLS RX mbuf %p with EOR", m));
1371 KASSERT(m->m_type == MT_DATA,
1372 (
"TLS RX mbuf %p is not MT_DATA", m));
1373 KASSERT((m->m_flags & M_NOTREADY) != 0,
1374 (
"TLS RX mbuf %p ready", m));
1375 KASSERT((m->m_flags & M_EXTPG) == 0,
1376 (
"TLS RX mbuf %p unmapped", m));
1378 if (m->m_len == 0) {
1389 ((sb->sb_flags & SB_NOCOALESCE) == 0) &&
1390 !(n->m_flags & (M_EXTPG)) &&
1391 m->m_len <= MCLBYTES / 4 &&
1392 m->m_len <= M_TRAILINGSPACE(n)) {
1393 m_copydata(m, 0, m->m_len, mtodo(n, n->m_len));
1394 n->m_len += m->m_len;
1395 sb->sb_ccc += m->m_len;
1396 sb->sb_tlscc += m->m_len;
1404 sb->sb_mtlstail = m;
1405 sballoc_ktls_rx(sb, m);
1421 while (sb->sb_mbcnt || sb->sb_tlsdcc) {
1426 if (sb->sb_ccc == 0 && (sb->sb_mb == NULL || sb->sb_mb->m_len))
1430 KASSERT(sb->sb_ccc == 0 && sb->sb_mb == 0 && sb->sb_mbcnt == 0,
1431 (
"%s: ccc %u mb %p mbcnt %u", __func__,
1432 sb->sb_ccc, (
void *)sb->sb_mb, sb->sb_mbcnt));
1439 SOCKBUF_LOCK_ASSERT(sb);
1458 struct mbuf *m, *next, *mfree;
1461 KASSERT(len >= 0, (
"%s: len is %d but it is supposed to be >= 0",
1463 KASSERT(len <= sb->sb_ccc, (
"%s: len: %d is > ccc: %u",
1464 __func__, len, sb->sb_ccc));
1466 next = (m = sb->sb_mb) ? m->m_nextpkt : 0;
1473 if (next == NULL && !is_tls) {
1474 if (sb->sb_tlsdcc != 0) {
1475 MPASS(len >= sb->sb_tlsdcc);
1476 len -= sb->sb_tlsdcc;
1477 sb->sb_ccc -= sb->sb_tlsdcc;
1486 KASSERT(next, (
"%s: no next, len %d", __func__, len));
1488 next = m->m_nextpkt;
1490 if (m->m_len > len) {
1491 KASSERT(!(m->m_flags & M_NOTAVAIL),
1492 (
"%s: m %p M_NOTAVAIL", __func__, m));
1497 if (sb->sb_sndptroff != 0)
1498 sb->sb_sndptroff -= len;
1499 if (m->m_type != MT_DATA && m->m_type != MT_OOBDATA)
1506 sbfree_ktls_rx(sb, m);
1514 if (m->m_flags & M_NOTREADY && !is_tls)
1531 while (m && m->m_len == 0) {
1545 sb->sb_mtlstail = NULL;
1550 m->m_nextpkt = next;
1559 sb->sb_mbtail = NULL;
1560 sb->sb_lastrecord = NULL;
1561 }
else if (m->m_nextpkt == NULL) {
1562 sb->sb_lastrecord = m;
1575 SOCKBUF_LOCK_ASSERT(sb);
1587 SOCKBUF_LOCK_ASSERT(sb);
1608 KASSERT(sb->sb_mb != NULL, (
"%s: sb_mb is NULL", __func__));
1609 if (sb->sb_sndptr == NULL || sb->sb_sndptroff > off) {
1611 if (sb->sb_sndptr == NULL) {
1612 sb->sb_sndptr = sb->sb_mb;
1613 sb->sb_sndptroff = 0;
1618 off -= sb->sb_sndptroff;
1633 if (mb != sb->sb_sndptr) {
1638 while (m && (len > 0)) {
1639 if (len >= m->m_len) {
1642 sb->sb_sndptroff += m->m_len;
1643 sb->sb_sndptr = m->m_next;
1661 KASSERT(sb->sb_mb != NULL, (
"%s: sb_mb is NULL", __func__));
1667 if (sb->sb_sndptr == NULL || sb->sb_sndptroff > off) {
1671 off -= sb->sb_sndptroff;
1673 while (off > 0 && m != NULL) {
1692 SOCKBUF_LOCK_ASSERT(sb);
1696 sb->sb_mb = m->m_nextpkt;
1728 MBUF_CHECKSLEEP(wait);
1729 if (CMSG_SPACE((u_int)size) > MCLBYTES)
1730 return ((
struct mbuf *) NULL);
1731 if (CMSG_SPACE((u_int)size) > MLEN)
1732 m = m_getcl(wait, MT_CONTROL, 0);
1734 m = m_get(wait, MT_CONTROL);
1736 return ((
struct mbuf *) NULL);
1737 cp = mtod(m,
struct cmsghdr *);
1739 KASSERT(CMSG_SPACE((u_int)size) <= M_TRAILINGSPACE(m),
1740 (
"sbcreatecontrol: short mbuf"));
1745 bzero(cp, CMSG_SPACE((u_int)size));
1747 (void)memcpy(CMSG_DATA(cp), p, size);
1748 m->m_len = CMSG_SPACE(size);
1749 cp->cmsg_len = CMSG_LEN(size);
1750 cp->cmsg_level =
level;
1751 cp->cmsg_type =
type;
1773 xsb->sb_cc = sb->sb_ccc;
1774 xsb->sb_hiwat = sb->sb_hiwat;
1775 xsb->sb_mbcnt = sb->sb_mbcnt;
1776 xsb->sb_mcnt = sb->sb_mcnt;
1777 xsb->sb_ccnt = sb->sb_ccnt;
1778 xsb->sb_mbmax = sb->sb_mbmax;
1779 xsb->sb_lowat = sb->sb_lowat;
1780 xsb->sb_flags = sb->sb_flags;
1781 xsb->sb_timeo = sb->sb_timeo;
1788 CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE, &
sb_max, 0,
1790 "Maximum socket buffer size");
1791SYSCTL_ULONG(_kern_ipc, KIPC_SOCKBUF_WASTE, sockbuf_waste_factor, CTLFLAG_RW,
const struct cf_level * level
device_property_type_t type
void m_freem(struct mbuf *mb)
int mb_unmapped_compress(struct mbuf *m)
int chgsbsize(struct uidinfo *uip, u_int *hiwat, u_int to, rlim_t max)
rlim_t() lim_cur(struct thread *td, int which)
void panic(const char *fmt,...)
void pgsigio(struct sigio **sigiop, int sig, int checkctty)
void wakeup(const void *ident)
int sysctl_handle_long(SYSCTL_HANDLER_ARGS)
int printf(const char *fmt,...)
void selwakeuppri(struct selinfo *sip, int pri)
void sowakeup_aio(struct socket *so, struct sockbuf *sb)
void ktls_seq(struct sockbuf *sb, struct mbuf *m)
void ktls_check_rx(struct sockbuf *sb)
void m_tag_delete_chain(struct mbuf *m, struct m_tag *t)
void m_demote(struct mbuf *m0, int all, int flags)
void m_copydata(const struct mbuf *m, int off, int len, caddr_t cp)
u_int m_length(struct mbuf *m0, struct mbuf **last)
void soroverflow(struct socket *so)
int sbwait(struct sockbuf *sb)
#define SBLINKRECORD(sb, m0)
void socantsendmore_locked(struct socket *so)
static void sbflush_internal(struct sockbuf *sb)
static int sbappendaddr_locked_internal(struct sockbuf *sb, const struct sockaddr *asa, struct mbuf *m0, struct mbuf *control, struct mbuf *ctrl_last)
void soroverflow_locked(struct socket *so)
void sbappendrecord(struct sockbuf *sb, struct mbuf *m0)
int sbsetopt(struct socket *so, int cmd, u_long cc)
void sbsndptr_adv(struct sockbuf *sb, struct mbuf *mb, uint32_t len)
void sbrelease(struct sockbuf *sb, struct socket *so)
void sbappendcontrol_locked(struct sockbuf *sb, struct mbuf *m0, struct mbuf *control, int flags)
SYSCTL_INT(_kern, KERN_DUMMY, dummy, CTLFLAG_RW|CTLFLAG_SKIP, &dummy, 0, "")
void sbfree(struct sockbuf *sb, struct mbuf *m)
static u_long sb_efficiency
void socantsendmore(struct socket *so)
void sbappendrecord_locked(struct sockbuf *sb, struct mbuf *m0)
static struct mbuf * sbcut_internal(struct sockbuf *sb, int len)
void sbflush_locked(struct sockbuf *sb)
static void sbready_compress(struct sockbuf *sb, struct mbuf *m0, struct mbuf *end)
static void sbm_clrprotoflags(struct mbuf *m, int flags)
void sbappendstream_locked(struct sockbuf *sb, struct mbuf *m, int flags)
int soreserve(struct socket *so, u_long sndcc, u_long rcvcc)
void sbdestroy(struct sockbuf *sb, struct socket *so)
void sbtoxsockbuf(struct sockbuf *sb, struct xsockbuf *xsb)
void sbflush(struct sockbuf *sb)
int sbappendaddr_nospacecheck_locked(struct sockbuf *sb, const struct sockaddr *asa, struct mbuf *m0, struct mbuf *control)
void sbdroprecord(struct sockbuf *sb)
int sbready(struct sockbuf *sb, struct mbuf *m0, int count)
struct mbuf * sbsndmbuf(struct sockbuf *sb, u_int off, u_int *moff)
void sbdrop(struct sockbuf *sb, int len)
void socantrcvmore_locked(struct socket *so)
void(* aio_swake)(struct socket *, struct sockbuf *)
void sbappendcontrol(struct sockbuf *sb, struct mbuf *m0, struct mbuf *control, int flags)
void sbappend(struct sockbuf *sb, struct mbuf *m, int flags)
void sbappend_locked(struct sockbuf *sb, struct mbuf *m, int flags)
void sballoc(struct sockbuf *sb, struct mbuf *m)
void sbdroprecord_locked(struct sockbuf *sb)
void sbappendstream(struct sockbuf *sb, struct mbuf *m, int flags)
void sbcompress(struct sockbuf *sb, struct mbuf *m, struct mbuf *n)
int sbreserve_locked(struct sockbuf *sb, u_long cc, struct socket *so, struct thread *td)
struct mbuf * sbcut_locked(struct sockbuf *sb, int len)
void sbdrop_locked(struct sockbuf *sb, int len)
SYSCTL_OID(_kern_ipc, KIPC_MAXSOCKBUF, maxsockbuf, CTLTYPE_ULONG|CTLFLAG_RW|CTLFLAG_MPSAFE, &sb_max, 0, sysctl_handle_sb_max, "LU", "Maximum socket buffer size")
static int sysctl_handle_sb_max(SYSCTL_HANDLER_ARGS)
struct mbuf * sbsndptr_noadv(struct sockbuf *sb, uint32_t off, uint32_t *moff)
void sbrelease_internal(struct sockbuf *sb, struct socket *so)
struct mbuf * sbcreatecontrol(caddr_t p, int size, int type, int level)
int sbappendaddr(struct sockbuf *sb, const struct sockaddr *asa, struct mbuf *m0, struct mbuf *control)
struct mbuf * sbcreatecontrol_how(void *p, int size, int type, int level, int wait)
SYSCTL_ULONG(_kern_ipc, KIPC_SOCKBUF_WASTE, sockbuf_waste_factor, CTLFLAG_RW, &sb_efficiency, 0, "Socket buffer size waste factor")
void socantrcvmore(struct socket *so)
int sbappendaddr_locked(struct sockbuf *sb, const struct sockaddr *asa, struct mbuf *m0, struct mbuf *control)
void sbrelease_locked(struct sockbuf *sb, struct socket *so)
void sowakeup(struct socket *so, struct sockbuf *sb)
void soisconnected(struct socket *so)
void soupcall_clear(struct socket *so, int which)