FreeBSD kernel IPv4 code
tcp_log_buf.c
Go to the documentation of this file.
1/*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2016-2018 Netflix, Inc.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 *
27 */
28
29#include <sys/cdefs.h>
30__FBSDID("$FreeBSD$");
31
32#include <sys/param.h>
33#include <sys/arb.h>
34#include <sys/kernel.h>
35#include <sys/lock.h>
36#include <sys/malloc.h>
37#include <sys/mutex.h>
38#include <sys/qmath.h>
39#include <sys/queue.h>
40#include <sys/refcount.h>
41#include <sys/rwlock.h>
42#include <sys/socket.h>
43#include <sys/socketvar.h>
44#include <sys/sysctl.h>
45#include <sys/tree.h>
46#include <sys/stats.h> /* Must come after qmath.h and tree.h */
47#include <sys/counter.h>
48
49#include <dev/tcp_log/tcp_log_dev.h>
50
51#include <net/if.h>
52#include <net/if_var.h>
53#include <net/vnet.h>
54
55#include <netinet/in.h>
56#include <netinet/in_pcb.h>
57#include <netinet/in_var.h>
58#include <netinet/tcp_var.h>
59#include <netinet/tcp_log_buf.h>
60
61/* Default expiry time */
62#define TCP_LOG_EXPIRE_TIME ((sbintime_t)60 * SBT_1S)
63
64/* Max interval at which to run the expiry timer */
65#define TCP_LOG_EXPIRE_INTVL ((sbintime_t)5 * SBT_1S)
66
67bool tcp_log_verbose;
68static uma_zone_t tcp_log_bucket_zone, tcp_log_node_zone, tcp_log_zone;
69static int tcp_log_session_limit = TCP_LOG_BUF_DEFAULT_SESSION_LIMIT;
70static uint32_t tcp_log_version = TCP_LOG_BUF_VER;
71RB_HEAD(tcp_log_id_tree, tcp_log_id_bucket);
72static struct tcp_log_id_tree tcp_log_id_head;
73static STAILQ_HEAD(, tcp_log_id_node) tcp_log_expireq_head =
74 STAILQ_HEAD_INITIALIZER(tcp_log_expireq_head);
75static struct mtx tcp_log_expireq_mtx;
76static struct callout tcp_log_expireq_callout;
77static u_long tcp_log_auto_ratio = 0;
78static volatile u_long tcp_log_auto_ratio_cur = 0;
79static uint32_t tcp_log_auto_mode = TCP_LOG_STATE_TAIL;
80static bool tcp_log_auto_all = false;
81static uint32_t tcp_disable_all_bb_logs = 0;
82
83RB_PROTOTYPE_STATIC(tcp_log_id_tree, tcp_log_id_bucket, tlb_rb, tcp_log_id_cmp)
84
85SYSCTL_NODE(_net_inet_tcp, OID_AUTO, bb, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
86 "TCP Black Box controls");
87
88SYSCTL_BOOL(_net_inet_tcp_bb, OID_AUTO, log_verbose, CTLFLAG_RW, &tcp_log_verbose,
89 0, "Force verbose logging for TCP traces");
90
91SYSCTL_INT(_net_inet_tcp_bb, OID_AUTO, log_session_limit,
92 CTLFLAG_RW, &tcp_log_session_limit, 0,
93 "Maximum number of events maintained for each TCP session");
94
95SYSCTL_UMA_MAX(_net_inet_tcp_bb, OID_AUTO, log_global_limit, CTLFLAG_RW,
96 &tcp_log_zone, "Maximum number of events maintained for all TCP sessions");
97
98SYSCTL_UMA_CUR(_net_inet_tcp_bb, OID_AUTO, log_global_entries, CTLFLAG_RD,
99 &tcp_log_zone, "Current number of events maintained for all TCP sessions");
100
101SYSCTL_UMA_MAX(_net_inet_tcp_bb, OID_AUTO, log_id_limit, CTLFLAG_RW,
102 &tcp_log_bucket_zone, "Maximum number of log IDs");
103
104SYSCTL_UMA_CUR(_net_inet_tcp_bb, OID_AUTO, log_id_entries, CTLFLAG_RD,
105 &tcp_log_bucket_zone, "Current number of log IDs");
106
107SYSCTL_UMA_MAX(_net_inet_tcp_bb, OID_AUTO, log_id_tcpcb_limit, CTLFLAG_RW,
108 &tcp_log_node_zone, "Maximum number of tcpcbs with log IDs");
109
110SYSCTL_UMA_CUR(_net_inet_tcp_bb, OID_AUTO, log_id_tcpcb_entries, CTLFLAG_RD,
111 &tcp_log_node_zone, "Current number of tcpcbs with log IDs");
112
113SYSCTL_U32(_net_inet_tcp_bb, OID_AUTO, log_version, CTLFLAG_RD, &tcp_log_version,
114 0, "Version of log formats exported");
115
116SYSCTL_U32(_net_inet_tcp_bb, OID_AUTO, disable_all, CTLFLAG_RW,
117 &tcp_disable_all_bb_logs, TCP_LOG_STATE_HEAD_AUTO,
118 "Disable all BB logging for all connections");
119
120SYSCTL_ULONG(_net_inet_tcp_bb, OID_AUTO, log_auto_ratio, CTLFLAG_RW,
121 &tcp_log_auto_ratio, 0, "Do auto capturing for 1 out of N sessions");
122
123SYSCTL_U32(_net_inet_tcp_bb, OID_AUTO, log_auto_mode, CTLFLAG_RW,
124 &tcp_log_auto_mode, TCP_LOG_STATE_HEAD_AUTO,
125 "Logging mode for auto-selected sessions (default is TCP_LOG_STATE_HEAD_AUTO)");
126
127SYSCTL_BOOL(_net_inet_tcp_bb, OID_AUTO, log_auto_all, CTLFLAG_RW,
128 &tcp_log_auto_all, false,
129 "Auto-select from all sessions (rather than just those with IDs)");
130
131#ifdef TCPLOG_DEBUG_COUNTERS
132counter_u64_t tcp_log_queued;
133counter_u64_t tcp_log_que_fail1;
134counter_u64_t tcp_log_que_fail2;
135counter_u64_t tcp_log_que_fail3;
136counter_u64_t tcp_log_que_fail4;
137counter_u64_t tcp_log_que_fail5;
138counter_u64_t tcp_log_que_copyout;
139counter_u64_t tcp_log_que_read;
140counter_u64_t tcp_log_que_freed;
141
142SYSCTL_COUNTER_U64(_net_inet_tcp_bb, OID_AUTO, queued, CTLFLAG_RD,
143 &tcp_log_queued, "Number of entries queued");
144SYSCTL_COUNTER_U64(_net_inet_tcp_bb, OID_AUTO, fail1, CTLFLAG_RD,
145 &tcp_log_que_fail1, "Number of entries queued but fail 1");
146SYSCTL_COUNTER_U64(_net_inet_tcp_bb, OID_AUTO, fail2, CTLFLAG_RD,
147 &tcp_log_que_fail2, "Number of entries queued but fail 2");
148SYSCTL_COUNTER_U64(_net_inet_tcp_bb, OID_AUTO, fail3, CTLFLAG_RD,
149 &tcp_log_que_fail3, "Number of entries queued but fail 3");
150SYSCTL_COUNTER_U64(_net_inet_tcp_bb, OID_AUTO, fail4, CTLFLAG_RD,
151 &tcp_log_que_fail4, "Number of entries queued but fail 4");
152SYSCTL_COUNTER_U64(_net_inet_tcp_bb, OID_AUTO, fail5, CTLFLAG_RD,
153 &tcp_log_que_fail5, "Number of entries queued but fail 4");
154SYSCTL_COUNTER_U64(_net_inet_tcp_bb, OID_AUTO, copyout, CTLFLAG_RD,
155 &tcp_log_que_copyout, "Number of entries copied out");
156SYSCTL_COUNTER_U64(_net_inet_tcp_bb, OID_AUTO, read, CTLFLAG_RD,
157 &tcp_log_que_read, "Number of entries read from the queue");
158SYSCTL_COUNTER_U64(_net_inet_tcp_bb, OID_AUTO, freed, CTLFLAG_RD,
159 &tcp_log_que_freed, "Number of entries freed after reading");
160#endif
161
162#ifdef INVARIANTS
163#define TCPLOG_DEBUG_RINGBUF
164#endif
165/* Number of requests to consider a PBCID "active". */
166#define ACTIVE_REQUEST_COUNT 10
167
168/* Statistic tracking for "active" PBCIDs. */
169static counter_u64_t tcp_log_pcb_ids_cur;
170static counter_u64_t tcp_log_pcb_ids_tot;
171
172SYSCTL_COUNTER_U64(_net_inet_tcp_bb, OID_AUTO, pcb_ids_cur, CTLFLAG_RD,
173 &tcp_log_pcb_ids_cur, "Number of pcb IDs allocated in the system");
174SYSCTL_COUNTER_U64(_net_inet_tcp_bb, OID_AUTO, pcb_ids_tot, CTLFLAG_RD,
175 &tcp_log_pcb_ids_tot, "Total number of pcb IDs that have been allocated");
176
177struct tcp_log_mem
178{
179 STAILQ_ENTRY(tcp_log_mem) tlm_queue;
180 struct tcp_log_buffer tlm_buf;
181 struct tcp_log_verbose tlm_v;
182#ifdef TCPLOG_DEBUG_RINGBUF
183 volatile int tlm_refcnt;
184#endif
185};
186
187/* 60 bytes for the header, + 16 bytes for padding */
188static uint8_t zerobuf[76];
189
190/*
191 * Lock order:
192 * 1. TCPID_TREE
193 * 2. TCPID_BUCKET
194 * 3. INP
195 *
196 * Rules:
197 * A. You need a lock on the Tree to add/remove buckets.
198 * B. You need a lock on the bucket to add/remove nodes from the bucket.
199 * C. To change information in a node, you need the INP lock if the tln_closed
200 * field is false. Otherwise, you need the bucket lock. (Note that the
201 * tln_closed field can change at any point, so you need to recheck the
202 * entry after acquiring the INP lock.)
203 * D. To remove a node from the bucket, you must have that entry locked,
204 * according to the criteria of Rule C. Also, the node must not be on
205 * the expiry queue.
206 * E. The exception to C is the expiry queue fields, which are locked by
207 * the TCPLOG_EXPIREQ lock.
208 *
209 * Buckets have a reference count. Each node is a reference. Further,
210 * other callers may add reference counts to keep a bucket from disappearing.
211 * You can add a reference as long as you own a lock sufficient to keep the
212 * bucket from disappearing. For example, a common use is:
213 * a. Have a locked INP, but need to lock the TCPID_BUCKET.
214 * b. Add a refcount on the bucket. (Safe because the INP lock prevents
215 * the TCPID_BUCKET from going away.)
216 * c. Drop the INP lock.
217 * d. Acquire a lock on the TCPID_BUCKET.
218 * e. Acquire a lock on the INP.
219 * f. Drop the refcount on the bucket.
220 * (At this point, the bucket may disappear.)
221 *
222 * Expire queue lock:
223 * You can acquire this with either the bucket or INP lock. Don't reverse it.
224 * When the expire code has committed to freeing a node, it resets the expiry
225 * time to SBT_MAX. That is the signal to everyone else that they should
226 * leave that node alone.
227 */
228static struct rwlock tcp_id_tree_lock;
229#define TCPID_TREE_WLOCK() rw_wlock(&tcp_id_tree_lock)
230#define TCPID_TREE_RLOCK() rw_rlock(&tcp_id_tree_lock)
231#define TCPID_TREE_UPGRADE() rw_try_upgrade(&tcp_id_tree_lock)
232#define TCPID_TREE_WUNLOCK() rw_wunlock(&tcp_id_tree_lock)
233#define TCPID_TREE_RUNLOCK() rw_runlock(&tcp_id_tree_lock)
234#define TCPID_TREE_WLOCK_ASSERT() rw_assert(&tcp_id_tree_lock, RA_WLOCKED)
235#define TCPID_TREE_RLOCK_ASSERT() rw_assert(&tcp_id_tree_lock, RA_RLOCKED)
236#define TCPID_TREE_UNLOCK_ASSERT() rw_assert(&tcp_id_tree_lock, RA_UNLOCKED)
237
238#define TCPID_BUCKET_LOCK_INIT(tlb) mtx_init(&((tlb)->tlb_mtx), "tcp log id bucket", NULL, MTX_DEF)
239#define TCPID_BUCKET_LOCK_DESTROY(tlb) mtx_destroy(&((tlb)->tlb_mtx))
240#define TCPID_BUCKET_LOCK(tlb) mtx_lock(&((tlb)->tlb_mtx))
241#define TCPID_BUCKET_UNLOCK(tlb) mtx_unlock(&((tlb)->tlb_mtx))
242#define TCPID_BUCKET_LOCK_ASSERT(tlb) mtx_assert(&((tlb)->tlb_mtx), MA_OWNED)
243#define TCPID_BUCKET_UNLOCK_ASSERT(tlb) mtx_assert(&((tlb)->tlb_mtx), MA_NOTOWNED)
244
245#define TCPID_BUCKET_REF(tlb) refcount_acquire(&((tlb)->tlb_refcnt))
246#define TCPID_BUCKET_UNREF(tlb) refcount_release(&((tlb)->tlb_refcnt))
247
248#define TCPLOG_EXPIREQ_LOCK() mtx_lock(&tcp_log_expireq_mtx)
249#define TCPLOG_EXPIREQ_UNLOCK() mtx_unlock(&tcp_log_expireq_mtx)
250
251SLIST_HEAD(tcp_log_id_head, tcp_log_id_node);
252
253struct tcp_log_id_bucket
254{
255 /*
256 * tlb_id must be first. This lets us use strcmp on
257 * (struct tcp_log_id_bucket *) and (char *) interchangeably.
258 */
259 char tlb_id[TCP_LOG_ID_LEN];
260 char tlb_tag[TCP_LOG_TAG_LEN];
261 RB_ENTRY(tcp_log_id_bucket) tlb_rb;
262 struct tcp_log_id_head tlb_head;
263 struct mtx tlb_mtx;
264 volatile u_int tlb_refcnt;
265 volatile u_int tlb_reqcnt;
266 uint32_t tlb_loglimit;
267 uint8_t tlb_logstate;
268};
269
270struct tcp_log_id_node
271{
272 SLIST_ENTRY(tcp_log_id_node) tln_list;
273 STAILQ_ENTRY(tcp_log_id_node) tln_expireq; /* Locked by the expireq lock */
274 sbintime_t tln_expiretime; /* Locked by the expireq lock */
275
276 /*
277 * If INP is NULL, that means the connection has closed. We've
278 * saved the connection endpoint information and the log entries
279 * in the tln_ie and tln_entries members. We've also saved a pointer
280 * to the enclosing bucket here. If INP is not NULL, the information is
281 * in the PCB and not here.
282 */
283 struct inpcb *tln_inp;
284 struct tcpcb *tln_tp;
285 struct tcp_log_id_bucket *tln_bucket;
286 struct in_endpoints tln_ie;
287 struct tcp_log_stailq tln_entries;
288 int tln_count;
289 volatile int tln_closed;
290 uint8_t tln_af;
291};
292
293enum tree_lock_state {
294 TREE_UNLOCKED = 0,
295 TREE_RLOCKED,
296 TREE_WLOCKED,
297};
298
299/* Do we want to select this session for auto-logging? */
300static __inline bool
301tcp_log_selectauto(void)
302{
303
304 /*
305 * If we are doing auto-capturing, figure out whether we will capture
306 * this session.
307 */
308 if (tcp_log_auto_ratio &&
309 (tcp_disable_all_bb_logs == 0) &&
310 (atomic_fetchadd_long(&tcp_log_auto_ratio_cur, 1) %
311 tcp_log_auto_ratio) == 0)
312 return (true);
313 return (false);
314}
315
316static __inline int
317tcp_log_id_cmp(struct tcp_log_id_bucket *a, struct tcp_log_id_bucket *b)
318{
319 KASSERT(a != NULL, ("tcp_log_id_cmp: argument a is unexpectedly NULL"));
320 KASSERT(b != NULL, ("tcp_log_id_cmp: argument b is unexpectedly NULL"));
321 return strncmp(a->tlb_id, b->tlb_id, TCP_LOG_ID_LEN);
322}
323
324RB_GENERATE_STATIC(tcp_log_id_tree, tcp_log_id_bucket, tlb_rb, tcp_log_id_cmp)
325
326static __inline void
327tcp_log_id_validate_tree_lock(int tree_locked)
328{
329
330#ifdef INVARIANTS
331 switch (tree_locked) {
332 case TREE_WLOCKED:
333 TCPID_TREE_WLOCK_ASSERT();
334 break;
335 case TREE_RLOCKED:
336 TCPID_TREE_RLOCK_ASSERT();
337 break;
338 case TREE_UNLOCKED:
339 TCPID_TREE_UNLOCK_ASSERT();
340 break;
341 default:
342 kassert_panic("%s:%d: unknown tree lock state", __func__,
343 __LINE__);
344 }
345#endif
346}
347
348static __inline void
349tcp_log_remove_bucket(struct tcp_log_id_bucket *tlb)
350{
351
352 TCPID_TREE_WLOCK_ASSERT();
353 KASSERT(SLIST_EMPTY(&tlb->tlb_head),
354 ("%s: Attempt to remove non-empty bucket", __func__));
355 if (RB_REMOVE(tcp_log_id_tree, &tcp_log_id_head, tlb) == NULL) {
356#ifdef INVARIANTS
357 kassert_panic("%s:%d: error removing element from tree",
358 __func__, __LINE__);
359#endif
360 }
361 TCPID_BUCKET_LOCK_DESTROY(tlb);
362 counter_u64_add(tcp_log_pcb_ids_cur, (int64_t)-1);
363 uma_zfree(tcp_log_bucket_zone, tlb);
364}
365
366/*
367 * Call with a referenced and locked bucket.
368 * Will return true if the bucket was freed; otherwise, false.
369 * tlb: The bucket to unreference.
370 * tree_locked: A pointer to the state of the tree lock. If the tree lock
371 * state changes, the function will update it.
372 * inp: If not NULL and the function needs to drop the inp lock to relock the
373 * tree, it will do so. (The caller must ensure inp will not become invalid,
374 * probably by holding a reference to it.)
375 */
376static bool
377tcp_log_unref_bucket(struct tcp_log_id_bucket *tlb, int *tree_locked,
378 struct inpcb *inp)
379{
380
381 KASSERT(tlb != NULL, ("%s: called with NULL tlb", __func__));
382 KASSERT(tree_locked != NULL, ("%s: called with NULL tree_locked",
383 __func__));
384
385 tcp_log_id_validate_tree_lock(*tree_locked);
386
387 /*
388 * Did we hold the last reference on the tlb? If so, we may need
389 * to free it. (Note that we can realistically only execute the
390 * loop twice: once without a write lock and once with a write
391 * lock.)
392 */
393 while (TCPID_BUCKET_UNREF(tlb)) {
394 /*
395 * We need a write lock on the tree to free this.
396 * If we can upgrade the tree lock, this is "easy". If we
397 * can't upgrade the tree lock, we need to do this the
398 * "hard" way: unwind all our locks and relock everything.
399 * In the meantime, anything could have changed. We even
400 * need to validate that we still need to free the bucket.
401 */
402 if (*tree_locked == TREE_RLOCKED && TCPID_TREE_UPGRADE())
403 *tree_locked = TREE_WLOCKED;
404 else if (*tree_locked != TREE_WLOCKED) {
405 TCPID_BUCKET_REF(tlb);
406 if (inp != NULL)
407 INP_WUNLOCK(inp);
408 TCPID_BUCKET_UNLOCK(tlb);
409 if (*tree_locked == TREE_RLOCKED)
410 TCPID_TREE_RUNLOCK();
411 TCPID_TREE_WLOCK();
412 *tree_locked = TREE_WLOCKED;
413 TCPID_BUCKET_LOCK(tlb);
414 if (inp != NULL)
415 INP_WLOCK(inp);
416 continue;
417 }
418
419 /*
420 * We have an empty bucket and a write lock on the tree.
421 * Remove the empty bucket.
422 */
423 tcp_log_remove_bucket(tlb);
424 return (true);
425 }
426 return (false);
427}
428
429/*
430 * Call with a locked bucket. This function will release the lock on the
431 * bucket before returning.
432 *
433 * The caller is responsible for freeing the tp->t_lin/tln node!
434 *
435 * Note: one of tp or both tlb and tln must be supplied.
436 *
437 * inp: A pointer to the inp. If the function needs to drop the inp lock to
438 * acquire the tree write lock, it will do so. (The caller must ensure inp
439 * will not become invalid, probably by holding a reference to it.)
440 * tp: A pointer to the tcpcb. (optional; if specified, tlb and tln are ignored)
441 * tlb: A pointer to the bucket. (optional; ignored if tp is specified)
442 * tln: A pointer to the node. (optional; ignored if tp is specified)
443 * tree_locked: A pointer to the state of the tree lock. If the tree lock
444 * state changes, the function will update it.
445 *
446 * Will return true if the INP lock was reacquired; otherwise, false.
447 */
448static bool
449tcp_log_remove_id_node(struct inpcb *inp, struct tcpcb *tp,
450 struct tcp_log_id_bucket *tlb, struct tcp_log_id_node *tln,
451 int *tree_locked)
452{
453 int orig_tree_locked;
454
455 KASSERT(tp != NULL || (tlb != NULL && tln != NULL),
456 ("%s: called with tp=%p, tlb=%p, tln=%p", __func__,
457 tp, tlb, tln));
458 KASSERT(tree_locked != NULL, ("%s: called with NULL tree_locked",
459 __func__));
460
461 if (tp != NULL) {
462 tlb = tp->t_lib;
463 tln = tp->t_lin;
464 KASSERT(tlb != NULL, ("%s: unexpectedly NULL tlb", __func__));
465 KASSERT(tln != NULL, ("%s: unexpectedly NULL tln", __func__));
466 }
467
468 tcp_log_id_validate_tree_lock(*tree_locked);
469 TCPID_BUCKET_LOCK_ASSERT(tlb);
470
471 /*
472 * Remove the node, clear the log bucket and node from the TCPCB, and
473 * decrement the bucket refcount. In the process, if this is the
474 * last reference, the bucket will be freed.
475 */
476 SLIST_REMOVE(&tlb->tlb_head, tln, tcp_log_id_node, tln_list);
477 if (tp != NULL) {
478 tp->t_lib = NULL;
479 tp->t_lin = NULL;
480 }
481 orig_tree_locked = *tree_locked;
482 if (!tcp_log_unref_bucket(tlb, tree_locked, inp))
483 TCPID_BUCKET_UNLOCK(tlb);
484 return (*tree_locked != orig_tree_locked);
485}
486
487#define RECHECK_INP_CLEAN(cleanup) do { \
488 if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) { \
489 rv = ECONNRESET; \
490 cleanup; \
491 goto done; \
492 } \
493 tp = intotcpcb(inp); \
494} while (0)
495
496#define RECHECK_INP() RECHECK_INP_CLEAN(/* noop */)
497
498static void
499tcp_log_grow_tlb(char *tlb_id, struct tcpcb *tp)
500{
501
502 INP_WLOCK_ASSERT(tp->t_inpcb);
503
504#ifdef STATS
505 if (V_tcp_perconn_stats_enable == 2 && tp->t_stats == NULL)
506 (void)tcp_stats_sample_rollthedice(tp, tlb_id, strlen(tlb_id));
507#endif
508}
509
510static void
511tcp_log_increment_reqcnt(struct tcp_log_id_bucket *tlb)
512{
513
514 atomic_fetchadd_int(&tlb->tlb_reqcnt, 1);
515}
516
517/*
518 * Associate the specified tag with a particular TCP log ID.
519 * Called with INPCB locked. Returns with it unlocked.
520 * Returns 0 on success or EOPNOTSUPP if the connection has no TCP log ID.
521 */
522int
523tcp_log_set_tag(struct tcpcb *tp, char *tag)
524{
525 struct tcp_log_id_bucket *tlb;
526 int tree_locked;
527
528 INP_WLOCK_ASSERT(tp->t_inpcb);
529
530 tree_locked = TREE_UNLOCKED;
531 tlb = tp->t_lib;
532 if (tlb == NULL) {
533 INP_WUNLOCK(tp->t_inpcb);
534 return (EOPNOTSUPP);
535 }
536
537 TCPID_BUCKET_REF(tlb);
538 INP_WUNLOCK(tp->t_inpcb);
539 TCPID_BUCKET_LOCK(tlb);
540 strlcpy(tlb->tlb_tag, tag, TCP_LOG_TAG_LEN);
541 if (!tcp_log_unref_bucket(tlb, &tree_locked, NULL))
542 TCPID_BUCKET_UNLOCK(tlb);
543
544 if (tree_locked == TREE_WLOCKED) {
545 TCPID_TREE_WLOCK_ASSERT();
546 TCPID_TREE_WUNLOCK();
547 } else if (tree_locked == TREE_RLOCKED) {
548 TCPID_TREE_RLOCK_ASSERT();
549 TCPID_TREE_RUNLOCK();
550 } else
551 TCPID_TREE_UNLOCK_ASSERT();
552
553 return (0);
554}
555
556/*
557 * Set the TCP log ID for a TCPCB.
558 * Called with INPCB locked. Returns with it unlocked.
559 */
560int
561tcp_log_set_id(struct tcpcb *tp, char *id)
562{
563 struct tcp_log_id_bucket *tlb, *tmp_tlb;
564 struct tcp_log_id_node *tln;
565 struct inpcb *inp;
566 int tree_locked, rv;
567 bool bucket_locked;
568
569 tlb = NULL;
570 tln = NULL;
571 inp = tp->t_inpcb;
572 tree_locked = TREE_UNLOCKED;
573 bucket_locked = false;
574
575restart:
576 INP_WLOCK_ASSERT(inp);
577
578 /* See if the ID is unchanged. */
579 if ((tp->t_lib != NULL && !strcmp(tp->t_lib->tlb_id, id)) ||
580 (tp->t_lib == NULL && *id == 0)) {
581 if (tp->t_lib != NULL) {
582 tcp_log_increment_reqcnt(tp->t_lib);
583 if ((tp->t_lib->tlb_logstate) &&
584 (tp->t_log_state_set == 0)) {
585 /* Clone in any logging */
586
587 tp->t_logstate = tp->t_lib->tlb_logstate;
588 }
589 if ((tp->t_lib->tlb_loglimit) &&
590 (tp->t_log_state_set == 0)) {
591 /* We also have a limit set */
592
593 tp->t_loglimit = tp->t_lib->tlb_loglimit;
594 }
595 }
596 rv = 0;
597 goto done;
598 }
599
600 /*
601 * If the TCPCB had a previous ID, we need to extricate it from
602 * the previous list.
603 *
604 * Drop the TCPCB lock and lock the tree and the bucket.
605 * Because this is called in the socket context, we (theoretically)
606 * don't need to worry about the INPCB completely going away
607 * while we are gone.
608 */
609 if (tp->t_lib != NULL) {
610 tlb = tp->t_lib;
611 TCPID_BUCKET_REF(tlb);
612 INP_WUNLOCK(inp);
613
614 if (tree_locked == TREE_UNLOCKED) {
615 TCPID_TREE_RLOCK();
616 tree_locked = TREE_RLOCKED;
617 }
618 TCPID_BUCKET_LOCK(tlb);
619 bucket_locked = true;
620 INP_WLOCK(inp);
621
622 /*
623 * Unreference the bucket. If our bucket went away, it is no
624 * longer locked or valid.
625 */
626 if (tcp_log_unref_bucket(tlb, &tree_locked, inp)) {
627 bucket_locked = false;
628 tlb = NULL;
629 }
630
631 /* Validate the INP. */
632 RECHECK_INP();
633
634 /*
635 * Evaluate whether the bucket changed while we were unlocked.
636 *
637 * Possible scenarios here:
638 * 1. Bucket is unchanged and the same one we started with.
639 * 2. The TCPCB no longer has a bucket and our bucket was
640 * freed.
641 * 3. The TCPCB has a new bucket, whether ours was freed.
642 * 4. The TCPCB no longer has a bucket and our bucket was
643 * not freed.
644 *
645 * In cases 2-4, we will start over. In case 1, we will
646 * proceed here to remove the bucket.
647 */
648 if (tlb == NULL || tp->t_lib != tlb) {
649 KASSERT(bucket_locked || tlb == NULL,
650 ("%s: bucket_locked (%d) and tlb (%p) are "
651 "inconsistent", __func__, bucket_locked, tlb));
652
653 if (bucket_locked) {
654 TCPID_BUCKET_UNLOCK(tlb);
655 bucket_locked = false;
656 tlb = NULL;
657 }
658 goto restart;
659 }
660
661 /*
662 * Store the (struct tcp_log_id_node) for reuse. Then, remove
663 * it from the bucket. In the process, we may end up relocking.
664 * If so, we need to validate that the INP is still valid, and
665 * the TCPCB entries match we expect.
666 *
667 * We will clear tlb and change the bucket_locked state just
668 * before calling tcp_log_remove_id_node(), since that function
669 * will unlock the bucket.
670 */
671 if (tln != NULL)
672 uma_zfree(tcp_log_node_zone, tln);
673 tln = tp->t_lin;
674 tlb = NULL;
675 bucket_locked = false;
676 if (tcp_log_remove_id_node(inp, tp, NULL, NULL, &tree_locked)) {
677 RECHECK_INP();
678
679 /*
680 * If the TCPCB moved to a new bucket while we had
681 * dropped the lock, restart.
682 */
683 if (tp->t_lib != NULL || tp->t_lin != NULL)
684 goto restart;
685 }
686
687 /*
688 * Yay! We successfully removed the TCPCB from its old
689 * bucket. Phew!
690 *
691 * On to bigger and better things...
692 */
693 }
694
695 /* At this point, the TCPCB should not be in any bucket. */
696 KASSERT(tp->t_lib == NULL, ("%s: tp->t_lib is not NULL", __func__));
697
698 /*
699 * If the new ID is not empty, we need to now assign this TCPCB to a
700 * new bucket.
701 */
702 if (*id) {
703 /* Get a new tln, if we don't already have one to reuse. */
704 if (tln == NULL) {
705 tln = uma_zalloc(tcp_log_node_zone, M_NOWAIT | M_ZERO);
706 if (tln == NULL) {
707 rv = ENOBUFS;
708 goto done;
709 }
710 tln->tln_inp = inp;
711 tln->tln_tp = tp;
712 }
713
714 /*
715 * Drop the INP lock for a bit. We don't need it, and dropping
716 * it prevents lock order reversals.
717 */
718 INP_WUNLOCK(inp);
719
720 /* Make sure we have at least a read lock on the tree. */
721 tcp_log_id_validate_tree_lock(tree_locked);
722 if (tree_locked == TREE_UNLOCKED) {
723 TCPID_TREE_RLOCK();
724 tree_locked = TREE_RLOCKED;
725 }
726
727refind:
728 /*
729 * Remember that we constructed (struct tcp_log_id_node) so
730 * we can safely cast the id to it for the purposes of finding.
731 */
732 KASSERT(tlb == NULL, ("%s:%d tlb unexpectedly non-NULL",
733 __func__, __LINE__));
734 tmp_tlb = RB_FIND(tcp_log_id_tree, &tcp_log_id_head,
735 (struct tcp_log_id_bucket *) id);
736
737 /*
738 * If we didn't find a matching bucket, we need to add a new
739 * one. This requires a write lock. But, of course, we will
740 * need to recheck some things when we re-acquire the lock.
741 */
742 if (tmp_tlb == NULL && tree_locked != TREE_WLOCKED) {
743 tree_locked = TREE_WLOCKED;
744 if (!TCPID_TREE_UPGRADE()) {
745 TCPID_TREE_RUNLOCK();
746 TCPID_TREE_WLOCK();
747
748 /*
749 * The tree may have changed while we were
750 * unlocked.
751 */
752 goto refind;
753 }
754 }
755
756 /* If we need to add a new bucket, do it now. */
757 if (tmp_tlb == NULL) {
758 /* Allocate new bucket. */
759 tlb = uma_zalloc(tcp_log_bucket_zone, M_NOWAIT);
760 if (tlb == NULL) {
761 rv = ENOBUFS;
762 goto done_noinp;
763 }
764 counter_u64_add(tcp_log_pcb_ids_cur, 1);
765 counter_u64_add(tcp_log_pcb_ids_tot, 1);
766
767 if ((tcp_log_auto_all == false) &&
768 tcp_log_auto_mode &&
769 tcp_log_selectauto()) {
770 /* Save off the log state */
771 tlb->tlb_logstate = tcp_log_auto_mode;
772 } else
773 tlb->tlb_logstate = TCP_LOG_STATE_OFF;
774 tlb->tlb_loglimit = 0;
775 tlb->tlb_tag[0] = '\0'; /* Default to an empty tag. */
776
777 /*
778 * Copy the ID to the bucket.
779 * NB: Don't use strlcpy() unless you are sure
780 * we've always validated NULL termination.
781 *
782 * TODO: When I'm done writing this, see if we
783 * we have correctly validated NULL termination and
784 * can use strlcpy(). :-)
785 */
786 strncpy(tlb->tlb_id, id, TCP_LOG_ID_LEN - 1);
787 tlb->tlb_id[TCP_LOG_ID_LEN - 1] = '\0';
788
789 /*
790 * Take the refcount for the first node and go ahead
791 * and lock this. Note that we zero the tlb_mtx
792 * structure, since 0xdeadc0de flips the right bits
793 * for the code to think that this mutex has already
794 * been initialized. :-(
795 */
796 SLIST_INIT(&tlb->tlb_head);
797 refcount_init(&tlb->tlb_refcnt, 1);
798 tlb->tlb_reqcnt = 1;
799 memset(&tlb->tlb_mtx, 0, sizeof(struct mtx));
800 TCPID_BUCKET_LOCK_INIT(tlb);
801 TCPID_BUCKET_LOCK(tlb);
802 bucket_locked = true;
803
804#define FREE_NEW_TLB() do { \
805 TCPID_BUCKET_LOCK_DESTROY(tlb); \
806 uma_zfree(tcp_log_bucket_zone, tlb); \
807 counter_u64_add(tcp_log_pcb_ids_cur, (int64_t)-1); \
808 counter_u64_add(tcp_log_pcb_ids_tot, (int64_t)-1); \
809 bucket_locked = false; \
810 tlb = NULL; \
811} while (0)
812 /*
813 * Relock the INP and make sure we are still
814 * unassigned.
815 */
816 INP_WLOCK(inp);
817 RECHECK_INP_CLEAN(FREE_NEW_TLB());
818 if (tp->t_lib != NULL) {
819 FREE_NEW_TLB();
820 goto restart;
821 }
822
823 /* Add the new bucket to the tree. */
824 tmp_tlb = RB_INSERT(tcp_log_id_tree, &tcp_log_id_head,
825 tlb);
826 KASSERT(tmp_tlb == NULL,
827 ("%s: Unexpected conflicting bucket (%p) while "
828 "adding new bucket (%p)", __func__, tmp_tlb, tlb));
829
830 /*
831 * If we found a conflicting bucket, free the new
832 * one we made and fall through to use the existing
833 * bucket.
834 */
835 if (tmp_tlb != NULL) {
836 FREE_NEW_TLB();
837 INP_WUNLOCK(inp);
838 }
839#undef FREE_NEW_TLB
840 }
841
842 /* If we found an existing bucket, use it. */
843 if (tmp_tlb != NULL) {
844 tlb = tmp_tlb;
845 TCPID_BUCKET_LOCK(tlb);
846 bucket_locked = true;
847
848 /*
849 * Relock the INP and make sure we are still
850 * unassigned.
851 */
852 INP_UNLOCK_ASSERT(inp);
853 INP_WLOCK(inp);
854 RECHECK_INP();
855 if (tp->t_lib != NULL) {
856 TCPID_BUCKET_UNLOCK(tlb);
857 bucket_locked = false;
858 tlb = NULL;
859 goto restart;
860 }
861
862 /* Take a reference on the bucket. */
863 TCPID_BUCKET_REF(tlb);
864
865 /* Record the request. */
866 tcp_log_increment_reqcnt(tlb);
867 }
868
869 tcp_log_grow_tlb(tlb->tlb_id, tp);
870
871 /* Add the new node to the list. */
872 SLIST_INSERT_HEAD(&tlb->tlb_head, tln, tln_list);
873 tp->t_lib = tlb;
874 tp->t_lin = tln;
875 if (tp->t_lib->tlb_logstate) {
876 /* Clone in any logging */
877
878 tp->t_logstate = tp->t_lib->tlb_logstate;
879 }
880 if (tp->t_lib->tlb_loglimit) {
881 /* The loglimit too */
882
883 tp->t_loglimit = tp->t_lib->tlb_loglimit;
884 }
885 tln = NULL;
886 }
887
888 rv = 0;
889
890done:
891 /* Unlock things, as needed, and return. */
892 INP_WUNLOCK(inp);
893done_noinp:
894 INP_UNLOCK_ASSERT(inp);
895 if (bucket_locked) {
896 TCPID_BUCKET_LOCK_ASSERT(tlb);
897 TCPID_BUCKET_UNLOCK(tlb);
898 } else if (tlb != NULL)
899 TCPID_BUCKET_UNLOCK_ASSERT(tlb);
900 if (tree_locked == TREE_WLOCKED) {
901 TCPID_TREE_WLOCK_ASSERT();
902 TCPID_TREE_WUNLOCK();
903 } else if (tree_locked == TREE_RLOCKED) {
904 TCPID_TREE_RLOCK_ASSERT();
905 TCPID_TREE_RUNLOCK();
906 } else
907 TCPID_TREE_UNLOCK_ASSERT();
908 if (tln != NULL)
909 uma_zfree(tcp_log_node_zone, tln);
910 return (rv);
911}
912
913/*
914 * Get the TCP log ID for a TCPCB.
915 * Called with INPCB locked.
916 * 'buf' must point to a buffer that is at least TCP_LOG_ID_LEN bytes long.
917 * Returns number of bytes copied.
918 */
919size_t
920tcp_log_get_id(struct tcpcb *tp, char *buf)
921{
922 size_t len;
923
924 INP_LOCK_ASSERT(tp->t_inpcb);
925 if (tp->t_lib != NULL) {
926 len = strlcpy(buf, tp->t_lib->tlb_id, TCP_LOG_ID_LEN);
927 KASSERT(len < TCP_LOG_ID_LEN,
928 ("%s:%d: tp->t_lib->tlb_id too long (%zu)",
929 __func__, __LINE__, len));
930 } else {
931 *buf = '\0';
932 len = 0;
933 }
934 return (len);
935}
936
937/*
938 * Get the tag associated with the TCPCB's log ID.
939 * Called with INPCB locked. Returns with it unlocked.
940 * 'buf' must point to a buffer that is at least TCP_LOG_TAG_LEN bytes long.
941 * Returns number of bytes copied.
942 */
943size_t
944tcp_log_get_tag(struct tcpcb *tp, char *buf)
945{
946 struct tcp_log_id_bucket *tlb;
947 size_t len;
948 int tree_locked;
949
950 INP_WLOCK_ASSERT(tp->t_inpcb);
951
952 tree_locked = TREE_UNLOCKED;
953 tlb = tp->t_lib;
954
955 if (tlb != NULL) {
956 TCPID_BUCKET_REF(tlb);
957 INP_WUNLOCK(tp->t_inpcb);
958 TCPID_BUCKET_LOCK(tlb);
959 len = strlcpy(buf, tlb->tlb_tag, TCP_LOG_TAG_LEN);
960 KASSERT(len < TCP_LOG_TAG_LEN,
961 ("%s:%d: tp->t_lib->tlb_tag too long (%zu)",
962 __func__, __LINE__, len));
963 if (!tcp_log_unref_bucket(tlb, &tree_locked, NULL))
964 TCPID_BUCKET_UNLOCK(tlb);
965
966 if (tree_locked == TREE_WLOCKED) {
967 TCPID_TREE_WLOCK_ASSERT();
968 TCPID_TREE_WUNLOCK();
969 } else if (tree_locked == TREE_RLOCKED) {
970 TCPID_TREE_RLOCK_ASSERT();
971 TCPID_TREE_RUNLOCK();
972 } else
973 TCPID_TREE_UNLOCK_ASSERT();
974 } else {
975 INP_WUNLOCK(tp->t_inpcb);
976 *buf = '\0';
977 len = 0;
978 }
979
980 return (len);
981}
982
983/*
984 * Get number of connections with the same log ID.
985 * Log ID is taken from given TCPCB.
986 * Called with INPCB locked.
987 */
988u_int
989tcp_log_get_id_cnt(struct tcpcb *tp)
990{
991
992 INP_WLOCK_ASSERT(tp->t_inpcb);
993 return ((tp->t_lib == NULL) ? 0 : tp->t_lib->tlb_refcnt);
994}
995
996#ifdef TCPLOG_DEBUG_RINGBUF
997/*
998 * Functions/macros to increment/decrement reference count for a log
999 * entry. This should catch when we do a double-free/double-remove or
1000 * a double-add.
1001 */
1002static inline void
1003_tcp_log_entry_refcnt_add(struct tcp_log_mem *log_entry, const char *func,
1004 int line)
1005{
1006 int refcnt;
1007
1008 refcnt = atomic_fetchadd_int(&log_entry->tlm_refcnt, 1);
1009 if (refcnt != 0)
1010 panic("%s:%d: log_entry(%p)->tlm_refcnt is %d (expected 0)",
1011 func, line, log_entry, refcnt);
1012}
1013#define tcp_log_entry_refcnt_add(l) \
1014 _tcp_log_entry_refcnt_add((l), __func__, __LINE__)
1015
1016static inline void
1017_tcp_log_entry_refcnt_rem(struct tcp_log_mem *log_entry, const char *func,
1018 int line)
1019{
1020 int refcnt;
1021
1022 refcnt = atomic_fetchadd_int(&log_entry->tlm_refcnt, -1);
1023 if (refcnt != 1)
1024 panic("%s:%d: log_entry(%p)->tlm_refcnt is %d (expected 1)",
1025 func, line, log_entry, refcnt);
1026}
1027#define tcp_log_entry_refcnt_rem(l) \
1028 _tcp_log_entry_refcnt_rem((l), __func__, __LINE__)
1029
1030#else /* !TCPLOG_DEBUG_RINGBUF */
1031
1032#define tcp_log_entry_refcnt_add(l)
1033#define tcp_log_entry_refcnt_rem(l)
1034
1035#endif
1036
1037/*
1038 * Cleanup after removing a log entry, but only decrement the count if we
1039 * are running INVARIANTS.
1040 */
1041static inline void
1042tcp_log_free_log_common(struct tcp_log_mem *log_entry, int *count __unused)
1043{
1044
1045 uma_zfree(tcp_log_zone, log_entry);
1046#ifdef INVARIANTS
1047 (*count)--;
1048 KASSERT(*count >= 0,
1049 ("%s: count unexpectedly negative", __func__));
1050#endif
1051}
1052
1053static void
1054tcp_log_free_entries(struct tcp_log_stailq *head, int *count)
1055{
1056 struct tcp_log_mem *log_entry;
1057
1058 /* Free the entries. */
1059 while ((log_entry = STAILQ_FIRST(head)) != NULL) {
1060 STAILQ_REMOVE_HEAD(head, tlm_queue);
1061 tcp_log_entry_refcnt_rem(log_entry);
1062 tcp_log_free_log_common(log_entry, count);
1063 }
1064}
1065
1066/* Cleanup after removing a log entry. */
1067static inline void
1068tcp_log_remove_log_cleanup(struct tcpcb *tp, struct tcp_log_mem *log_entry)
1069{
1070 uma_zfree(tcp_log_zone, log_entry);
1071 tp->t_lognum--;
1072 KASSERT(tp->t_lognum >= 0,
1073 ("%s: tp->t_lognum unexpectedly negative", __func__));
1074}
1075
1076/* Remove a log entry from the head of a list. */
1077static inline void
1078tcp_log_remove_log_head(struct tcpcb *tp, struct tcp_log_mem *log_entry)
1079{
1080
1081 KASSERT(log_entry == STAILQ_FIRST(&tp->t_logs),
1082 ("%s: attempt to remove non-HEAD log entry", __func__));
1083 STAILQ_REMOVE_HEAD(&tp->t_logs, tlm_queue);
1084 tcp_log_entry_refcnt_rem(log_entry);
1085 tcp_log_remove_log_cleanup(tp, log_entry);
1086}
1087
1088#ifdef TCPLOG_DEBUG_RINGBUF
1089/*
1090 * Initialize the log entry's reference count, which we want to
1091 * survive allocations.
1092 */
1093static int
1094tcp_log_zone_init(void *mem, int size, int flags __unused)
1095{
1096 struct tcp_log_mem *tlm;
1097
1098 KASSERT(size >= sizeof(struct tcp_log_mem),
1099 ("%s: unexpectedly short (%d) allocation", __func__, size));
1100 tlm = (struct tcp_log_mem *)mem;
1101 tlm->tlm_refcnt = 0;
1102 return (0);
1103}
1104
1105/*
1106 * Double check that the refcnt is zero on allocation and return.
1107 */
1108static int
1109tcp_log_zone_ctor(void *mem, int size, void *args __unused, int flags __unused)
1110{
1111 struct tcp_log_mem *tlm;
1112
1113 KASSERT(size >= sizeof(struct tcp_log_mem),
1114 ("%s: unexpectedly short (%d) allocation", __func__, size));
1115 tlm = (struct tcp_log_mem *)mem;
1116 if (tlm->tlm_refcnt != 0)
1117 panic("%s:%d: tlm(%p)->tlm_refcnt is %d (expected 0)",
1118 __func__, __LINE__, tlm, tlm->tlm_refcnt);
1119 return (0);
1120}
1121
1122static void
1123tcp_log_zone_dtor(void *mem, int size, void *args __unused)
1124{
1125 struct tcp_log_mem *tlm;
1126
1127 KASSERT(size >= sizeof(struct tcp_log_mem),
1128 ("%s: unexpectedly short (%d) allocation", __func__, size));
1129 tlm = (struct tcp_log_mem *)mem;
1130 if (tlm->tlm_refcnt != 0)
1131 panic("%s:%d: tlm(%p)->tlm_refcnt is %d (expected 0)",
1132 __func__, __LINE__, tlm, tlm->tlm_refcnt);
1133}
1134#endif /* TCPLOG_DEBUG_RINGBUF */
1135
1136/* Do global initialization. */
1137void
1138tcp_log_init(void)
1139{
1140
1141 tcp_log_zone = uma_zcreate("tcp_log", sizeof(struct tcp_log_mem),
1142#ifdef TCPLOG_DEBUG_RINGBUF
1143 tcp_log_zone_ctor, tcp_log_zone_dtor, tcp_log_zone_init,
1144#else
1145 NULL, NULL, NULL,
1146#endif
1147 NULL, UMA_ALIGN_PTR, 0);
1148 (void)uma_zone_set_max(tcp_log_zone, TCP_LOG_BUF_DEFAULT_GLOBAL_LIMIT);
1149 tcp_log_bucket_zone = uma_zcreate("tcp_log_bucket",
1150 sizeof(struct tcp_log_id_bucket), NULL, NULL, NULL, NULL,
1151 UMA_ALIGN_PTR, 0);
1152 tcp_log_node_zone = uma_zcreate("tcp_log_node",
1153 sizeof(struct tcp_log_id_node), NULL, NULL, NULL, NULL,
1154 UMA_ALIGN_PTR, 0);
1155#ifdef TCPLOG_DEBUG_COUNTERS
1156 tcp_log_queued = counter_u64_alloc(M_WAITOK);
1157 tcp_log_que_fail1 = counter_u64_alloc(M_WAITOK);
1158 tcp_log_que_fail2 = counter_u64_alloc(M_WAITOK);
1159 tcp_log_que_fail3 = counter_u64_alloc(M_WAITOK);
1160 tcp_log_que_fail4 = counter_u64_alloc(M_WAITOK);
1161 tcp_log_que_fail5 = counter_u64_alloc(M_WAITOK);
1162 tcp_log_que_copyout = counter_u64_alloc(M_WAITOK);
1163 tcp_log_que_read = counter_u64_alloc(M_WAITOK);
1164 tcp_log_que_freed = counter_u64_alloc(M_WAITOK);
1165#endif
1166 tcp_log_pcb_ids_cur = counter_u64_alloc(M_WAITOK);
1167 tcp_log_pcb_ids_tot = counter_u64_alloc(M_WAITOK);
1168
1169 rw_init_flags(&tcp_id_tree_lock, "TCP ID tree", RW_NEW);
1170 mtx_init(&tcp_log_expireq_mtx, "TCP log expireq", NULL, MTX_DEF);
1171 callout_init(&tcp_log_expireq_callout, 1);
1172}
1173
1174/* Do per-TCPCB initialization. */
1175void
1176tcp_log_tcpcbinit(struct tcpcb *tp)
1177{
1178
1179 /* A new TCPCB should start out zero-initialized. */
1180 STAILQ_INIT(&tp->t_logs);
1181
1182 /*
1183 * If we are doing auto-capturing, figure out whether we will capture
1184 * this session.
1185 */
1186 tp->t_loglimit = tcp_log_session_limit;
1187 if ((tcp_log_auto_all == true) &&
1188 tcp_log_auto_mode &&
1189 tcp_log_selectauto()) {
1190 tp->t_logstate = tcp_log_auto_mode;
1191 tp->t_flags2 |= TF2_LOG_AUTO;
1192 }
1193}
1194
1195/* Remove entries */
1196static void
1197tcp_log_expire(void *unused __unused)
1198{
1199 struct tcp_log_id_bucket *tlb;
1200 struct tcp_log_id_node *tln;
1201 sbintime_t expiry_limit;
1202 int tree_locked;
1203
1204 TCPLOG_EXPIREQ_LOCK();
1205 if (callout_pending(&tcp_log_expireq_callout)) {
1206 /* Callout was reset. */
1207 TCPLOG_EXPIREQ_UNLOCK();
1208 return;
1209 }
1210
1211 /*
1212 * Process entries until we reach one that expires too far in the
1213 * future. Look one second in the future.
1214 */
1215 expiry_limit = getsbinuptime() + SBT_1S;
1216 tree_locked = TREE_UNLOCKED;
1217
1218 while ((tln = STAILQ_FIRST(&tcp_log_expireq_head)) != NULL &&
1219 tln->tln_expiretime <= expiry_limit) {
1220 if (!callout_active(&tcp_log_expireq_callout)) {
1221 /*
1222 * Callout was stopped. I guess we should
1223 * just quit at this point.
1224 */
1225 TCPLOG_EXPIREQ_UNLOCK();
1226 return;
1227 }
1228
1229 /*
1230 * Remove the node from the head of the list and unlock
1231 * the list. Change the expiry time to SBT_MAX as a signal
1232 * to other threads that we now own this.
1233 */
1234 STAILQ_REMOVE_HEAD(&tcp_log_expireq_head, tln_expireq);
1235 tln->tln_expiretime = SBT_MAX;
1236 TCPLOG_EXPIREQ_UNLOCK();
1237
1238 /*
1239 * Remove the node from the bucket.
1240 */
1241 tlb = tln->tln_bucket;
1242 TCPID_BUCKET_LOCK(tlb);
1243 if (tcp_log_remove_id_node(NULL, NULL, tlb, tln, &tree_locked)) {
1244 tcp_log_id_validate_tree_lock(tree_locked);
1245 if (tree_locked == TREE_WLOCKED)
1246 TCPID_TREE_WUNLOCK();
1247 else
1248 TCPID_TREE_RUNLOCK();
1249 tree_locked = TREE_UNLOCKED;
1250 }
1251
1252 /* Drop the INP reference. */
1253 INP_WLOCK(tln->tln_inp);
1254 if (!in_pcbrele_wlocked(tln->tln_inp))
1255 INP_WUNLOCK(tln->tln_inp);
1256
1257 /* Free the log records. */
1258 tcp_log_free_entries(&tln->tln_entries, &tln->tln_count);
1259
1260 /* Free the node. */
1261 uma_zfree(tcp_log_node_zone, tln);
1262
1263 /* Relock the expiry queue. */
1264 TCPLOG_EXPIREQ_LOCK();
1265 }
1266
1267 /*
1268 * We've expired all the entries we can. Do we need to reschedule
1269 * ourselves?
1270 */
1271 callout_deactivate(&tcp_log_expireq_callout);
1272 if (tln != NULL) {
1273 /*
1274 * Get max(now + TCP_LOG_EXPIRE_INTVL, tln->tln_expiretime) and
1275 * set the next callout to that. (This helps ensure we generally
1276 * run the callout no more often than desired.)
1277 */
1278 expiry_limit = getsbinuptime() + TCP_LOG_EXPIRE_INTVL;
1279 if (expiry_limit < tln->tln_expiretime)
1280 expiry_limit = tln->tln_expiretime;
1281 callout_reset_sbt(&tcp_log_expireq_callout, expiry_limit,
1282 SBT_1S, tcp_log_expire, NULL, C_ABSOLUTE);
1283 }
1284
1285 /* We're done. */
1286 TCPLOG_EXPIREQ_UNLOCK();
1287 return;
1288}
1289
1290/*
1291 * Move log data from the TCPCB to a new node. This will reset the TCPCB log
1292 * entries and log count; however, it will not touch other things from the
1293 * TCPCB (e.g. t_lin, t_lib).
1294 *
1295 * NOTE: Must hold a lock on the INP.
1296 */
1297static void
1298tcp_log_move_tp_to_node(struct tcpcb *tp, struct tcp_log_id_node *tln)
1299{
1300
1301 INP_WLOCK_ASSERT(tp->t_inpcb);
1302
1303 tln->tln_ie = tp->t_inpcb->inp_inc.inc_ie;
1304 if (tp->t_inpcb->inp_inc.inc_flags & INC_ISIPV6)
1305 tln->tln_af = AF_INET6;
1306 else
1307 tln->tln_af = AF_INET;
1308 tln->tln_entries = tp->t_logs;
1309 tln->tln_count = tp->t_lognum;
1310 tln->tln_bucket = tp->t_lib;
1311
1312 /* Clear information from the PCB. */
1313 STAILQ_INIT(&tp->t_logs);
1314 tp->t_lognum = 0;
1315}
1316
1317/* Do per-TCPCB cleanup */
1318void
1319tcp_log_tcpcbfini(struct tcpcb *tp)
1320{
1321 struct tcp_log_id_node *tln, *tln_first;
1322 struct tcp_log_mem *log_entry;
1323 sbintime_t callouttime;
1324
1325 INP_WLOCK_ASSERT(tp->t_inpcb);
1326
1327 TCP_LOG_EVENT(tp, NULL, NULL, NULL, TCP_LOG_CONNEND, 0, 0, NULL, false);
1328
1329 /*
1330 * If we were gathering packets to be automatically dumped, try to do
1331 * it now. If this succeeds, the log information in the TCPCB will be
1332 * cleared. Otherwise, we'll handle the log information as we do
1333 * for other states.
1334 */
1335 switch(tp->t_logstate) {
1336 case TCP_LOG_STATE_HEAD_AUTO:
1337 (void)tcp_log_dump_tp_logbuf(tp, "auto-dumped from head",
1338 M_NOWAIT, false);
1339 break;
1340 case TCP_LOG_STATE_TAIL_AUTO:
1341 (void)tcp_log_dump_tp_logbuf(tp, "auto-dumped from tail",
1342 M_NOWAIT, false);
1343 break;
1344 case TCP_LOG_STATE_CONTINUAL:
1345 (void)tcp_log_dump_tp_logbuf(tp, "auto-dumped from continual",
1346 M_NOWAIT, false);
1347 break;
1348 }
1349
1350 /*
1351 * There are two ways we could keep logs: per-socket or per-ID. If
1352 * we are tracking logs with an ID, then the logs survive the
1353 * destruction of the TCPCB.
1354 *
1355 * If the TCPCB is associated with an ID node, move the logs from the
1356 * TCPCB to the ID node. In theory, this is safe, for reasons which I
1357 * will now explain for my own benefit when I next need to figure out
1358 * this code. :-)
1359 *
1360 * We own the INP lock. Therefore, no one else can change the contents
1361 * of this node (Rule C). Further, no one can remove this node from
1362 * the bucket while we hold the lock (Rule D). Basically, no one can
1363 * mess with this node. That leaves two states in which we could be:
1364 *
1365 * 1. Another thread is currently waiting to acquire the INP lock, with
1366 * plans to do something with this node. When we drop the INP lock,
1367 * they will have a chance to do that. They will recheck the
1368 * tln_closed field (see note to Rule C) and then acquire the
1369 * bucket lock before proceeding further.
1370 *
1371 * 2. Another thread will try to acquire a lock at some point in the
1372 * future. If they try to acquire a lock before we set the
1373 * tln_closed field, they will follow state #1. If they try to
1374 * acquire a lock after we set the tln_closed field, they will be
1375 * able to make changes to the node, at will, following Rule C.
1376 *
1377 * Therefore, we currently own this node and can make any changes
1378 * we want. But, as soon as we set the tln_closed field to true, we
1379 * have effectively dropped our lock on the node. (For this reason, we
1380 * also need to make sure our writes are ordered correctly. An atomic
1381 * operation with "release" semantics should be sufficient.)
1382 */
1383
1384 if (tp->t_lin != NULL) {
1385 /* Copy the relevant information to the log entry. */
1386 tln = tp->t_lin;
1387 KASSERT(tln->tln_inp == tp->t_inpcb,
1388 ("%s: Mismatched inp (tln->tln_inp=%p, tp->t_inpcb=%p)",
1389 __func__, tln->tln_inp, tp->t_inpcb));
1390 tcp_log_move_tp_to_node(tp, tln);
1391
1392 /* Clear information from the PCB. */
1393 tp->t_lin = NULL;
1394 tp->t_lib = NULL;
1395
1396 /*
1397 * Take a reference on the INP. This ensures that the INP
1398 * remains valid while the node is on the expiry queue. This
1399 * ensures the INP is valid for other threads that may be
1400 * racing to lock this node when we move it to the expire
1401 * queue.
1402 */
1403 in_pcbref(tp->t_inpcb);
1404
1405 /*
1406 * Store the entry on the expiry list. The exact behavior
1407 * depends on whether we have entries to keep. If so, we
1408 * put the entry at the tail of the list and expire in
1409 * TCP_LOG_EXPIRE_TIME. Otherwise, we expire "now" and put
1410 * the entry at the head of the list. (Handling the cleanup
1411 * via the expiry timer lets us avoid locking messy-ness here.)
1412 */
1413 tln->tln_expiretime = getsbinuptime();
1414 TCPLOG_EXPIREQ_LOCK();
1415 if (tln->tln_count) {
1416 tln->tln_expiretime += TCP_LOG_EXPIRE_TIME;
1417 if (STAILQ_EMPTY(&tcp_log_expireq_head) &&
1418 !callout_active(&tcp_log_expireq_callout)) {
1419 /*
1420 * We are adding the first entry and a callout
1421 * is not currently scheduled; therefore, we
1422 * need to schedule one.
1423 */
1424 callout_reset_sbt(&tcp_log_expireq_callout,
1425 tln->tln_expiretime, SBT_1S, tcp_log_expire,
1426 NULL, C_ABSOLUTE);
1427 }
1428 STAILQ_INSERT_TAIL(&tcp_log_expireq_head, tln,
1429 tln_expireq);
1430 } else {
1431 callouttime = tln->tln_expiretime +
1432 TCP_LOG_EXPIRE_INTVL;
1433 tln_first = STAILQ_FIRST(&tcp_log_expireq_head);
1434
1435 if ((tln_first == NULL ||
1436 callouttime < tln_first->tln_expiretime) &&
1437 (callout_pending(&tcp_log_expireq_callout) ||
1438 !callout_active(&tcp_log_expireq_callout))) {
1439 /*
1440 * The list is empty, or we want to run the
1441 * expire code before the first entry's timer
1442 * fires. Also, we are in a case where a callout
1443 * is not actively running. We want to reset
1444 * the callout to occur sooner.
1445 */
1446 callout_reset_sbt(&tcp_log_expireq_callout,
1447 callouttime, SBT_1S, tcp_log_expire, NULL,
1448 C_ABSOLUTE);
1449 }
1450
1451 /*
1452 * Insert to the head, or just after the head, as
1453 * appropriate. (This might result in small
1454 * mis-orderings as a bunch of "expire now" entries
1455 * gather at the start of the list, but that should
1456 * not produce big problems, since the expire timer
1457 * will walk through all of them.)
1458 */
1459 if (tln_first == NULL ||
1460 tln->tln_expiretime < tln_first->tln_expiretime)
1461 STAILQ_INSERT_HEAD(&tcp_log_expireq_head, tln,
1462 tln_expireq);
1463 else
1464 STAILQ_INSERT_AFTER(&tcp_log_expireq_head,
1465 tln_first, tln, tln_expireq);
1466 }
1467 TCPLOG_EXPIREQ_UNLOCK();
1468
1469 /*
1470 * We are done messing with the tln. After this point, we
1471 * can't touch it. (Note that the "release" semantics should
1472 * be included with the TCPLOG_EXPIREQ_UNLOCK() call above.
1473 * Therefore, they should be unnecessary here. However, it
1474 * seems like a good idea to include them anyway, since we
1475 * really are releasing a lock here.)
1476 */
1477 atomic_store_rel_int(&tln->tln_closed, 1);
1478 } else {
1479 /* Remove log entries. */
1480 while ((log_entry = STAILQ_FIRST(&tp->t_logs)) != NULL)
1481 tcp_log_remove_log_head(tp, log_entry);
1482 KASSERT(tp->t_lognum == 0,
1483 ("%s: After freeing entries, tp->t_lognum=%d (expected 0)",
1484 __func__, tp->t_lognum));
1485 }
1486
1487 /*
1488 * Change the log state to off (just in case anything tries to sneak
1489 * in a last-minute log).
1490 */
1491 tp->t_logstate = TCP_LOG_STATE_OFF;
1492}
1493
1494static void
1495tcp_log_purge_tp_logbuf(struct tcpcb *tp)
1496{
1497 struct tcp_log_mem *log_entry;
1498 struct inpcb *inp;
1499
1500 inp = tp->t_inpcb;
1501 INP_WLOCK_ASSERT(inp);
1502 if (tp->t_lognum == 0)
1503 return;
1504
1505 while ((log_entry = STAILQ_FIRST(&tp->t_logs)) != NULL)
1506 tcp_log_remove_log_head(tp, log_entry);
1507 KASSERT(tp->t_lognum == 0,
1508 ("%s: After freeing entries, tp->t_lognum=%d (expected 0)",
1509 __func__, tp->t_lognum));
1510 tp->t_logstate = TCP_LOG_STATE_OFF;
1511}
1512
1513/*
1514 * This logs an event for a TCP socket. Normally, this is called via
1515 * TCP_LOG_EVENT or TCP_LOG_EVENT_VERBOSE. See the documentation for
1516 * TCP_LOG_EVENT().
1517 */
1518
1519struct tcp_log_buffer *
1520tcp_log_event_(struct tcpcb *tp, struct tcphdr *th, struct sockbuf *rxbuf,
1521 struct sockbuf *txbuf, uint8_t eventid, int errornum, uint32_t len,
1522 union tcp_log_stackspecific *stackinfo, int th_hostorder,
1523 const char *output_caller, const char *func, int line, const struct timeval *itv)
1524{
1525 struct tcp_log_mem *log_entry;
1526 struct tcp_log_buffer *log_buf;
1527 int attempt_count = 0;
1528 struct tcp_log_verbose *log_verbose;
1529 uint32_t logsn;
1530
1531 KASSERT((func == NULL && line == 0) || (func != NULL && line > 0),
1532 ("%s called with inconsistent func (%p) and line (%d) arguments",
1533 __func__, func, line));
1534
1535 INP_WLOCK_ASSERT(tp->t_inpcb);
1536 if (tcp_disable_all_bb_logs) {
1537 /*
1538 * The global shutdown logging
1539 * switch has been thrown. Call
1540 * the purge function that frees
1541 * purges out the logs and
1542 * turns off logging.
1543 */
1544 tcp_log_purge_tp_logbuf(tp);
1545 return (NULL);
1546 }
1547 KASSERT(tp->t_logstate == TCP_LOG_STATE_HEAD ||
1548 tp->t_logstate == TCP_LOG_STATE_TAIL ||
1549 tp->t_logstate == TCP_LOG_STATE_CONTINUAL ||
1550 tp->t_logstate == TCP_LOG_STATE_HEAD_AUTO ||
1551 tp->t_logstate == TCP_LOG_STATE_TAIL_AUTO,
1552 ("%s called with unexpected tp->t_logstate (%d)", __func__,
1553 tp->t_logstate));
1554
1555 /*
1556 * Get the serial number. We do this early so it will
1557 * increment even if we end up skipping the log entry for some
1558 * reason.
1559 */
1560 logsn = tp->t_logsn++;
1561
1562 /*
1563 * Can we get a new log entry? If so, increment the lognum counter
1564 * here.
1565 */
1566retry:
1567 if (tp->t_lognum < tp->t_loglimit) {
1568 if ((log_entry = uma_zalloc(tcp_log_zone, M_NOWAIT)) != NULL)
1569 tp->t_lognum++;
1570 } else
1571 log_entry = NULL;
1572
1573 /* Do we need to try to reuse? */
1574 if (log_entry == NULL) {
1575 /*
1576 * Sacrifice auto-logged sessions without a log ID if
1577 * tcp_log_auto_all is false. (If they don't have a log
1578 * ID by now, it is probable that either they won't get one
1579 * or we are resource-constrained.)
1580 */
1581 if (tp->t_lib == NULL && (tp->t_flags2 & TF2_LOG_AUTO) &&
1582 !tcp_log_auto_all) {
1583 if (tcp_log_state_change(tp, TCP_LOG_STATE_CLEAR)) {
1584#ifdef INVARIANTS
1585 panic("%s:%d: tcp_log_state_change() failed "
1586 "to set tp %p to TCP_LOG_STATE_CLEAR",
1587 __func__, __LINE__, tp);
1588#endif
1589 tp->t_logstate = TCP_LOG_STATE_OFF;
1590 }
1591 return (NULL);
1592 }
1593 /*
1594 * If we are in TCP_LOG_STATE_HEAD_AUTO state, try to dump
1595 * the buffers. If successful, deactivate tracing. Otherwise,
1596 * leave it active so we will retry.
1597 */
1598 if (tp->t_logstate == TCP_LOG_STATE_HEAD_AUTO &&
1599 !tcp_log_dump_tp_logbuf(tp, "auto-dumped from head",
1600 M_NOWAIT, false)) {
1601 tp->t_logstate = TCP_LOG_STATE_OFF;
1602 return(NULL);
1603 } else if ((tp->t_logstate == TCP_LOG_STATE_CONTINUAL) &&
1604 !tcp_log_dump_tp_logbuf(tp, "auto-dumped from continual",
1605 M_NOWAIT, false)) {
1606 if (attempt_count == 0) {
1607 attempt_count++;
1608 goto retry;
1609 }
1610#ifdef TCPLOG_DEBUG_COUNTERS
1611 counter_u64_add(tcp_log_que_fail4, 1);
1612#endif
1613 return(NULL);
1614 } else if (tp->t_logstate == TCP_LOG_STATE_HEAD_AUTO)
1615 return(NULL);
1616
1617 /* If in HEAD state, just deactivate the tracing and return. */
1618 if (tp->t_logstate == TCP_LOG_STATE_HEAD) {
1619 tp->t_logstate = TCP_LOG_STATE_OFF;
1620 return(NULL);
1621 }
1622
1623 /*
1624 * Get a buffer to reuse. If that fails, just give up.
1625 * (We can't log anything without a buffer in which to
1626 * put it.)
1627 *
1628 * Note that we don't change the t_lognum counter
1629 * here. Because we are re-using the buffer, the total
1630 * number won't change.
1631 */
1632 if ((log_entry = STAILQ_FIRST(&tp->t_logs)) == NULL)
1633 return(NULL);
1634 STAILQ_REMOVE_HEAD(&tp->t_logs, tlm_queue);
1635 tcp_log_entry_refcnt_rem(log_entry);
1636 }
1637
1638 KASSERT(log_entry != NULL,
1639 ("%s: log_entry unexpectedly NULL", __func__));
1640
1641 /* Extract the log buffer and verbose buffer pointers. */
1642 log_buf = &log_entry->tlm_buf;
1643 log_verbose = &log_entry->tlm_v;
1644
1645 /* Basic entries. */
1646 if (itv == NULL)
1647 getmicrouptime(&log_buf->tlb_tv);
1648 else
1649 memcpy(&log_buf->tlb_tv, itv, sizeof(struct timeval));
1650 log_buf->tlb_ticks = ticks;
1651 log_buf->tlb_sn = logsn;
1652 log_buf->tlb_stackid = tp->t_fb->tfb_id;
1653 log_buf->tlb_eventid = eventid;
1654 log_buf->tlb_eventflags = 0;
1655 log_buf->tlb_errno = errornum;
1656
1657 /* Socket buffers */
1658 if (rxbuf != NULL) {
1659 log_buf->tlb_eventflags |= TLB_FLAG_RXBUF;
1660 log_buf->tlb_rxbuf.tls_sb_acc = rxbuf->sb_acc;
1661 log_buf->tlb_rxbuf.tls_sb_ccc = rxbuf->sb_ccc;
1662 log_buf->tlb_rxbuf.tls_sb_spare = 0;
1663 }
1664 if (txbuf != NULL) {
1665 log_buf->tlb_eventflags |= TLB_FLAG_TXBUF;
1666 log_buf->tlb_txbuf.tls_sb_acc = txbuf->sb_acc;
1667 log_buf->tlb_txbuf.tls_sb_ccc = txbuf->sb_ccc;
1668 log_buf->tlb_txbuf.tls_sb_spare = 0;
1669 }
1670 /* Copy values from tp to the log entry. */
1671#define COPY_STAT(f) log_buf->tlb_ ## f = tp->f
1672#define COPY_STAT_T(f) log_buf->tlb_ ## f = tp->t_ ## f
1673 COPY_STAT_T(state);
1674 COPY_STAT_T(starttime);
1675 COPY_STAT(iss);
1676 COPY_STAT_T(flags);
1677 COPY_STAT(snd_una);
1678 COPY_STAT(snd_max);
1679 COPY_STAT(snd_cwnd);
1680 COPY_STAT(snd_nxt);
1681 COPY_STAT(snd_recover);
1682 COPY_STAT(snd_wnd);
1683 COPY_STAT(snd_ssthresh);
1684 COPY_STAT_T(srtt);
1685 COPY_STAT_T(rttvar);
1686 COPY_STAT(rcv_up);
1687 COPY_STAT(rcv_adv);
1688 COPY_STAT(rcv_nxt);
1689 COPY_STAT(rcv_wnd);
1690 COPY_STAT_T(dupacks);
1691 COPY_STAT_T(segqlen);
1692 COPY_STAT(snd_numholes);
1693 COPY_STAT(snd_scale);
1694 COPY_STAT(rcv_scale);
1695 COPY_STAT_T(flags2);
1696 COPY_STAT_T(fbyte_in);
1697 COPY_STAT_T(fbyte_out);
1698#undef COPY_STAT
1699#undef COPY_STAT_T
1700 log_buf->tlb_flex1 = 0;
1701 log_buf->tlb_flex2 = 0;
1702 /* Copy stack-specific info. */
1703 if (stackinfo != NULL) {
1704 memcpy(&log_buf->tlb_stackinfo, stackinfo,
1705 sizeof(log_buf->tlb_stackinfo));
1706 log_buf->tlb_eventflags |= TLB_FLAG_STACKINFO;
1707 }
1708
1709 /* The packet */
1710 log_buf->tlb_len = len;
1711 if (th) {
1712 int optlen;
1713
1714 log_buf->tlb_eventflags |= TLB_FLAG_HDR;
1715 log_buf->tlb_th = *th;
1716 if (th_hostorder)
1717 tcp_fields_to_net(&log_buf->tlb_th);
1718 optlen = (th->th_off << 2) - sizeof (struct tcphdr);
1719 if (optlen > 0)
1720 memcpy(log_buf->tlb_opts, th + 1, optlen);
1721 }
1722
1723 /* Verbose information */
1724 if (func != NULL) {
1725 log_buf->tlb_eventflags |= TLB_FLAG_VERBOSE;
1726 if (output_caller != NULL)
1727 strlcpy(log_verbose->tlv_snd_frm, output_caller,
1728 TCP_FUNC_LEN);
1729 else
1730 *log_verbose->tlv_snd_frm = 0;
1731 strlcpy(log_verbose->tlv_trace_func, func, TCP_FUNC_LEN);
1732 log_verbose->tlv_trace_line = line;
1733 }
1734
1735 /* Insert the new log at the tail. */
1736 STAILQ_INSERT_TAIL(&tp->t_logs, log_entry, tlm_queue);
1737 tcp_log_entry_refcnt_add(log_entry);
1738 return (log_buf);
1739}
1740
1741/*
1742 * Change the logging state for a TCPCB. Returns 0 on success or an
1743 * error code on failure.
1744 */
1745int
1746tcp_log_state_change(struct tcpcb *tp, int state)
1747{
1748 struct tcp_log_mem *log_entry;
1749
1750 INP_WLOCK_ASSERT(tp->t_inpcb);
1751 switch(state) {
1752 case TCP_LOG_STATE_CLEAR:
1753 while ((log_entry = STAILQ_FIRST(&tp->t_logs)) != NULL)
1754 tcp_log_remove_log_head(tp, log_entry);
1755 /* Fall through */
1756
1757 case TCP_LOG_STATE_OFF:
1758 tp->t_logstate = TCP_LOG_STATE_OFF;
1759 break;
1760
1761 case TCP_LOG_STATE_TAIL:
1762 case TCP_LOG_STATE_HEAD:
1763 case TCP_LOG_STATE_CONTINUAL:
1764 case TCP_LOG_STATE_HEAD_AUTO:
1765 case TCP_LOG_STATE_TAIL_AUTO:
1766 tp->t_logstate = state;
1767 break;
1768
1769 default:
1770 return (EINVAL);
1771 }
1772 if (tcp_disable_all_bb_logs) {
1773 /* We are prohibited from doing any logs */
1774 tp->t_logstate = TCP_LOG_STATE_OFF;
1775 }
1776 tp->t_flags2 &= ~(TF2_LOG_AUTO);
1777
1778 return (0);
1779}
1780
1781/* If tcp_drain() is called, flush half the log entries. */
1782void
1783tcp_log_drain(struct tcpcb *tp)
1784{
1785 struct tcp_log_mem *log_entry, *next;
1786 int target, skip;
1787
1788 INP_WLOCK_ASSERT(tp->t_inpcb);
1789 if ((target = tp->t_lognum / 2) == 0)
1790 return;
1791
1792 /*
1793 * If we are logging the "head" packets, we want to discard
1794 * from the tail of the queue. Otherwise, we want to discard
1795 * from the head.
1796 */
1797 if (tp->t_logstate == TCP_LOG_STATE_HEAD ||
1798 tp->t_logstate == TCP_LOG_STATE_HEAD_AUTO) {
1799 skip = tp->t_lognum - target;
1800 STAILQ_FOREACH(log_entry, &tp->t_logs, tlm_queue)
1801 if (!--skip)
1802 break;
1803 KASSERT(log_entry != NULL,
1804 ("%s: skipped through all entries!", __func__));
1805 if (log_entry == NULL)
1806 return;
1807 while ((next = STAILQ_NEXT(log_entry, tlm_queue)) != NULL) {
1808 STAILQ_REMOVE_AFTER(&tp->t_logs, log_entry, tlm_queue);
1809 tcp_log_entry_refcnt_rem(next);
1810 tcp_log_remove_log_cleanup(tp, next);
1811#ifdef INVARIANTS
1812 target--;
1813#endif
1814 }
1815 KASSERT(target == 0,
1816 ("%s: After removing from tail, target was %d", __func__,
1817 target));
1818 } else if (tp->t_logstate == TCP_LOG_STATE_CONTINUAL) {
1819 (void)tcp_log_dump_tp_logbuf(tp, "auto-dumped from continual",
1820 M_NOWAIT, false);
1821 } else {
1822 while ((log_entry = STAILQ_FIRST(&tp->t_logs)) != NULL &&
1823 target--)
1824 tcp_log_remove_log_head(tp, log_entry);
1825 KASSERT(target <= 0,
1826 ("%s: After removing from head, target was %d", __func__,
1827 target));
1828 KASSERT(tp->t_lognum > 0,
1829 ("%s: After removing from head, tp->t_lognum was %d",
1830 __func__, target));
1831 KASSERT(log_entry != NULL,
1832 ("%s: After removing from head, the tailq was empty",
1833 __func__));
1834 }
1835}
1836
1837static inline int
1838tcp_log_copyout(struct sockopt *sopt, void *src, void *dst, size_t len)
1839{
1840
1841 if (sopt->sopt_td != NULL)
1842 return (copyout(src, dst, len));
1843 bcopy(src, dst, len);
1844 return (0);
1845}
1846
1847static int
1848tcp_log_logs_to_buf(struct sockopt *sopt, struct tcp_log_stailq *log_tailqp,
1849 struct tcp_log_buffer **end, int count)
1850{
1851 struct tcp_log_buffer *out_entry;
1852 struct tcp_log_mem *log_entry;
1853 size_t entrysize;
1854 int error;
1855#ifdef INVARIANTS
1856 int orig_count = count;
1857#endif
1858
1859 /* Copy the data out. */
1860 error = 0;
1861 out_entry = (struct tcp_log_buffer *) sopt->sopt_val;
1862 STAILQ_FOREACH(log_entry, log_tailqp, tlm_queue) {
1863 count--;
1864 KASSERT(count >= 0,
1865 ("%s:%d: Exceeded expected count (%d) processing list %p",
1866 __func__, __LINE__, orig_count, log_tailqp));
1867
1868#ifdef TCPLOG_DEBUG_COUNTERS
1869 counter_u64_add(tcp_log_que_copyout, 1);
1870#endif
1871
1872 /*
1873 * Skip copying out the header if it isn't present.
1874 * Instead, copy out zeros (to ensure we don't leak info).
1875 * TODO: Make sure we truly do zero everything we don't
1876 * explicitly set.
1877 */
1878 if (log_entry->tlm_buf.tlb_eventflags & TLB_FLAG_HDR)
1879 entrysize = sizeof(struct tcp_log_buffer);
1880 else
1881 entrysize = offsetof(struct tcp_log_buffer, tlb_th);
1882 error = tcp_log_copyout(sopt, &log_entry->tlm_buf, out_entry,
1883 entrysize);
1884 if (error)
1885 break;
1886 if (!(log_entry->tlm_buf.tlb_eventflags & TLB_FLAG_HDR)) {
1887 error = tcp_log_copyout(sopt, zerobuf,
1888 ((uint8_t *)out_entry) + entrysize,
1889 sizeof(struct tcp_log_buffer) - entrysize);
1890 }
1891
1892 /*
1893 * Copy out the verbose bit, if needed. Either way,
1894 * increment the output pointer the correct amount.
1895 */
1896 if (log_entry->tlm_buf.tlb_eventflags & TLB_FLAG_VERBOSE) {
1897 error = tcp_log_copyout(sopt, &log_entry->tlm_v,
1898 out_entry->tlb_verbose,
1899 sizeof(struct tcp_log_verbose));
1900 if (error)
1901 break;
1902 out_entry = (struct tcp_log_buffer *)
1903 (((uint8_t *) (out_entry + 1)) +
1904 sizeof(struct tcp_log_verbose));
1905 } else
1906 out_entry++;
1907 }
1908 *end = out_entry;
1909 KASSERT(error || count == 0,
1910 ("%s:%d: Less than expected count (%d) processing list %p"
1911 " (%d remain)", __func__, __LINE__, orig_count,
1912 log_tailqp, count));
1913
1914 return (error);
1915}
1916
1917/*
1918 * Copy out the buffer. Note that we do incremental copying, so
1919 * sooptcopyout() won't work. However, the goal is to produce the same
1920 * end result as if we copied in the entire user buffer, updated it,
1921 * and then used sooptcopyout() to copy it out.
1922 *
1923 * NOTE: This should be called with a write lock on the PCB; however,
1924 * the function will drop it after it extracts the data from the TCPCB.
1925 */
1926int
1927tcp_log_getlogbuf(struct sockopt *sopt, struct tcpcb *tp)
1928{
1929 struct tcp_log_stailq log_tailq;
1930 struct tcp_log_mem *log_entry, *log_next;
1931 struct tcp_log_buffer *out_entry;
1932 struct inpcb *inp;
1933 size_t outsize, entrysize;
1934 int error, outnum;
1935
1936 INP_WLOCK_ASSERT(tp->t_inpcb);
1937 inp = tp->t_inpcb;
1938
1939 /*
1940 * Determine which log entries will fit in the buffer. As an
1941 * optimization, skip this if all the entries will clearly fit
1942 * in the buffer. (However, get an exact size if we are using
1943 * INVARIANTS.)
1944 */
1945#ifndef INVARIANTS
1946 if (sopt->sopt_valsize / (sizeof(struct tcp_log_buffer) +
1947 sizeof(struct tcp_log_verbose)) >= tp->t_lognum) {
1948 log_entry = STAILQ_LAST(&tp->t_logs, tcp_log_mem, tlm_queue);
1949 log_next = NULL;
1950 outsize = 0;
1951 outnum = tp->t_lognum;
1952 } else {
1953#endif
1954 outsize = outnum = 0;
1955 log_entry = NULL;
1956 STAILQ_FOREACH(log_next, &tp->t_logs, tlm_queue) {
1957 entrysize = sizeof(struct tcp_log_buffer);
1958 if (log_next->tlm_buf.tlb_eventflags &
1959 TLB_FLAG_VERBOSE)
1960 entrysize += sizeof(struct tcp_log_verbose);
1961 if ((sopt->sopt_valsize - outsize) < entrysize)
1962 break;
1963 outsize += entrysize;
1964 outnum++;
1965 log_entry = log_next;
1966 }
1967 KASSERT(outsize <= sopt->sopt_valsize,
1968 ("%s: calculated output size (%zu) greater than available"
1969 "space (%zu)", __func__, outsize, sopt->sopt_valsize));
1970#ifndef INVARIANTS
1971 }
1972#endif
1973
1974 /*
1975 * Copy traditional sooptcopyout() behavior: if sopt->sopt_val
1976 * is NULL, silently skip the copy. However, in this case, we
1977 * will leave the list alone and return. Functionally, this
1978 * gives userspace a way to poll for an approximate buffer
1979 * size they will need to get the log entries.
1980 */
1981 if (sopt->sopt_val == NULL) {
1982 INP_WUNLOCK(inp);
1983 if (outsize == 0) {
1984 outsize = outnum * (sizeof(struct tcp_log_buffer) +
1985 sizeof(struct tcp_log_verbose));
1986 }
1987 if (sopt->sopt_valsize > outsize)
1988 sopt->sopt_valsize = outsize;
1989 return (0);
1990 }
1991
1992 /*
1993 * Break apart the list. We'll save the ones we want to copy
1994 * out locally and remove them from the TCPCB list. We can
1995 * then drop the INPCB lock while we do the copyout.
1996 *
1997 * There are roughly three cases:
1998 * 1. There was nothing to copy out. That's easy: drop the
1999 * lock and return.
2000 * 2. We are copying out the entire list. Again, that's easy:
2001 * move the whole list.
2002 * 3. We are copying out a partial list. That's harder. We
2003 * need to update the list book-keeping entries.
2004 */
2005 if (log_entry != NULL && log_next == NULL) {
2006 /* Move entire list. */
2007 KASSERT(outnum == tp->t_lognum,
2008 ("%s:%d: outnum (%d) should match tp->t_lognum (%d)",
2009 __func__, __LINE__, outnum, tp->t_lognum));
2010 log_tailq = tp->t_logs;
2011 tp->t_lognum = 0;
2012 STAILQ_INIT(&tp->t_logs);
2013 } else if (log_entry != NULL) {
2014 /* Move partial list. */
2015 KASSERT(outnum < tp->t_lognum,
2016 ("%s:%d: outnum (%d) not less than tp->t_lognum (%d)",
2017 __func__, __LINE__, outnum, tp->t_lognum));
2018 STAILQ_FIRST(&log_tailq) = STAILQ_FIRST(&tp->t_logs);
2019 STAILQ_FIRST(&tp->t_logs) = STAILQ_NEXT(log_entry, tlm_queue);
2020 KASSERT(STAILQ_NEXT(log_entry, tlm_queue) != NULL,
2021 ("%s:%d: tp->t_logs is unexpectedly shorter than expected"
2022 "(tp: %p, log_tailq: %p, outnum: %d, tp->t_lognum: %d)",
2023 __func__, __LINE__, tp, &log_tailq, outnum, tp->t_lognum));
2024 STAILQ_NEXT(log_entry, tlm_queue) = NULL;
2025 log_tailq.stqh_last = &STAILQ_NEXT(log_entry, tlm_queue);
2026 tp->t_lognum -= outnum;
2027 } else
2028 STAILQ_INIT(&log_tailq);
2029
2030 /* Drop the PCB lock. */
2031 INP_WUNLOCK(inp);
2032
2033 /* Copy the data out. */
2034 error = tcp_log_logs_to_buf(sopt, &log_tailq, &out_entry, outnum);
2035
2036 if (error) {
2037 /* Restore list */
2038 INP_WLOCK(inp);
2039 if ((inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) == 0) {
2040 tp = intotcpcb(inp);
2041
2042 /* Merge the two lists. */
2043 STAILQ_CONCAT(&log_tailq, &tp->t_logs);
2044 tp->t_logs = log_tailq;
2045 tp->t_lognum += outnum;
2046 }
2047 INP_WUNLOCK(inp);
2048 } else {
2049 /* Sanity check entries */
2050 KASSERT(((caddr_t)out_entry - (caddr_t)sopt->sopt_val) ==
2051 outsize, ("%s: Actual output size (%zu) != "
2052 "calculated output size (%zu)", __func__,
2053 (size_t)((caddr_t)out_entry - (caddr_t)sopt->sopt_val),
2054 outsize));
2055
2056 /* Free the entries we just copied out. */
2057 STAILQ_FOREACH_SAFE(log_entry, &log_tailq, tlm_queue, log_next) {
2058 tcp_log_entry_refcnt_rem(log_entry);
2059 uma_zfree(tcp_log_zone, log_entry);
2060 }
2061 }
2062
2063 sopt->sopt_valsize = (size_t)((caddr_t)out_entry -
2064 (caddr_t)sopt->sopt_val);
2065 return (error);
2066}
2067
2068static void
2069tcp_log_free_queue(struct tcp_log_dev_queue *param)
2070{
2071 struct tcp_log_dev_log_queue *entry;
2072
2073 KASSERT(param != NULL, ("%s: called with NULL param", __func__));
2074 if (param == NULL)
2075 return;
2076
2077 entry = (struct tcp_log_dev_log_queue *)param;
2078
2079 /* Free the entries. */
2080 tcp_log_free_entries(&entry->tldl_entries, &entry->tldl_count);
2081
2082 /* Free the buffer, if it is allocated. */
2083 if (entry->tldl_common.tldq_buf != NULL)
2084 free(entry->tldl_common.tldq_buf, M_TCPLOGDEV);
2085
2086 /* Free the queue entry. */
2087 free(entry, M_TCPLOGDEV);
2088}
2089
2090static struct tcp_log_common_header *
2091tcp_log_expandlogbuf(struct tcp_log_dev_queue *param)
2092{
2093 struct tcp_log_dev_log_queue *entry;
2094 struct tcp_log_header *hdr;
2095 uint8_t *end;
2096 struct sockopt sopt;
2097 int error;
2098
2099 entry = (struct tcp_log_dev_log_queue *)param;
2100
2101 /* Take a worst-case guess at space needs. */
2102 sopt.sopt_valsize = sizeof(struct tcp_log_header) +
2103 entry->tldl_count * (sizeof(struct tcp_log_buffer) +
2104 sizeof(struct tcp_log_verbose));
2105 hdr = malloc(sopt.sopt_valsize, M_TCPLOGDEV, M_NOWAIT);
2106 if (hdr == NULL) {
2107#ifdef TCPLOG_DEBUG_COUNTERS
2108 counter_u64_add(tcp_log_que_fail5, entry->tldl_count);
2109#endif
2110 return (NULL);
2111 }
2112 sopt.sopt_val = hdr + 1;
2113 sopt.sopt_valsize -= sizeof(struct tcp_log_header);
2114 sopt.sopt_td = NULL;
2115
2116 error = tcp_log_logs_to_buf(&sopt, &entry->tldl_entries,
2117 (struct tcp_log_buffer **)&end, entry->tldl_count);
2118 if (error) {
2119 free(hdr, M_TCPLOGDEV);
2120 return (NULL);
2121 }
2122
2123 /* Free the entries. */
2124 tcp_log_free_entries(&entry->tldl_entries, &entry->tldl_count);
2125 entry->tldl_count = 0;
2126
2127 memset(hdr, 0, sizeof(struct tcp_log_header));
2128 hdr->tlh_version = TCP_LOG_BUF_VER;
2129 hdr->tlh_type = TCP_LOG_DEV_TYPE_BBR;
2130 hdr->tlh_length = end - (uint8_t *)hdr;
2131 hdr->tlh_ie = entry->tldl_ie;
2132 hdr->tlh_af = entry->tldl_af;
2133 getboottime(&hdr->tlh_offset);
2134 strlcpy(hdr->tlh_id, entry->tldl_id, TCP_LOG_ID_LEN);
2135 strlcpy(hdr->tlh_tag, entry->tldl_tag, TCP_LOG_TAG_LEN);
2136 strlcpy(hdr->tlh_reason, entry->tldl_reason, TCP_LOG_REASON_LEN);
2137 return ((struct tcp_log_common_header *)hdr);
2138}
2139
2140/*
2141 * Queue the tcpcb's log buffer for transmission via the log buffer facility.
2142 *
2143 * NOTE: This should be called with a write lock on the PCB.
2144 *
2145 * how should be M_WAITOK or M_NOWAIT. If M_WAITOK, the function will drop
2146 * and reacquire the INP lock if it needs to do so.
2147 *
2148 * If force is false, this will only dump auto-logged sessions if
2149 * tcp_log_auto_all is true or if there is a log ID defined for the session.
2150 */
2151int
2152tcp_log_dump_tp_logbuf(struct tcpcb *tp, char *reason, int how, bool force)
2153{
2154 struct tcp_log_dev_log_queue *entry;
2155 struct inpcb *inp;
2156#ifdef TCPLOG_DEBUG_COUNTERS
2157 int num_entries;
2158#endif
2159
2160 inp = tp->t_inpcb;
2161 INP_WLOCK_ASSERT(inp);
2162
2163 /* If there are no log entries, there is nothing to do. */
2164 if (tp->t_lognum == 0)
2165 return (0);
2166
2167 /* Check for a log ID. */
2168 if (tp->t_lib == NULL && (tp->t_flags2 & TF2_LOG_AUTO) &&
2169 !tcp_log_auto_all && !force) {
2170 struct tcp_log_mem *log_entry;
2171
2172 /*
2173 * We needed a log ID and none was found. Free the log entries
2174 * and return success. Also, cancel further logging. If the
2175 * session doesn't have a log ID by now, we'll assume it isn't
2176 * going to get one.
2177 */
2178 while ((log_entry = STAILQ_FIRST(&tp->t_logs)) != NULL)
2179 tcp_log_remove_log_head(tp, log_entry);
2180 KASSERT(tp->t_lognum == 0,
2181 ("%s: After freeing entries, tp->t_lognum=%d (expected 0)",
2182 __func__, tp->t_lognum));
2183 tp->t_logstate = TCP_LOG_STATE_OFF;
2184 return (0);
2185 }
2186
2187 /*
2188 * Allocate memory. If we must wait, we'll need to drop the locks
2189 * and reacquire them (and do all the related business that goes
2190 * along with that).
2191 */
2192 entry = malloc(sizeof(struct tcp_log_dev_log_queue), M_TCPLOGDEV,
2193 M_NOWAIT);
2194 if (entry == NULL && (how & M_NOWAIT)) {
2195#ifdef TCPLOG_DEBUG_COUNTERS
2196 counter_u64_add(tcp_log_que_fail3, 1);
2197#endif
2198 return (ENOBUFS);
2199 }
2200 if (entry == NULL) {
2201 INP_WUNLOCK(inp);
2202 entry = malloc(sizeof(struct tcp_log_dev_log_queue),
2203 M_TCPLOGDEV, M_WAITOK);
2204 INP_WLOCK(inp);
2205 /*
2206 * Note that this check is slightly overly-restrictive in
2207 * that the TCB can survive either of these events.
2208 * However, there is currently not a good way to ensure
2209 * that is the case. So, if we hit this M_WAIT path, we
2210 * may end up dropping some entries. That seems like a
2211 * small price to pay for safety.
2212 */
2213 if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) {
2214 free(entry, M_TCPLOGDEV);
2215#ifdef TCPLOG_DEBUG_COUNTERS
2216 counter_u64_add(tcp_log_que_fail2, 1);
2217#endif
2218 return (ECONNRESET);
2219 }
2220 tp = intotcpcb(inp);
2221 if (tp->t_lognum == 0) {
2222 free(entry, M_TCPLOGDEV);
2223 return (0);
2224 }
2225 }
2226
2227 /* Fill in the unique parts of the queue entry. */
2228 if (tp->t_lib != NULL) {
2229 strlcpy(entry->tldl_id, tp->t_lib->tlb_id, TCP_LOG_ID_LEN);
2230 strlcpy(entry->tldl_tag, tp->t_lib->tlb_tag, TCP_LOG_TAG_LEN);
2231 } else {
2232 strlcpy(entry->tldl_id, "UNKNOWN", TCP_LOG_ID_LEN);
2233 strlcpy(entry->tldl_tag, "UNKNOWN", TCP_LOG_TAG_LEN);
2234 }
2235 if (reason != NULL)
2236 strlcpy(entry->tldl_reason, reason, TCP_LOG_REASON_LEN);
2237 else
2238 strlcpy(entry->tldl_reason, "UNKNOWN", TCP_LOG_ID_LEN);
2239 entry->tldl_ie = inp->inp_inc.inc_ie;
2240 if (inp->inp_inc.inc_flags & INC_ISIPV6)
2241 entry->tldl_af = AF_INET6;
2242 else
2243 entry->tldl_af = AF_INET;
2244 entry->tldl_entries = tp->t_logs;
2245 entry->tldl_count = tp->t_lognum;
2246
2247 /* Fill in the common parts of the queue entry. */
2248 entry->tldl_common.tldq_buf = NULL;
2249 entry->tldl_common.tldq_xform = tcp_log_expandlogbuf;
2250 entry->tldl_common.tldq_dtor = tcp_log_free_queue;
2251
2252 /* Clear the log data from the TCPCB. */
2253#ifdef TCPLOG_DEBUG_COUNTERS
2254 num_entries = tp->t_lognum;
2255#endif
2256 tp->t_lognum = 0;
2257 STAILQ_INIT(&tp->t_logs);
2258
2259 /* Add the entry. If no one is listening, free the entry. */
2260 if (tcp_log_dev_add_log((struct tcp_log_dev_queue *)entry)) {
2261 tcp_log_free_queue((struct tcp_log_dev_queue *)entry);
2262#ifdef TCPLOG_DEBUG_COUNTERS
2263 counter_u64_add(tcp_log_que_fail1, num_entries);
2264 } else {
2265 counter_u64_add(tcp_log_queued, num_entries);
2266#endif
2267 }
2268 return (0);
2269}
2270
2271/*
2272 * Queue the log_id_node's log buffers for transmission via the log buffer
2273 * facility.
2274 *
2275 * NOTE: This should be called with the bucket locked and referenced.
2276 *
2277 * how should be M_WAITOK or M_NOWAIT. If M_WAITOK, the function will drop
2278 * and reacquire the bucket lock if it needs to do so. (The caller must
2279 * ensure that the tln is no longer on any lists so no one else will mess
2280 * with this while the lock is dropped!)
2281 */
2282static int
2283tcp_log_dump_node_logbuf(struct tcp_log_id_node *tln, char *reason, int how)
2284{
2285 struct tcp_log_dev_log_queue *entry;
2286 struct tcp_log_id_bucket *tlb;
2287
2288 tlb = tln->tln_bucket;
2289 TCPID_BUCKET_LOCK_ASSERT(tlb);
2290 KASSERT(tlb->tlb_refcnt > 0,
2291 ("%s:%d: Called with unreferenced bucket (tln=%p, tlb=%p)",
2292 __func__, __LINE__, tln, tlb));
2293 KASSERT(tln->tln_closed,
2294 ("%s:%d: Called for node with tln_closed==false (tln=%p)",
2295 __func__, __LINE__, tln));
2296
2297 /* If there are no log entries, there is nothing to do. */
2298 if (tln->tln_count == 0)
2299 return (0);
2300
2301 /*
2302 * Allocate memory. If we must wait, we'll need to drop the locks
2303 * and reacquire them (and do all the related business that goes
2304 * along with that).
2305 */
2306 entry = malloc(sizeof(struct tcp_log_dev_log_queue), M_TCPLOGDEV,
2307 M_NOWAIT);
2308 if (entry == NULL && (how & M_NOWAIT))
2309 return (ENOBUFS);
2310 if (entry == NULL) {
2311 TCPID_BUCKET_UNLOCK(tlb);
2312 entry = malloc(sizeof(struct tcp_log_dev_log_queue),
2313 M_TCPLOGDEV, M_WAITOK);
2314 TCPID_BUCKET_LOCK(tlb);
2315 }
2316
2317 /* Fill in the common parts of the queue entry.. */
2318 entry->tldl_common.tldq_buf = NULL;
2319 entry->tldl_common.tldq_xform = tcp_log_expandlogbuf;
2320 entry->tldl_common.tldq_dtor = tcp_log_free_queue;
2321
2322 /* Fill in the unique parts of the queue entry. */
2323 strlcpy(entry->tldl_id, tlb->tlb_id, TCP_LOG_ID_LEN);
2324 strlcpy(entry->tldl_tag, tlb->tlb_tag, TCP_LOG_TAG_LEN);
2325 if (reason != NULL)
2326 strlcpy(entry->tldl_reason, reason, TCP_LOG_REASON_LEN);
2327 else
2328 strlcpy(entry->tldl_reason, "UNKNOWN", TCP_LOG_ID_LEN);
2329 entry->tldl_ie = tln->tln_ie;
2330 entry->tldl_entries = tln->tln_entries;
2331 entry->tldl_count = tln->tln_count;
2332 entry->tldl_af = tln->tln_af;
2333
2334 /* Add the entry. If no one is listening, free the entry. */
2335 if (tcp_log_dev_add_log((struct tcp_log_dev_queue *)entry))
2336 tcp_log_free_queue((struct tcp_log_dev_queue *)entry);
2337
2338 return (0);
2339}
2340
2341/*
2342 * Queue the log buffers for all sessions in a bucket for transmissions via
2343 * the log buffer facility.
2344 *
2345 * NOTE: This should be called with a locked bucket; however, the function
2346 * will drop the lock.
2347 */
2348#define LOCAL_SAVE 10
2349static void
2350tcp_log_dumpbucketlogs(struct tcp_log_id_bucket *tlb, char *reason)
2351{
2352 struct tcp_log_id_node local_entries[LOCAL_SAVE];
2353 struct inpcb *inp;
2354 struct tcpcb *tp;
2355 struct tcp_log_id_node *cur_tln, *prev_tln, *tmp_tln;
2356 int i, num_local_entries, tree_locked;
2357 bool expireq_locked;
2358
2359 TCPID_BUCKET_LOCK_ASSERT(tlb);
2360
2361 /*
2362 * Take a reference on the bucket to keep it from disappearing until
2363 * we are done.
2364 */
2365 TCPID_BUCKET_REF(tlb);
2366
2367 /*
2368 * We'll try to create these without dropping locks. However, we
2369 * might very well need to drop locks to get memory. If that's the
2370 * case, we'll save up to 10 on the stack, and sacrifice the rest.
2371 * (Otherwise, we need to worry about finding our place again in a
2372 * potentially changed list. It just doesn't seem worth the trouble
2373 * to do that.
2374 */
2375 expireq_locked = false;
2376 num_local_entries = 0;
2377 prev_tln = NULL;
2378 tree_locked = TREE_UNLOCKED;
2379 SLIST_FOREACH_SAFE(cur_tln, &tlb->tlb_head, tln_list, tmp_tln) {
2380 /*
2381 * If this isn't associated with a TCPCB, we can pull it off
2382 * the list now. We need to be careful that the expire timer
2383 * hasn't already taken ownership (tln_expiretime == SBT_MAX).
2384 * If so, we let the expire timer code free the data.
2385 */
2386 if (cur_tln->tln_closed) {
2387no_inp:
2388 /*
2389 * Get the expireq lock so we can get a consistent
2390 * read of tln_expiretime and so we can remove this
2391 * from the expireq.
2392 */
2393 if (!expireq_locked) {
2394 TCPLOG_EXPIREQ_LOCK();
2395 expireq_locked = true;
2396 }
2397
2398 /*
2399 * We ignore entries with tln_expiretime == SBT_MAX.
2400 * The expire timer code already owns those.
2401 */
2402 KASSERT(cur_tln->tln_expiretime > (sbintime_t) 0,
2403 ("%s:%d: node on the expire queue without positive "
2404 "expire time", __func__, __LINE__));
2405 if (cur_tln->tln_expiretime == SBT_MAX) {
2406 prev_tln = cur_tln;
2407 continue;
2408 }
2409
2410 /* Remove the entry from the expireq. */
2411 STAILQ_REMOVE(&tcp_log_expireq_head, cur_tln,
2412 tcp_log_id_node, tln_expireq);
2413
2414 /* Remove the entry from the bucket. */
2415 if (prev_tln != NULL)
2416 SLIST_REMOVE_AFTER(prev_tln, tln_list);
2417 else
2418 SLIST_REMOVE_HEAD(&tlb->tlb_head, tln_list);
2419
2420 /*
2421 * Drop the INP and bucket reference counts. Due to
2422 * lock-ordering rules, we need to drop the expire
2423 * queue lock.
2424 */
2425 TCPLOG_EXPIREQ_UNLOCK();
2426 expireq_locked = false;
2427
2428 /* Drop the INP reference. */
2429 INP_WLOCK(cur_tln->tln_inp);
2430 if (!in_pcbrele_wlocked(cur_tln->tln_inp))
2431 INP_WUNLOCK(cur_tln->tln_inp);
2432
2433 if (tcp_log_unref_bucket(tlb, &tree_locked, NULL)) {
2434#ifdef INVARIANTS
2435 panic("%s: Bucket refcount unexpectedly 0.",
2436 __func__);
2437#endif
2438 /*
2439 * Recover as best we can: free the entry we
2440 * own.
2441 */
2442 tcp_log_free_entries(&cur_tln->tln_entries,
2443 &cur_tln->tln_count);
2444 uma_zfree(tcp_log_node_zone, cur_tln);
2445 goto done;
2446 }
2447
2448 if (tcp_log_dump_node_logbuf(cur_tln, reason,
2449 M_NOWAIT)) {
2450 /*
2451 * If we have sapce, save the entries locally.
2452 * Otherwise, free them.
2453 */
2454 if (num_local_entries < LOCAL_SAVE) {
2455 local_entries[num_local_entries] =
2456 *cur_tln;
2457 num_local_entries++;
2458 } else {
2459 tcp_log_free_entries(
2460 &cur_tln->tln_entries,
2461 &cur_tln->tln_count);
2462 }
2463 }
2464
2465 /* No matter what, we are done with the node now. */
2466 uma_zfree(tcp_log_node_zone, cur_tln);
2467
2468 /*
2469 * Because we removed this entry from the list, prev_tln
2470 * (which tracks the previous entry still on the tlb
2471 * list) remains unchanged.
2472 */
2473 continue;
2474 }
2475
2476 /*
2477 * If we get to this point, the session data is still held in
2478 * the TCPCB. So, we need to pull the data out of that.
2479 *
2480 * We will need to drop the expireq lock so we can lock the INP.
2481 * We can then try to extract the data the "easy" way. If that
2482 * fails, we'll save the log entries for later.
2483 */
2484 if (expireq_locked) {
2485 TCPLOG_EXPIREQ_UNLOCK();
2486 expireq_locked = false;
2487 }
2488
2489 /* Lock the INP and then re-check the state. */
2490 inp = cur_tln->tln_inp;
2491 INP_WLOCK(inp);
2492 /*
2493 * If we caught this while it was transitioning, the data
2494 * might have moved from the TCPCB to the tln (signified by
2495 * setting tln_closed to true. If so, treat this like an
2496 * inactive connection.
2497 */
2498 if (cur_tln->tln_closed) {
2499 /*
2500 * It looks like we may have caught this connection
2501 * while it was transitioning from active to inactive.
2502 * Treat this like an inactive connection.
2503 */
2504 INP_WUNLOCK(inp);
2505 goto no_inp;
2506 }
2507
2508 /*
2509 * Try to dump the data from the tp without dropping the lock.
2510 * If this fails, try to save off the data locally.
2511 */
2512 tp = cur_tln->tln_tp;
2513 if (tcp_log_dump_tp_logbuf(tp, reason, M_NOWAIT, true) &&
2514 num_local_entries < LOCAL_SAVE) {
2515 tcp_log_move_tp_to_node(tp,
2516 &local_entries[num_local_entries]);
2517 local_entries[num_local_entries].tln_closed = 1;
2518 KASSERT(local_entries[num_local_entries].tln_bucket ==
2519 tlb, ("%s: %d: bucket mismatch for node %p",
2520 __func__, __LINE__, cur_tln));
2521 num_local_entries++;
2522 }
2523
2524 INP_WUNLOCK(inp);
2525
2526 /*
2527 * We are goint to leave the current tln on the list. It will
2528 * become the previous tln.
2529 */
2530 prev_tln = cur_tln;
2531 }
2532
2533 /* Drop our locks, if any. */
2534 KASSERT(tree_locked == TREE_UNLOCKED,
2535 ("%s: %d: tree unexpectedly locked", __func__, __LINE__));
2536 switch (tree_locked) {
2537 case TREE_WLOCKED:
2538 TCPID_TREE_WUNLOCK();
2539 tree_locked = TREE_UNLOCKED;
2540 break;
2541 case TREE_RLOCKED:
2542 TCPID_TREE_RUNLOCK();
2543 tree_locked = TREE_UNLOCKED;
2544 break;
2545 }
2546 if (expireq_locked) {
2547 TCPLOG_EXPIREQ_UNLOCK();
2548 expireq_locked = false;
2549 }
2550
2551 /*
2552 * Try again for any saved entries. tcp_log_dump_node_logbuf() is
2553 * guaranteed to free the log entries within the node. And, since
2554 * the node itself is on our stack, we don't need to free it.
2555 */
2556 for (i = 0; i < num_local_entries; i++)
2557 tcp_log_dump_node_logbuf(&local_entries[i], reason, M_WAITOK);
2558
2559 /* Drop our reference. */
2560 if (!tcp_log_unref_bucket(tlb, &tree_locked, NULL))
2561 TCPID_BUCKET_UNLOCK(tlb);
2562
2563done:
2564 /* Drop our locks, if any. */
2565 switch (tree_locked) {
2566 case TREE_WLOCKED:
2567 TCPID_TREE_WUNLOCK();
2568 break;
2569 case TREE_RLOCKED:
2570 TCPID_TREE_RUNLOCK();
2571 break;
2572 }
2573 if (expireq_locked)
2574 TCPLOG_EXPIREQ_UNLOCK();
2575}
2576#undef LOCAL_SAVE
2577
2578/*
2579 * Queue the log buffers for all sessions in a bucket for transmissions via
2580 * the log buffer facility.
2581 *
2582 * NOTE: This should be called with a locked INP; however, the function
2583 * will drop the lock.
2584 */
2585void
2586tcp_log_dump_tp_bucket_logbufs(struct tcpcb *tp, char *reason)
2587{
2588 struct tcp_log_id_bucket *tlb;
2589 int tree_locked;
2590
2591 /* Figure out our bucket and lock it. */
2592 INP_WLOCK_ASSERT(tp->t_inpcb);
2593 tlb = tp->t_lib;
2594 if (tlb == NULL) {
2595 /*
2596 * No bucket; treat this like a request to dump a single
2597 * session's traces.
2598 */
2599 (void)tcp_log_dump_tp_logbuf(tp, reason, M_WAITOK, true);
2600 INP_WUNLOCK(tp->t_inpcb);
2601 return;
2602 }
2603 TCPID_BUCKET_REF(tlb);
2604 INP_WUNLOCK(tp->t_inpcb);
2605 TCPID_BUCKET_LOCK(tlb);
2606
2607 /* If we are the last reference, we have nothing more to do here. */
2608 tree_locked = TREE_UNLOCKED;
2609 if (tcp_log_unref_bucket(tlb, &tree_locked, NULL)) {
2610 switch (tree_locked) {
2611 case TREE_WLOCKED:
2612 TCPID_TREE_WUNLOCK();
2613 break;
2614 case TREE_RLOCKED:
2615 TCPID_TREE_RUNLOCK();
2616 break;
2617 }
2618 return;
2619 }
2620
2621 /* Turn this over to tcp_log_dumpbucketlogs() to finish the work. */
2622 tcp_log_dumpbucketlogs(tlb, reason);
2623}
2624
2625/*
2626 * Mark the end of a flow with the current stack. A stack can add
2627 * stack-specific info to this trace event by overriding this
2628 * function (see bbr_log_flowend() for example).
2629 */
2630void
2631tcp_log_flowend(struct tcpcb *tp)
2632{
2633 if (tp->t_logstate != TCP_LOG_STATE_OFF) {
2634 struct socket *so = tp->t_inpcb->inp_socket;
2635 TCP_LOG_EVENT(tp, NULL, &so->so_rcv, &so->so_snd,
2636 TCP_LOG_FLOWEND, 0, 0, NULL, false);
2637 }
2638}
SYSCTL_INT
SYSCTL_INT(_net_inet_accf_http, OID_AUTO, parsehttpversion, CTLFLAG_RW, &parse_http_version, 1, "Parse http version so that non 1.x requests work")
SYSCTL_NODE
static SYSCTL_NODE(_net_inet_accf, OID_AUTO, http, CTLFLAG_RW|CTLFLAG_MPSAFE, 0, "HTTP accept filter")
SYSCTL_BOOL
SYSCTL_BOOL(_net_inet_ip, OID_AUTO, broadcast_lowest, CTLFLAG_VNET|CTLFLAG_RW, &VNET_NAME(broadcast_lowest), 0, "Treat lowest address on a subnet (host 0) as broadcast")
uint32_t
__uint32_t uint32_t
Definition: in.h:62
uint8_t
__uint8_t uint8_t
Definition: in.h:52
SYSCTL_ULONG
SYSCTL_ULONG(_net_inet_ip_mcast, OID_AUTO, maxgrpsrc, CTLFLAG_RWTUN, &in_mcast_maxgrpsrc, 0, "Max source filters per group")
in_pcbref
void in_pcbref(struct inpcb *inp)
Definition: in_pcb.c:1762
in_pcbrele_wlocked
bool in_pcbrele_wlocked(struct inpcb *inp)
Definition: in_pcb.c:1792
in_pcb.h
INP_LOCK_ASSERT
#define INP_LOCK_ASSERT(inp)
Definition: in_pcb.h:527
INC_ISIPV6
#define INC_ISIPV6
Definition: in_pcb.h:124
INP_WLOCK
#define INP_WLOCK(inp)
Definition: in_pcb.h:518
INP_WLOCK_ASSERT
#define INP_WLOCK_ASSERT(inp)
Definition: in_pcb.h:529
INP_TIMEWAIT
#define INP_TIMEWAIT
Definition: in_pcb.h:644
INP_UNLOCK_ASSERT
#define INP_UNLOCK_ASSERT(inp)
Definition: in_pcb.h:530
INP_DROPPED
#define INP_DROPPED
Definition: in_pcb.h:646
INP_WUNLOCK
#define INP_WUNLOCK(inp)
Definition: in_pcb.h:522
in_var.h
next
ipfw_dyn_rule * next
Definition: ip_fw.h:0
count
u_int16_t count
Definition: ip_fw.h:18
state
u_int32_t state
Definition: ip_fw.h:10
SYSCTL_UMA_CUR
SYSCTL_UMA_CUR(_net_inet_ip, OID_AUTO, fragpackets, CTLFLAG_VNET, &VNET_NAME(ipq_zone), "Current number of IPv4 fragment reassembly queue entries")
in_conninfo::inc_flags
u_int8_t inc_flags
Definition: in_pcb.h:114
in_conninfo::inc_ie
struct in_endpoints inc_ie
Definition: in_pcb.h:118
in_endpoints
Definition: in_pcb.h:96
inpcb
Definition: in_pcb.h:217
inpcb::inp_socket
struct socket * inp_socket
Definition: in_pcb.h:254
inpcb::inp_flags
int inp_flags
Definition: in_pcb.h:246
inpcb::inp_inc
struct in_conninfo inp_inc
Definition: in_pcb.h:270
tcp_function_block::tfb_id
uint8_t tfb_id
Definition: tcp_var.h:380
tcp_log_buffer
Definition: tcp_log_buf.h:112
tcp_log_buffer::tlb_opts
uint8_t tlb_opts[TCP_MAXOLEN]
Definition: tcp_log_buf.h:167
tcp_log_buffer::tlb_sn
uint32_t tlb_sn
Definition: tcp_log_buf.h:116
tcp_log_buffer::tlb_rxbuf
struct tcp_log_sockbuf tlb_rxbuf
Definition: tcp_log_buf.h:128
tcp_log_buffer::tlb_th
struct tcphdr tlb_th
Definition: tcp_log_buf.h:166
tcp_log_buffer::tlb_stackid
uint8_t tlb_stackid
Definition: tcp_log_buf.h:117
tcp_log_buffer::tlb_flex2
uint32_t tlb_flex2
Definition: tcp_log_buf.h:153
tcp_log_buffer::tlb_tv
struct timeval tlb_tv
Definition: tcp_log_buf.h:114
tcp_log_buffer::tlb_flex1
uint32_t tlb_flex1
Definition: tcp_log_buf.h:152
tcp_log_buffer::tlb_eventflags
uint16_t tlb_eventflags
Definition: tcp_log_buf.h:119
tcp_log_buffer::tlb_errno
int tlb_errno
Definition: tcp_log_buf.h:125
tcp_log_buffer::tlb_txbuf
struct tcp_log_sockbuf tlb_txbuf
Definition: tcp_log_buf.h:129
tcp_log_buffer::tlb_stackinfo
union tcp_log_stackspecific tlb_stackinfo
Definition: tcp_log_buf.h:161
tcp_log_buffer::tlb_len
uint32_t tlb_len
Definition: tcp_log_buf.h:165
tcp_log_buffer::tlb_eventid
uint8_t tlb_eventid
Definition: tcp_log_buf.h:118
tcp_log_buffer::tlb_ticks
uint32_t tlb_ticks
Definition: tcp_log_buf.h:115
tcp_log_buffer::tlb_verbose
struct tcp_log_verbose tlb_verbose[0]
Definition: tcp_log_buf.h:170
tcp_log_id_bucket
Definition: tcp_log_buf.c:254
tcp_log_id_bucket::tlb_id
char tlb_id[TCP_LOG_ID_LEN]
Definition: tcp_log_buf.c:259
tcp_log_id_bucket::tlb_tag
char tlb_tag[TCP_LOG_TAG_LEN]
Definition: tcp_log_buf.c:260
tcp_log_id_node
Definition: tcp_log_buf.c:271
tcp_log_sockbuf::tls_sb_acc
uint32_t tls_sb_acc
Definition: tcp_log_buf.h:49
tcp_log_sockbuf::tls_sb_ccc
uint32_t tls_sb_ccc
Definition: tcp_log_buf.h:50
tcp_log_sockbuf::tls_sb_spare
uint32_t tls_sb_spare
Definition: tcp_log_buf.h:51
tcp_log_verbose
Definition: tcp_log_buf.h:56
tcp_log_verbose::tlv_trace_func
char tlv_trace_func[TCP_FUNC_LEN]
Definition: tcp_log_buf.h:59
tcp_log_verbose::tlv_trace_line
uint32_t tlv_trace_line
Definition: tcp_log_buf.h:61
tcp_log_verbose::tlv_snd_frm
char tlv_snd_frm[TCP_FUNC_LEN]
Definition: tcp_log_buf.h:58
tcpcb
Definition: tcp_var.h:132
tcpcb::t_loglimit
int t_loglimit
Definition: tcp_var.h:251
tcpcb::t_logstate
uint32_t t_logstate
Definition: tcp_var.h:138
tcpcb::t_logs
struct tcp_log_stailq t_logs
Definition: tcp_var.h:255
tcpcb::t_lib
struct tcp_log_id_bucket * t_lib
Definition: tcp_var.h:257
tcpcb::t_stats
struct statsblob * t_stats
Definition: tcp_var.h:259
tcpcb::t_logsn
uint32_t t_logsn
Definition: tcp_var.h:260
tcpcb::t_log_state_set
uint32_t t_log_state_set
Definition: tcp_var.h:144
tcpcb::t_lin
struct tcp_log_id_node * t_lin
Definition: tcp_var.h:256
tcpcb::t_flags2
u_int t_flags2
Definition: tcp_var.h:166
tcpcb::t_fb
struct tcp_function_block * t_fb
Definition: tcp_var.h:135
tcpcb::t_lognum
int t_lognum
Definition: tcp_var.h:250
tcpcb::t_inpcb
struct inpcb * t_inpcb
Definition: tcp_var.h:134
SYSCTL_COUNTER_U64
SYSCTL_COUNTER_U64(_net_inet_tcp_hpts_stats, OID_AUTO, hopeless, CTLFLAG_RD, &hpts_hopelessly_behind, "Number of times hpts could not catch up and was behind hopelessly")
tcp_log_getlogbuf
int tcp_log_getlogbuf(struct sockopt *sopt, struct tcpcb *tp)
Definition: tcp_log_buf.c:1927
TCPID_BUCKET_REF
#define TCPID_BUCKET_REF(tlb)
Definition: tcp_log_buf.c:245
tree_lock_state
tree_lock_state
Definition: tcp_log_buf.c:293
TREE_RLOCKED
@ TREE_RLOCKED
Definition: tcp_log_buf.c:295
TREE_UNLOCKED
@ TREE_UNLOCKED
Definition: tcp_log_buf.c:294
TREE_WLOCKED
@ TREE_WLOCKED
Definition: tcp_log_buf.c:296
tcp_log_remove_log_cleanup
static void tcp_log_remove_log_cleanup(struct tcpcb *tp, struct tcp_log_mem *log_entry)
Definition: tcp_log_buf.c:1068
tcp_log_copyout
static int tcp_log_copyout(struct sockopt *sopt, void *src, void *dst, size_t len)
Definition: tcp_log_buf.c:1838
tcp_log_entry_refcnt_rem
#define tcp_log_entry_refcnt_rem(l)
Definition: tcp_log_buf.c:1033
tcp_log_grow_tlb
static void tcp_log_grow_tlb(char *tlb_id, struct tcpcb *tp)
Definition: tcp_log_buf.c:499
tcp_log_dump_node_logbuf
static int tcp_log_dump_node_logbuf(struct tcp_log_id_node *tln, char *reason, int how)
Definition: tcp_log_buf.c:2283
RB_HEAD
RB_HEAD(tcp_log_id_tree, tcp_log_id_bucket)
TCP_LOG_EXPIRE_TIME
#define TCP_LOG_EXPIRE_TIME
Definition: tcp_log_buf.c:62
tcp_log_drain
void tcp_log_drain(struct tcpcb *tp)
Definition: tcp_log_buf.c:1783
RECHECK_INP_CLEAN
#define RECHECK_INP_CLEAN(cleanup)
Definition: tcp_log_buf.c:487
tcp_log_remove_log_head
static void tcp_log_remove_log_head(struct tcpcb *tp, struct tcp_log_mem *log_entry)
Definition: tcp_log_buf.c:1078
TCPID_BUCKET_LOCK
#define TCPID_BUCKET_LOCK(tlb)
Definition: tcp_log_buf.c:240
TCPID_TREE_RLOCK_ASSERT
#define TCPID_TREE_RLOCK_ASSERT()
Definition: tcp_log_buf.c:235
tcp_log_node_zone
static uma_zone_t tcp_log_node_zone
Definition: tcp_log_buf.c:68
TCPID_TREE_WLOCK
#define TCPID_TREE_WLOCK()
Definition: tcp_log_buf.c:229
tcp_log_tcpcbinit
void tcp_log_tcpcbinit(struct tcpcb *tp)
Definition: tcp_log_buf.c:1176
tcp_log_get_id_cnt
u_int tcp_log_get_id_cnt(struct tcpcb *tp)
Definition: tcp_log_buf.c:989
tcp_log_flowend
void tcp_log_flowend(struct tcpcb *tp)
Definition: tcp_log_buf.c:2631
TCPID_BUCKET_LOCK_DESTROY
#define TCPID_BUCKET_LOCK_DESTROY(tlb)
Definition: tcp_log_buf.c:239
TCPID_BUCKET_UNLOCK
#define TCPID_BUCKET_UNLOCK(tlb)
Definition: tcp_log_buf.c:241
TCPID_TREE_RLOCK
#define TCPID_TREE_RLOCK()
Definition: tcp_log_buf.c:230
tcp_log_session_limit
static int tcp_log_session_limit
Definition: tcp_log_buf.c:69
tcp_log_selectauto
static __inline bool tcp_log_selectauto(void)
Definition: tcp_log_buf.c:301
TCPID_TREE_WUNLOCK
#define TCPID_TREE_WUNLOCK()
Definition: tcp_log_buf.c:232
tcp_log_event_
struct tcp_log_buffer * tcp_log_event_(struct tcpcb *tp, struct tcphdr *th, struct sockbuf *rxbuf, struct sockbuf *txbuf, uint8_t eventid, int errornum, uint32_t len, union tcp_log_stackspecific *stackinfo, int th_hostorder, const char *output_caller, const char *func, int line, const struct timeval *itv)
Definition: tcp_log_buf.c:1520
TCPID_TREE_UNLOCK_ASSERT
#define TCPID_TREE_UNLOCK_ASSERT()
Definition: tcp_log_buf.c:236
tcp_log_bucket_zone
static uma_zone_t tcp_log_bucket_zone
Definition: tcp_log_buf.c:68
COPY_STAT
#define COPY_STAT(f)
tcp_log_set_tag
int tcp_log_set_tag(struct tcpcb *tp, char *tag)
Definition: tcp_log_buf.c:523
tcp_log_tcpcbfini
void tcp_log_tcpcbfini(struct tcpcb *tp)
Definition: tcp_log_buf.c:1319
tcp_id_tree_lock
static struct rwlock tcp_id_tree_lock
Definition: tcp_log_buf.c:228
TCPID_BUCKET_LOCK_ASSERT
#define TCPID_BUCKET_LOCK_ASSERT(tlb)
Definition: tcp_log_buf.c:242
tcp_log_version
static uint32_t tcp_log_version
Definition: tcp_log_buf.c:70
tcp_log_get_tag
size_t tcp_log_get_tag(struct tcpcb *tp, char *buf)
Definition: tcp_log_buf.c:944
TCPID_TREE_WLOCK_ASSERT
#define TCPID_TREE_WLOCK_ASSERT()
Definition: tcp_log_buf.c:234
__FBSDID
__FBSDID("$FreeBSD$")
RECHECK_INP
#define RECHECK_INP()
Definition: tcp_log_buf.c:496
COPY_STAT_T
#define COPY_STAT_T(f)
zerobuf
static uint8_t zerobuf[76]
Definition: tcp_log_buf.c:185
TCPID_BUCKET_LOCK_INIT
#define TCPID_BUCKET_LOCK_INIT(tlb)
Definition: tcp_log_buf.c:238
STAILQ_HEAD
static STAILQ_HEAD(tcp_log_id_node)
Definition: tcp_log_buf.c:73
tcp_log_purge_tp_logbuf
static void tcp_log_purge_tp_logbuf(struct tcpcb *tp)
Definition: tcp_log_buf.c:1495
LOCAL_SAVE
#define LOCAL_SAVE
Definition: tcp_log_buf.c:2348
tcp_log_increment_reqcnt
static void tcp_log_increment_reqcnt(struct tcp_log_id_bucket *tlb)
Definition: tcp_log_buf.c:511
tcp_log_set_id
int tcp_log_set_id(struct tcpcb *tp, char *id)
Definition: tcp_log_buf.c:561
TCPID_BUCKET_UNREF
#define TCPID_BUCKET_UNREF(tlb)
Definition: tcp_log_buf.c:246
TCPID_TREE_RUNLOCK
#define TCPID_TREE_RUNLOCK()
Definition: tcp_log_buf.c:233
TCPID_TREE_UPGRADE
#define TCPID_TREE_UPGRADE()
Definition: tcp_log_buf.c:231
tcp_log_state_change
int tcp_log_state_change(struct tcpcb *tp, int state)
Definition: tcp_log_buf.c:1746
tcp_log_entry_refcnt_add
#define tcp_log_entry_refcnt_add(l)
Definition: tcp_log_buf.c:1032
TCP_LOG_EXPIRE_INTVL
#define TCP_LOG_EXPIRE_INTVL
Definition: tcp_log_buf.c:65
tcp_log_logs_to_buf
static int tcp_log_logs_to_buf(struct sockopt *sopt, struct tcp_log_stailq *log_tailqp, struct tcp_log_buffer **end, int count)
Definition: tcp_log_buf.c:1848
tcp_log_free_queue
static void tcp_log_free_queue(struct tcp_log_dev_queue *param)
Definition: tcp_log_buf.c:2069
FREE_NEW_TLB
#define FREE_NEW_TLB()
tcp_log_init
void tcp_log_init(void)
Definition: tcp_log_buf.c:1138
tcp_log_unref_bucket
static bool tcp_log_unref_bucket(struct tcp_log_id_bucket *tlb, int *tree_locked, struct inpcb *inp)
Definition: tcp_log_buf.c:377
tcp_log_dump_tp_logbuf
int tcp_log_dump_tp_logbuf(struct tcpcb *tp, char *reason, int how, bool force)
Definition: tcp_log_buf.c:2152
tcp_log_dumpbucketlogs
static void tcp_log_dumpbucketlogs(struct tcp_log_id_bucket *tlb, char *reason)
Definition: tcp_log_buf.c:2350
tcp_log_verbose
bool tcp_log_verbose
Definition: tcp_log_buf.c:67
tcp_log_expire
static void tcp_log_expire(void *unused __unused)
Definition: tcp_log_buf.c:1197
TCPLOG_EXPIREQ_LOCK
#define TCPLOG_EXPIREQ_LOCK()
Definition: tcp_log_buf.c:248
TCPLOG_EXPIREQ_UNLOCK
#define TCPLOG_EXPIREQ_UNLOCK()
Definition: tcp_log_buf.c:249
tcp_log_free_log_common
static void tcp_log_free_log_common(struct tcp_log_mem *log_entry, int *count __unused)
Definition: tcp_log_buf.c:1042
tcp_log_expandlogbuf
static struct tcp_log_common_header * tcp_log_expandlogbuf(struct tcp_log_dev_queue *param)
Definition: tcp_log_buf.c:2091
tcp_log_id_cmp
static __inline int tcp_log_id_cmp(struct tcp_log_id_bucket *a, struct tcp_log_id_bucket *b)
Definition: tcp_log_buf.c:317
tcp_log_dump_tp_bucket_logbufs
void tcp_log_dump_tp_bucket_logbufs(struct tcpcb *tp, char *reason)
Definition: tcp_log_buf.c:2586
TCPID_BUCKET_UNLOCK_ASSERT
#define TCPID_BUCKET_UNLOCK_ASSERT(tlb)
Definition: tcp_log_buf.c:243
SLIST_HEAD
SLIST_HEAD(tcp_log_id_head, tcp_log_id_node)
tcp_log_id_validate_tree_lock
static __inline void tcp_log_id_validate_tree_lock(int tree_locked)
Definition: tcp_log_buf.c:327
tcp_log_get_id
size_t tcp_log_get_id(struct tcpcb *tp, char *buf)
Definition: tcp_log_buf.c:920
tcp_log_free_entries
static void tcp_log_free_entries(struct tcp_log_stailq *head, int *count)
Definition: tcp_log_buf.c:1054
tcp_log_id_head
static struct tcp_log_id_tree tcp_log_id_head
Definition: tcp_log_buf.c:72
tcp_log_zone
static uma_zone_t tcp_log_zone
Definition: tcp_log_buf.c:68
tcp_log_remove_bucket
static __inline void tcp_log_remove_bucket(struct tcp_log_id_bucket *tlb)
Definition: tcp_log_buf.c:349
tcp_log_move_tp_to_node
static void tcp_log_move_tp_to_node(struct tcpcb *tp, struct tcp_log_id_node *tln)
Definition: tcp_log_buf.c:1298
tcp_log_remove_id_node
static bool tcp_log_remove_id_node(struct inpcb *inp, struct tcpcb *tp, struct tcp_log_id_bucket *tlb, struct tcp_log_id_node *tln, int *tree_locked)
Definition: tcp_log_buf.c:449
tcp_log_buf.h
TLB_FLAG_HDR
#define TLB_FLAG_HDR
Definition: tcp_log_buf.h:122
TCP_LOG_STATE_HEAD
@ TCP_LOG_STATE_HEAD
Definition: tcp_log_buf.h:246
TCP_LOG_STATE_TAIL
@ TCP_LOG_STATE_TAIL
Definition: tcp_log_buf.h:245
TCP_LOG_STATE_CONTINUAL
@ TCP_LOG_STATE_CONTINUAL
Definition: tcp_log_buf.h:250
TCP_LOG_STATE_CLEAR
@ TCP_LOG_STATE_CLEAR
Definition: tcp_log_buf.h:243
TCP_LOG_STATE_TAIL_AUTO
@ TCP_LOG_STATE_TAIL_AUTO
Definition: tcp_log_buf.h:251
TCP_LOG_STATE_OFF
@ TCP_LOG_STATE_OFF
Definition: tcp_log_buf.h:244
TCP_LOG_STATE_HEAD_AUTO
@ TCP_LOG_STATE_HEAD_AUTO
Definition: tcp_log_buf.h:247
TLB_FLAG_RXBUF
#define TLB_FLAG_RXBUF
Definition: tcp_log_buf.h:120
TLB_FLAG_TXBUF
#define TLB_FLAG_TXBUF
Definition: tcp_log_buf.h:121
TCP_FUNC_LEN
#define TCP_FUNC_LEN
Definition: tcp_log_buf.h:57
TLB_FLAG_VERBOSE
#define TLB_FLAG_VERBOSE
Definition: tcp_log_buf.h:123
TCP_LOG_BUF_DEFAULT_SESSION_LIMIT
#define TCP_LOG_BUF_DEFAULT_SESSION_LIMIT
Definition: tcp_log_buf.h:298
TCP_LOG_TAG_LEN
#define TCP_LOG_TAG_LEN
Definition: tcp_log_buf.h:34
TCP_LOG_BUF_DEFAULT_GLOBAL_LIMIT
#define TCP_LOG_BUF_DEFAULT_GLOBAL_LIMIT
Definition: tcp_log_buf.h:299
TCP_LOG_REASON_LEN
#define TCP_LOG_REASON_LEN
Definition: tcp_log_buf.h:33
TCP_LOG_EVENT
#define TCP_LOG_EVENT(tp, th, rxbuf, txbuf, eventid, errornum, len, stackinfo, th_hostorder)
Definition: tcp_log_buf.h:334
TCP_LOG_FLOWEND
@ TCP_LOG_FLOWEND
Definition: tcp_log_buf.h:198
TCP_LOG_CONNEND
@ TCP_LOG_CONNEND
Definition: tcp_log_buf.h:227
TLB_FLAG_STACKINFO
#define TLB_FLAG_STACKINFO
Definition: tcp_log_buf.h:124
TCP_LOG_BUF_VER
#define TCP_LOG_BUF_VER
Definition: tcp_log_buf.h:35
tcp_stats_sample_rollthedice
int tcp_stats_sample_rollthedice(struct tcpcb *tp, void *seed_bytes, size_t seed_len)
Definition: tcp_stats.c:187
tcp_var.h
tcp_fields_to_net
static void tcp_fields_to_net(struct tcphdr *th)
Definition: tcp_var.h:1255
intotcpcb
#define intotcpcb(ip)
Definition: tcp_var.h:645
TF2_LOG_AUTO
#define TF2_LOG_AUTO
Definition: tcp_var.h:562
tcp_log_stackspecific
Definition: tcp_log_buf.h:106