FreeBSD kernel IPv4 code
|
#include <sys/cdefs.h>
#include "opt_inet.h"
#include "opt_inet6.h"
#include "opt_rss.h"
#include "opt_tcpdebug.h"
#include <sys/param.h>
#include <sys/bus.h>
#include <sys/interrupt.h>
#include <sys/module.h>
#include <sys/kernel.h>
#include <sys/hhook.h>
#include <sys/malloc.h>
#include <sys/mbuf.h>
#include <sys/proc.h>
#include <sys/socket.h>
#include <sys/socketvar.h>
#include <sys/sysctl.h>
#include <sys/systm.h>
#include <sys/refcount.h>
#include <sys/sched.h>
#include <sys/queue.h>
#include <sys/smp.h>
#include <sys/counter.h>
#include <sys/time.h>
#include <sys/kthread.h>
#include <sys/kern_prefetch.h>
#include <vm/uma.h>
#include <vm/vm.h>
#include <net/route.h>
#include <net/vnet.h>
#include <netinet/in.h>
#include <netinet/in_kdtrace.h>
#include <netinet/in_pcb.h>
#include <netinet/ip.h>
#include <netinet/ip_icmp.h>
#include <netinet/icmp_var.h>
#include <netinet/ip_var.h>
#include <netinet/ip6.h>
#include <netinet6/in6_pcb.h>
#include <netinet6/ip6_var.h>
#include <netinet/tcp.h>
#include <netinet/tcp_fsm.h>
#include <netinet/tcp_seq.h>
#include <netinet/tcp_timer.h>
#include <netinet/tcp_var.h>
#include <netinet/tcpip.h>
#include <netinet/cc/cc.h>
#include <netinet/tcp_hpts.h>
#include <netinet/tcp_log_buf.h>
Go to the source code of this file.
Data Structures | |
struct | tcp_hpts_entry |
struct | tcp_hpts_entry::hptsh |
struct | hptsh |
struct | tcp_hptsi |
struct | hpts_domain_info |
Macros | |
#define | TCPSTATES /* for logging */ |
#define | NUM_OF_HPTSI_SLOTS 102400 |
#define | HPTS_MTX_ASSERT(hpts) mtx_assert(&(hpts)->p_mtx, MA_OWNED) |
#define | HPTS_LOCK(hpts) mtx_lock(&(hpts)->p_mtx) |
#define | HPTS_UNLOCK(hpts) mtx_unlock(&(hpts)->p_mtx) |
#define | timersub(tvp, uvp, vvp) |
#define | HPTS_MAX_SLEEP_ALLOWED (NUM_OF_HPTSI_SLOTS/2) |
Enumerations | |
enum | { IHPTS_NONE = 0 , IHPTS_ONQUEUE , IHPTS_MOVING } |
Functions | |
__FBSDID ("$FreeBSD$") | |
struct tcp_hpts_entry | __aligned (CACHE_LINE_SIZE) |
MALLOC_DEFINE (M_TCPHPTS, "tcp_hpts", "TCP hpts") | |
static int32_t | tcp_hptsi (struct tcp_hpts_entry *hpts, int from_callout) |
static void | tcp_hpts_thread (void *ctx) |
static void | tcp_init_hptsi (void *st) |
SYSCTL_NODE (_net_inet_tcp, OID_AUTO, hpts, CTLFLAG_RW|CTLFLAG_MPSAFE, 0, "TCP Hpts controls") | |
SYSCTL_NODE (_net_inet_tcp_hpts, OID_AUTO, stats, CTLFLAG_RD|CTLFLAG_MPSAFE, 0, "TCP Hpts statistics") | |
SYSCTL_COUNTER_U64 (_net_inet_tcp_hpts_stats, OID_AUTO, hopeless, CTLFLAG_RD, &hpts_hopelessly_behind, "Number of times hpts could not catch up and was behind hopelessly") | |
SYSCTL_COUNTER_U64 (_net_inet_tcp_hpts_stats, OID_AUTO, loops, CTLFLAG_RD, &hpts_loops, "Number of times hpts had to loop to catch up") | |
SYSCTL_COUNTER_U64 (_net_inet_tcp_hpts_stats, OID_AUTO, no_tcbsfound, CTLFLAG_RD, &back_tosleep, "Number of times hpts found no tcbs") | |
SYSCTL_COUNTER_U64 (_net_inet_tcp_hpts_stats, OID_AUTO, comb_wheel_wrap, CTLFLAG_RD, &combined_wheel_wrap, "Number of times the wheel lagged enough to have an insert see wrap") | |
SYSCTL_COUNTER_U64 (_net_inet_tcp_hpts_stats, OID_AUTO, wheel_wrap, CTLFLAG_RD, &wheel_wrap, "Number of times the wheel lagged enough to have an insert see wrap") | |
SYSCTL_COUNTER_U64 (_net_inet_tcp_hpts_stats, OID_AUTO, direct_call, CTLFLAG_RD, &hpts_direct_call, "Number of times hpts was called by syscall/trap or other entry") | |
SYSCTL_COUNTER_U64 (_net_inet_tcp_hpts_stats, OID_AUTO, timeout_wakeup, CTLFLAG_RD, &hpts_wake_timeout, "Number of times hpts threads woke up via the callout expiring") | |
SYSCTL_COUNTER_U64 (_net_inet_tcp_hpts_stats, OID_AUTO, direct_awakening, CTLFLAG_RD, &hpts_direct_awakening, "Number of times hpts threads woke up via the callout expiring") | |
SYSCTL_COUNTER_U64 (_net_inet_tcp_hpts_stats, OID_AUTO, back_tosleep, CTLFLAG_RD, &hpts_back_tosleep, "Number of times hpts threads woke up via the callout expiring and went back to sleep no work") | |
SYSCTL_COUNTER_U64 (_net_inet_tcp_hpts_stats, OID_AUTO, cpusel_flowid, CTLFLAG_RD, &cpu_uses_flowid, "Number of times when setting cpuid we used the flowid field") | |
SYSCTL_COUNTER_U64 (_net_inet_tcp_hpts_stats, OID_AUTO, cpusel_random, CTLFLAG_RD, &cpu_uses_random, "Number of times when setting cpuid we used the a random value") | |
TUNABLE_INT ("net.inet.tcp.bind_hptss", &tcp_bind_threads) | |
TUNABLE_INT ("net.inet.tcp.use_irq", &tcp_use_irq_cpu) | |
SYSCTL_INT (_net_inet_tcp_hpts, OID_AUTO, bind_hptss, CTLFLAG_RD, &tcp_bind_threads, 2, "Thread Binding tunable") | |
SYSCTL_INT (_net_inet_tcp_hpts, OID_AUTO, use_irq, CTLFLAG_RD, &tcp_use_irq_cpu, 0, "Use of irq CPU tunable") | |
SYSCTL_INT (_net_inet_tcp_hpts, OID_AUTO, precision, CTLFLAG_RW, &tcp_hpts_precision, 120, "Value for PRE() precision of callout") | |
SYSCTL_INT (_net_inet_tcp_hpts, OID_AUTO, cnt_thresh, CTLFLAG_RW, &conn_cnt_thresh, 0, "How many connections (below) make us use the callout based mechanism") | |
SYSCTL_INT (_net_inet_tcp_hpts, OID_AUTO, logging, CTLFLAG_RW, &hpts_does_tp_logging, 0, "Do we add to any tp that has logging on pacer logs") | |
SYSCTL_INT (_net_inet_tcp_hpts, OID_AUTO, use_assigned_cpu, CTLFLAG_RW, &hpts_use_assigned_cpu, 0, "Do we start any hpts timer on the assigned cpu?") | |
SYSCTL_INT (_net_inet_tcp_hpts, OID_AUTO, use_oldest, CTLFLAG_RW, &hpts_uses_oldest, OLDEST_THRESHOLD, "Do syscalls look for the hpts that has been the longest since running (or just use cpu no if 0)?") | |
SYSCTL_INT (_net_inet_tcp_hpts, OID_AUTO, dyn_minsleep, CTLFLAG_RW, &dynamic_min_sleep, 250, "What is the dynamic minsleep value?") | |
SYSCTL_INT (_net_inet_tcp_hpts, OID_AUTO, dyn_maxsleep, CTLFLAG_RW, &dynamic_max_sleep, 5000, "What is the dynamic maxsleep value?") | |
SYSCTL_INT (_net_inet_tcp_hpts, OID_AUTO, loopmax, CTLFLAG_RW, &max_pacer_loops, 10, "What is the maximum number of times the pacer will loop trying to catch up") | |
static int | sysctl_net_inet_tcp_hpts_max_sleep (SYSCTL_HANDLER_ARGS) |
static int | sysctl_net_inet_tcp_hpts_min_sleep (SYSCTL_HANDLER_ARGS) |
SYSCTL_PROC (_net_inet_tcp_hpts, OID_AUTO, maxsleep, CTLTYPE_UINT|CTLFLAG_RW|CTLFLAG_NEEDGIANT, &hpts_sleep_max, 0, &sysctl_net_inet_tcp_hpts_max_sleep, "IU", "Maximum time hpts will sleep") | |
SYSCTL_PROC (_net_inet_tcp_hpts, OID_AUTO, minsleep, CTLTYPE_UINT|CTLFLAG_RW|CTLFLAG_NEEDGIANT, &tcp_min_hptsi_time, 0, &sysctl_net_inet_tcp_hpts_min_sleep, "IU", "The minimum time the hpts must sleep before processing more slots") | |
SYSCTL_INT (_net_inet_tcp_hpts, OID_AUTO, more_sleep, CTLFLAG_RW, &ticks_indicate_more_sleep, 0, "If we only process this many or less on a timeout, we need longer sleep on the next callout") | |
SYSCTL_INT (_net_inet_tcp_hpts, OID_AUTO, less_sleep, CTLFLAG_RW, &ticks_indicate_less_sleep, 0, "If we process this many or more on a timeout, we need less sleep on the next callout") | |
SYSCTL_INT (_net_inet_tcp_hpts, OID_AUTO, nowake_over_thresh, CTLFLAG_RW, &tcp_hpts_no_wake_over_thresh, 0, "When we are over the threshold on the pacer do we prohibit wakeups?") | |
static void | tcp_hpts_log (struct tcp_hpts_entry *hpts, struct tcpcb *tp, struct timeval *tv, int slots_to_run, int idx, int from_callout) |
static void | tcp_wakehpts (struct tcp_hpts_entry *hpts) |
static void | hpts_timeout_swi (void *arg) |
static void | inp_hpts_insert (struct inpcb *inp, struct tcp_hpts_entry *hpts) |
static struct tcp_hpts_entry * | tcp_hpts_lock (struct inpcb *inp) |
static void | inp_hpts_release (struct inpcb *inp) |
void | tcp_hpts_remove (struct inpcb *inp) |
bool | tcp_in_hpts (struct inpcb *inp) |
static int | hpts_slot (uint32_t wheel_slot, uint32_t plus) |
static int | tick_to_wheel (uint32_t cts_in_wticks) |
static int | hpts_slots_diff (int prev_slot, int slot_now) |
static int32_t | max_slots_available (struct tcp_hpts_entry *hpts, uint32_t wheel_slot, uint32_t *target_slot) |
uint32_t | tcp_hpts_insert_diag (struct inpcb *inp, uint32_t slot, int32_t line, struct hpts_diag *diag) |
uint16_t | hpts_random_cpu (struct inpcb *inp) |
static uint16_t | hpts_cpuid (struct inpcb *inp, int *failed) |
static void | tcp_drop_in_pkts (struct tcpcb *tp) |
static void | tcp_hpts_set_max_sleep (struct tcp_hpts_entry *hpts, int wrap_loop_cnt) |
void | __tcp_set_hpts (struct inpcb *inp, int32_t line) |
static void | __tcp_run_hpts (struct tcp_hpts_entry *hpts) |
static struct tcp_hpts_entry * | tcp_choose_hpts_to_run () |
void | tcp_run_hpts (void) |
SYSINIT (tcphptsi, SI_SUB_SOFTINTR, SI_ORDER_ANY, tcp_init_hptsi, NULL) | |
MODULE_VERSION (tcphpts, 1) | |
#define HPTS_LOCK | ( | hpts | ) | mtx_lock(&(hpts)->p_mtx) |
Definition at line 182 of file tcp_hpts.c.
#define HPTS_MAX_SLEEP_ALLOWED (NUM_OF_HPTSI_SLOTS/2) |
Definition at line 380 of file tcp_hpts.c.
#define HPTS_MTX_ASSERT | ( | hpts | ) | mtx_assert(&(hpts)->p_mtx, MA_OWNED) |
Definition at line 181 of file tcp_hpts.c.
#define HPTS_UNLOCK | ( | hpts | ) | mtx_unlock(&(hpts)->p_mtx) |
Definition at line 183 of file tcp_hpts.c.
#define NUM_OF_HPTSI_SLOTS 102400 |
Definition at line 178 of file tcp_hpts.c.
#define TCPSTATES /* for logging */ |
Some notes about usage.
The tcp_hpts system is designed to provide a high precision timer system for tcp. Its main purpose is to provide a mechanism for pacing packets out onto the wire. It can be used in two ways by a given TCP stack (and those two methods can be used simultaneously).
First, and probably the main thing its used by Rack and BBR, it can be used to call tcp_output() of a transport stack at some time in the future. The normal way this is done is that tcp_output() of the stack schedules itself to be called again by calling tcp_hpts_insert(tcpcb, slot). The slot is the time from now that the stack wants to be called but it must be converted to tcp_hpts's notion of slot. This is done with one of the macros HPTS_MS_TO_SLOTS or HPTS_USEC_TO_SLOTS. So a typical call from the tcp_output() routine might look like:
tcp_hpts_insert(tp, HPTS_USEC_TO_SLOTS(550));
The above would schedule tcp_ouput() to be called in 550 useconds. Note that if using this mechanism the stack will want to add near its top a check to prevent unwanted calls (from user land or the arrival of incoming ack's). So it would add something like:
if (tcp_in_hpts(inp)) return;
to prevent output processing until the time alotted has gone by. Of course this is a bare bones example and the stack will probably have more consideration then just the above.
In order to run input queued segments from the HPTS context the tcp stack must define an input function for tfb_do_queued_segments(). This function understands how to dequeue a array of packets that were input and knows how to call the correct processing routine.
Locking in this is important as well so most likely the stack will need to define the tfb_do_segment_nounlock() splitting tfb_do_segment() into two parts. The main processing part that does not unlock the INP and returns a value of 1 or 0. It returns 0 if all is well and the lock was not released. It returns 1 if we had to destroy the TCB (a reset received etc). The remains of tfb_do_segment() then become just a simple call to the tfb_do_segment_nounlock() function and check the return code and possibly unlock.
The stack must also set the flag on the INP that it supports this feature i.e. INP_SUPPORTS_MBUFQ. The LRO code recoginizes this flag as well and will queue packets when it is set. There are other flags as well INP_MBUF_QUEUE_READY and INP_DONT_SACK_QUEUE. The first flag tells the LRO code that we are in the pacer for output so there is no need to wake up the hpts system to get immediate input. The second tells the LRO code that its okay if a SACK arrives you can still defer input and let the current hpts timer run (this is usually set when a rack timer is up so we know SACK's are happening on the connection already and don't want to wakeup yet).
There is a common functions within the rack_bbr_common code version i.e. ctf_do_queued_segments(). This function knows how to take the input queue of packets from tp->t_in_pkts and process them digging out all the arguments, calling any bpf tap and calling into tfb_do_segment_nounlock(). The common function (ctf_do_queued_segments()) requires that you have defined the tfb_do_segment_nounlock() as described above.
Definition at line 138 of file tcp_hpts.c.
#define timersub | ( | tvp, | |
uvp, | |||
vvp | |||
) |
Definition at line 265 of file tcp_hpts.c.
anonymous enum |
Enumerator | |
---|---|
IHPTS_NONE | |
IHPTS_ONQUEUE | |
IHPTS_MOVING |
Definition at line 282 of file tcp_hpts.c.
struct tcp_hpts_entry __aligned | ( | CACHE_LINE_SIZE | ) |
__FBSDID | ( | "$FreeBSD$" | ) |
|
static |
Definition at line 1534 of file tcp_hpts.c.
References conn_cnt_thresh, dynamic_max_sleep, dynamic_min_sleep, hpts_direct_call, HPTS_MTX_ASSERT, HPTS_TICKS_PER_SLOT, hpts_timeout_swi(), hpts_use_assigned_cpu, tcp_hpts_entry::overidden_sleep, tcp_hpts_entry::p_cpu, tcp_hpts_entry::p_hpts_active, tcp_hpts_entry::p_hpts_sleep_time, tcp_hpts_entry::p_mtx, tcp_hpts_entry::p_mysleep, tcp_hpts_entry::p_on_min_sleep, tcp_hpts_entry::p_on_queue_cnt, tcp_hpts_entry::sleeping, tcp_hpts_entry::syscall_cnt, tcp_bind_threads, tcp_hpts_precision, tcp_hpts_set_max_sleep(), tcp_hptsi(), ticks_indicate_less_sleep, and ticks_indicate_more_sleep.
Referenced by tcp_run_hpts().
void __tcp_set_hpts | ( | struct inpcb * | inp, |
int32_t | line | ||
) |
Definition at line 1517 of file tcp_hpts.c.
References hpts_cpuid(), inpcb::inp_hpts_cpu, inpcb::inp_hpts_cpu_set, inpcb::inp_in_hpts, INP_WLOCK_ASSERT, tcp_hpts_entry::p_mtx, and tcp_hpts_lock().
Definition at line 1014 of file tcp_hpts.c.
References hpts_domain_info::count, hpts_domain_info::cpu, cpu_uses_flowid, cpu_uses_random, hpts_domains, hpts_random_cpu(), inpcb::inp_flowid, inpcb::inp_flowtype, inpcb::inp_hpts_cpu, inpcb::inp_hpts_cpu_set, inpcb::inp_irq_cpu, inpcb::inp_irq_cpu_set, inpcb::inp_numa_domain, tcp_bind_threads, and tcp_use_irq_cpu.
Referenced by __tcp_set_hpts().
Definition at line 994 of file tcp_hpts.c.
References inpcb::inp_hpts_cpu, inpcb::inp_hpts_cpu_set, tcp_hptsi::rp_num_hptss, and tcp_pace.
Referenced by hpts_cpuid(), and in_pcballoc().
Definition at line 617 of file tcp_hpts.c.
References NUM_OF_HPTSI_SLOTS.
Referenced by tcp_hpts_insert_diag(), tcp_hpts_set_max_sleep(), tcp_hptsi(), and tcp_init_hptsi().
|
inlinestatic |
Definition at line 641 of file tcp_hpts.c.
References NUM_OF_HPTSI_SLOTS.
Referenced by max_slots_available(), tcp_hpts_insert_diag(), and tcp_hptsi().
|
static |
Definition at line 499 of file tcp_hpts.c.
References tcp_hpts_entry::ie_cookie.
Referenced by __tcp_run_hpts(), tcp_hpts_insert_diag(), tcp_hpts_thread(), and tcp_init_hptsi().
|
static |
Definition at line 508 of file tcp_hpts.c.
References HPTS_MTX_ASSERT, IHPTS_MOVING, IHPTS_NONE, IHPTS_ONQUEUE, in_pcbref(), INP_DROPPED, inpcb::inp_flags, inpcb::inp_hpts_cpu, inpcb::inp_hpts_gencnt, inpcb::inp_hptsslot, inpcb::inp_in_hpts, INP_TIMEWAIT, INP_WLOCK_ASSERT, tcp_hpts_entry::p_cpu, tcp_hpts_entry::p_hptss, and tcp_hpts_entry::p_on_queue_cnt.
Referenced by tcp_hpts_insert_diag(), and tcp_hptsi().
|
static |
Definition at line 547 of file tcp_hpts.c.
References IHPTS_NONE, in_pcbrele_wlocked(), and inpcb::inp_in_hpts.
Referenced by tcp_hpts_remove(), and tcp_hptsi().
MALLOC_DEFINE | ( | M_TCPHPTS | , |
"tcp_hpts" | , | ||
"TCP hpts" | |||
) |
|
inlinestatic |
Definition at line 673 of file tcp_hpts.c.
References combined_wheel_wrap, hpts_slots_diff(), NUM_OF_HPTSI_SLOTS, tcp_hpts_entry::p_cur_slot, tcp_hpts_entry::p_hpts_active, tcp_hpts_entry::p_nxt_slot, tcp_hpts_entry::p_prev_slot, tcp_hpts_entry::p_runningslot, and tcp_hpts_entry::p_wheel_complete.
Referenced by tcp_hpts_insert_diag(), and tcp_hptsi().
MODULE_VERSION | ( | tcphpts | , |
1 | |||
) |
SYSCTL_COUNTER_U64 | ( | _net_inet_tcp_hpts_stats | , |
OID_AUTO | , | ||
back_tosleep | , | ||
CTLFLAG_RD | , | ||
& | hpts_back_tosleep, | ||
"Number of times hpts threads woke up via the callout expiring and went back to sleep no work" | |||
) |
SYSCTL_COUNTER_U64 | ( | _net_inet_tcp_hpts_stats | , |
OID_AUTO | , | ||
comb_wheel_wrap | , | ||
CTLFLAG_RD | , | ||
& | combined_wheel_wrap, | ||
"Number of times the wheel lagged enough to have an insert see wrap" | |||
) |
SYSCTL_COUNTER_U64 | ( | _net_inet_tcp_hpts_stats | , |
OID_AUTO | , | ||
cpusel_flowid | , | ||
CTLFLAG_RD | , | ||
& | cpu_uses_flowid, | ||
"Number of times when setting cpuid we used the flowid field" | |||
) |
SYSCTL_COUNTER_U64 | ( | _net_inet_tcp_hpts_stats | , |
OID_AUTO | , | ||
cpusel_random | , | ||
CTLFLAG_RD | , | ||
& | cpu_uses_random, | ||
"Number of times when setting cpuid we used the a random value" | |||
) |
SYSCTL_COUNTER_U64 | ( | _net_inet_tcp_hpts_stats | , |
OID_AUTO | , | ||
direct_awakening | , | ||
CTLFLAG_RD | , | ||
& | hpts_direct_awakening, | ||
"Number of times hpts threads woke up via the callout expiring" | |||
) |
SYSCTL_COUNTER_U64 | ( | _net_inet_tcp_hpts_stats | , |
OID_AUTO | , | ||
direct_call | , | ||
CTLFLAG_RD | , | ||
& | hpts_direct_call, | ||
"Number of times hpts was called by syscall/trap or other entry" | |||
) |
SYSCTL_COUNTER_U64 | ( | _net_inet_tcp_hpts_stats | , |
OID_AUTO | , | ||
hopeless | , | ||
CTLFLAG_RD | , | ||
& | hpts_hopelessly_behind, | ||
"Number of times hpts could not catch up and was behind hopelessly" | |||
) |
SYSCTL_COUNTER_U64 | ( | _net_inet_tcp_hpts_stats | , |
OID_AUTO | , | ||
loops | , | ||
CTLFLAG_RD | , | ||
& | hpts_loops, | ||
"Number of times hpts had to loop to catch up" | |||
) |
SYSCTL_COUNTER_U64 | ( | _net_inet_tcp_hpts_stats | , |
OID_AUTO | , | ||
no_tcbsfound | , | ||
CTLFLAG_RD | , | ||
& | back_tosleep, | ||
"Number of times hpts found no tcbs" | |||
) |
SYSCTL_COUNTER_U64 | ( | _net_inet_tcp_hpts_stats | , |
OID_AUTO | , | ||
timeout_wakeup | , | ||
CTLFLAG_RD | , | ||
& | hpts_wake_timeout, | ||
"Number of times hpts threads woke up via the callout expiring" | |||
) |
SYSCTL_COUNTER_U64 | ( | _net_inet_tcp_hpts_stats | , |
OID_AUTO | , | ||
wheel_wrap | , | ||
CTLFLAG_RD | , | ||
& | wheel_wrap, | ||
"Number of times the wheel lagged enough to have an insert see wrap" | |||
) |
SYSCTL_INT | ( | _net_inet_tcp_hpts | , |
OID_AUTO | , | ||
bind_hptss | , | ||
CTLFLAG_RD | , | ||
& | tcp_bind_threads, | ||
2 | , | ||
"Thread Binding tunable" | |||
) |
SYSCTL_INT | ( | _net_inet_tcp_hpts | , |
OID_AUTO | , | ||
cnt_thresh | , | ||
CTLFLAG_RW | , | ||
& | conn_cnt_thresh, | ||
0 | , | ||
"How many connections (below) make us use the callout based mechanism" | |||
) |
SYSCTL_INT | ( | _net_inet_tcp_hpts | , |
OID_AUTO | , | ||
dyn_maxsleep | , | ||
CTLFLAG_RW | , | ||
& | dynamic_max_sleep, | ||
5000 | , | ||
"What is the dynamic maxsleep value?" | |||
) |
SYSCTL_INT | ( | _net_inet_tcp_hpts | , |
OID_AUTO | , | ||
dyn_minsleep | , | ||
CTLFLAG_RW | , | ||
& | dynamic_min_sleep, | ||
250 | , | ||
"What is the dynamic minsleep value?" | |||
) |
SYSCTL_INT | ( | _net_inet_tcp_hpts | , |
OID_AUTO | , | ||
less_sleep | , | ||
CTLFLAG_RW | , | ||
& | ticks_indicate_less_sleep, | ||
0 | , | ||
"If we process this many or more on a | timeout, | ||
we need less sleep on the next callout" | |||
) |
SYSCTL_INT | ( | _net_inet_tcp_hpts | , |
OID_AUTO | , | ||
logging | , | ||
CTLFLAG_RW | , | ||
& | hpts_does_tp_logging, | ||
0 | , | ||
"Do we add to any tp that has logging on pacer logs" | |||
) |
SYSCTL_INT | ( | _net_inet_tcp_hpts | , |
OID_AUTO | , | ||
loopmax | , | ||
CTLFLAG_RW | , | ||
& | max_pacer_loops, | ||
10 | , | ||
"What is the maximum number of times the pacer will loop trying to catch up" | |||
) |
SYSCTL_INT | ( | _net_inet_tcp_hpts | , |
OID_AUTO | , | ||
more_sleep | , | ||
CTLFLAG_RW | , | ||
& | ticks_indicate_more_sleep, | ||
0 | , | ||
"If we only process this many or less on a | timeout, | ||
we need longer sleep on the next callout" | |||
) |
SYSCTL_INT | ( | _net_inet_tcp_hpts | , |
OID_AUTO | , | ||
nowake_over_thresh | , | ||
CTLFLAG_RW | , | ||
& | tcp_hpts_no_wake_over_thresh, | ||
0 | , | ||
"When we are over the threshold on the pacer do we prohibit wakeups?" | |||
) |
SYSCTL_INT | ( | _net_inet_tcp_hpts | , |
OID_AUTO | , | ||
precision | , | ||
CTLFLAG_RW | , | ||
& | tcp_hpts_precision, | ||
120 | , | ||
"Value for PRE() precision of callout" | |||
) |
SYSCTL_INT | ( | _net_inet_tcp_hpts | , |
OID_AUTO | , | ||
use_assigned_cpu | , | ||
CTLFLAG_RW | , | ||
& | hpts_use_assigned_cpu, | ||
0 | , | ||
"Do we start any hpts timer on the assigned cpu?" | |||
) |
SYSCTL_INT | ( | _net_inet_tcp_hpts | , |
OID_AUTO | , | ||
use_irq | , | ||
CTLFLAG_RD | , | ||
& | tcp_use_irq_cpu, | ||
0 | , | ||
"Use of irq CPU tunable" | |||
) |
SYSCTL_INT | ( | _net_inet_tcp_hpts | , |
OID_AUTO | , | ||
use_oldest | , | ||
CTLFLAG_RW | , | ||
& | hpts_uses_oldest, | ||
OLDEST_THRESHOLD | , | ||
"Do syscalls look for the hpts that has been the longest since running (or just use cpu no if 0)?" | |||
) |
|
static |
Definition at line 385 of file tcp_hpts.c.
References dynamic_min_sleep, HPTS_MAX_SLEEP_ALLOWED, and hpts_sleep_max.
|
static |
Definition at line 403 of file tcp_hpts.c.
References LOWEST_SLEEP_ALLOWED, and tcp_min_hptsi_time.
SYSCTL_NODE | ( | _net_inet_tcp | , |
OID_AUTO | , | ||
hpts | , | ||
CTLFLAG_RW| | CTLFLAG_MPSAFE, | ||
0 | , | ||
"TCP Hpts controls" | |||
) |
SYSCTL_NODE | ( | _net_inet_tcp_hpts | , |
OID_AUTO | , | ||
stats | , | ||
CTLFLAG_RD| | CTLFLAG_MPSAFE, | ||
0 | , | ||
"TCP Hpts statistics" | |||
) |
SYSCTL_PROC | ( | _net_inet_tcp_hpts | , |
OID_AUTO | , | ||
maxsleep | , | ||
CTLTYPE_UINT|CTLFLAG_RW| | CTLFLAG_NEEDGIANT, | ||
& | hpts_sleep_max, | ||
0 | , | ||
& | sysctl_net_inet_tcp_hpts_max_sleep, | ||
"IU" | , | ||
"Maximum time hpts will sleep" | |||
) |
SYSCTL_PROC | ( | _net_inet_tcp_hpts | , |
OID_AUTO | , | ||
minsleep | , | ||
CTLTYPE_UINT|CTLFLAG_RW| | CTLFLAG_NEEDGIANT, | ||
& | tcp_min_hptsi_time, | ||
0 | , | ||
& | sysctl_net_inet_tcp_hpts_min_sleep, | ||
"IU" | , | ||
"The minimum time the hpts must sleep before processing more slots" | |||
) |
SYSINIT | ( | tcphptsi | , |
SI_SUB_SOFTINTR | , | ||
SI_ORDER_ANY | , | ||
tcp_init_hptsi | , | ||
NULL | |||
) |
|
static |
Definition at line 1602 of file tcp_hpts.c.
References cts_last_ran, hpts_uses_oldest, tcp_hpts_entry::p_on_queue_cnt, tcp_hptsi::rp_ent, tcp_hptsi::rp_num_hptss, tcp_get_usecs(), tcp_pace, and TSTMP_GT.
Referenced by tcp_run_hpts().
|
static |
Definition at line 1069 of file tcp_hpts.c.
References tcpcb::t_in_pkt.
uint32_t tcp_hpts_insert_diag | ( | struct inpcb * | inp, |
uint32_t | slot, | ||
int32_t | line, | ||
struct hpts_diag * | diag | ||
) |
Definition at line 815 of file tcp_hpts.c.
References hpts_diag::co_ret, hpts_diag::have_slept, hpts_diag::hpts_sleep_time, hpts_slot(), hpts_slots_diff(), HPTS_TICKS_PER_SLOT, hpts_timeout_swi(), HPTS_UNLOCK, hpts_use_assigned_cpu, HPTS_USEC_IN_SEC, IHPTS_MOVING, INP_DROPPED, inpcb::inp_flags, inp_hpts_insert(), inpcb::inp_hpts_request, inpcb::inp_hptsslot, hpts_diag::inp_hptsslot, inpcb::inp_in_hpts, INP_TIMEWAIT, INP_WLOCK_ASSERT, max_slots_available(), hpts_diag::maxslots, hpts_diag::need_new_to, tcp_hpts_entry::p_cpu, tcp_hpts_entry::p_cur_slot, hpts_diag::p_cur_slot, tcp_hpts_entry::p_curtick, hpts_diag::p_curtick, tcp_hpts_entry::p_direct_wake, tcp_hpts_entry::p_hpts_active, hpts_diag::p_hpts_active, p_hpts_sleep_time, tcp_hpts_entry::p_hpts_sleep_time, tcp_hpts_entry::p_lasttick, hpts_diag::p_lasttick, tcp_hpts_entry::p_nxt_slot, hpts_diag::p_nxt_slot, tcp_hpts_entry::p_on_min_sleep, hpts_diag::p_on_min_sleep, tcp_hpts_entry::p_prev_slot, hpts_diag::p_prev_slot, tcp_hpts_entry::p_runningslot, hpts_diag::p_runningslot, tcp_hpts_entry::p_wheel_complete, hpts_diag::slot_remaining, hpts_diag::slot_req, tcp_bind_threads, tcp_hpts_lock(), tcp_hpts_precision, tcp_in_hpts(), tcp_tv_to_hptstick(), tcp_wakehpts(), tick_to_wheel(), hpts_diag::wheel_cts, hpts_diag::wheel_slot, and hpts_diag::yet_to_sleep.
Referenced by bbr_start_hpts_timer(), and rack_start_hpts_timer().
|
static |
Definition at line 534 of file tcp_hpts.c.
References HPTS_LOCK, inpcb::inp_hpts_cpu, INP_LOCK_ASSERT, tcp_hptsi::rp_ent, and tcp_pace.
Referenced by __tcp_set_hpts(), tcp_hpts_insert_diag(), and tcp_hpts_remove().
|
static |
Definition at line 446 of file tcp_hpts.c.
References tcp_log_bbr::applimited, BBR_LOG_HPTSDIAG, tcp_log_bbr::delivered, tcp_log_bbr::epoch, tcp_log_bbr::flex1, tcp_log_bbr::flex2, tcp_log_bbr::flex3, tcp_log_bbr::flex4, tcp_log_bbr::flex5, tcp_log_bbr::flex6, tcp_log_bbr::flex7, tcp_log_bbr::flex8, tcp_log_bbr::inflight, inpcb::inp_socket, tcp_log_bbr::lost, tcp_log_bbr::lt_epoch, tcp_hpts_entry::overidden_sleep, tcp_hpts_entry::p_cpu, tcp_hpts_entry::p_cur_slot, tcp_hpts_entry::p_curtick, tcp_hpts_entry::p_delayed_by, tcp_hpts_entry::p_hpts_sleep_time, tcp_hpts_entry::p_nxt_slot, tcp_hpts_entry::p_on_queue_cnt, tcp_hpts_entry::p_prev_slot, tcp_hpts_entry::p_runningslot, tcp_log_bbr::pacing_gain, tcp_log_bbr::pkt_epoch, tcp_log_bbr::pkts_out, tcp_hpts_entry::saved_curslot, tcp_hpts_entry::saved_curtick, tcp_hpts_entry::saved_prev_slot, tcpcb::t_inpcb, TCP_LOG_EVENTP, tcp_tv_to_usectick(), tcp_log_bbr::timeStamp, tcp_log_stackspecific::u_bbr, and tcp_log_bbr::use_lt_bw.
Referenced by tcp_hptsi().
void tcp_hpts_remove | ( | struct inpcb * | inp | ) |
Dynamic adjustment of sleeping times is done in "new" mode where we are depending on syscall returns and lro returns to push hpts forward mainly and the timer is only a backstop.
When we are in the "new" mode i.e. conn_cnt > conn_cnt_thresh then we do a dynamic adjustment on the time we sleep. Our threshold is if the lateness of the first client served (in ticks) is greater than or equal too ticks_indicate_more_sleep (10ms or 10000 ticks). If we were that late, the actual sleep time is adjusted down by 50%. If the ticks_ran is less than ticks_indicate_more_sleep (100 ticks or 1000usecs).
Definition at line 563 of file tcp_hpts.c.
References HPTS_UNLOCK, IHPTS_MOVING, IHPTS_ONQUEUE, inpcb::inp_hpts_gencnt, inp_hpts_release(), inpcb::inp_hpts_request, inpcb::inp_hptsslot, inpcb::inp_in_hpts, INP_WLOCK_ASSERT, tcp_hpts_entry::p_hptss, tcp_hpts_entry::p_on_queue_cnt, and tcp_hpts_lock().
Referenced by bbr_do_segment_nounlock(), bbr_exit_persist(), bbr_output_wtime(), bbr_timer_audit(), bbr_timer_cancel(), rack_do_goodput_measurement(), rack_do_segment_nounlock(), rack_exit_persist(), rack_output(), rack_timer_audit(), rack_timer_cancel(), tcp_close(), tcp_ctloutput_set(), and tcp_twstart().
|
static |
Definition at line 1088 of file tcp_hpts.c.
References hpts_sleep_max, hpts_slot(), NUM_OF_HPTSI_SLOTS, tcp_hpts_entry::p_cur_slot, tcp_hpts_entry::p_hpts_sleep_time, tcp_hpts_entry::p_hptss, and tcp_hpts_entry::p_on_queue_cnt.
Referenced by __tcp_run_hpts(), and tcp_hptsi().
|
static |
Definition at line 1655 of file tcp_hpts.c.
References conn_cnt_thresh, dynamic_max_sleep, dynamic_min_sleep, hpts_back_tosleep, hpts_direct_awakening, HPTS_MTX_ASSERT, HPTS_TICKS_PER_SLOT, hpts_timeout_swi(), hpts_use_assigned_cpu, hpts_wake_timeout, tcp_hpts_entry::overidden_sleep, tcp_hpts_entry::p_cpu, tcp_hpts_entry::p_direct_wake, tcp_hpts_entry::p_hpts_active, tcp_hpts_entry::p_hpts_sleep_time, tcp_hpts_entry::p_hpts_wake_scheduled, tcp_hpts_entry::p_mtx, tcp_hpts_entry::p_mysleep, tcp_hpts_entry::p_on_min_sleep, tcp_hpts_entry::p_on_queue_cnt, tcp_hpts_entry::sleeping, tcp_bind_threads, tcp_hpts_precision, tcp_hptsi(), tcp_min_hptsi_time, ticks_indicate_less_sleep, and ticks_indicate_more_sleep.
Referenced by tcp_init_hptsi().
|
static |
Definition at line 1113 of file tcp_hpts.c.
References cts_last_ran, hpts_does_tp_logging, hpts_hopelessly_behind, HPTS_LOCK, hpts_loops, HPTS_MTX_ASSERT, hpts_slot(), hpts_slots_diff(), HPTS_TICKS_PER_SLOT, HPTS_UNLOCK, IHPTS_MOVING, IHPTS_NONE, IHPTS_ONQUEUE, in_pcbrele_wlocked(), INP_DROPPED, inpcb::inp_flags, inpcb::inp_flags2, inpcb::inp_hpts_calls, inpcb::inp_hpts_cpu_set, inpcb::inp_hpts_gencnt, inp_hpts_insert(), inp_hpts_release(), inpcb::inp_hpts_request, inpcb::inp_hptsslot, inpcb::inp_in_hpts, inpcb::inp_ppcb, inpcb::inp_socket, INP_SUPPORTS_MBUFQ, INP_TIMEWAIT, INP_WLOCK, INP_WUNLOCK, intotcpcb, max_pacer_loops, max_slots_available(), NUM_OF_HPTSI_SLOTS, tcp_hpts_entry::p_cur_slot, tcp_hpts_entry::p_curtick, tcp_hpts_entry::p_delayed_by, tcp_hpts_entry::p_hptss, tcp_hpts_entry::p_lasttick, tcp_hpts_entry::p_num, tcp_hpts_entry::p_nxt_slot, tcp_hpts_entry::p_on_queue_cnt, tcp_hpts_entry::p_prev_slot, tcp_hpts_entry::p_runningslot, tcp_hpts_entry::p_wheel_complete, tcp_hpts_entry::saved_curslot, tcp_hpts_entry::saved_curtick, tcp_hpts_entry::saved_lasttick, tcp_hpts_entry::saved_prev_slot, tcpcb::t_fb, tcpcb::t_fb_ptr, tcpcb::t_in_pkt, tcpcb::t_logstate, TAILQ_HEAD(), tcp_gethptstick(), tcp_hpts_log(), tcp_hpts_set_max_sleep(), TCP_LOG_STATE_OFF, tcp_set_hpts, tcp_tv_to_usectick(), tcp_function_block::tfb_do_queued_segments, tick_to_wheel(), and wheel_wrap.
Referenced by __tcp_run_hpts(), and tcp_hpts_thread().
bool tcp_in_hpts | ( | struct inpcb * | inp | ) |
Definition at line 610 of file tcp_hpts.c.
References IHPTS_ONQUEUE, and inpcb::inp_in_hpts.
Referenced by bbr_cong_signal(), bbr_do_segment_nounlock(), bbr_exit_persist(), bbr_fill_in_logging_data(), bbr_output_wtime(), bbr_start_hpts_timer(), bbr_timer_audit(), bbr_timer_cancel(), rack_do_goodput_measurement(), rack_do_segment_nounlock(), rack_exit_persist(), rack_fast_output(), rack_fast_rsm_output(), rack_log_ack(), rack_log_doseg_done(), rack_log_fsb(), rack_log_input_packet(), rack_log_map_chg(), rack_log_progress_event(), rack_log_retran_reason(), rack_log_rtt_upd(), rack_log_to_cancel(), rack_log_to_event(), rack_log_to_start(), rack_log_type_bbrsnd(), rack_log_type_just_return(), rack_output(), rack_start_hpts_timer(), rack_timer_audit(), rack_timer_cancel(), and tcp_hpts_insert_diag().
|
static |
Definition at line 1803 of file tcp_hpts.c.
References back_tosleep, combined_wheel_wrap, count, hpts_domain_info::count, hpts_domain_info::cpu, cpu_uses_flowid, cpu_uses_random, cts_last_ran, hpts_back_tosleep, tcp_hpts_entry::hpts_ctx, hpts_direct_awakening, hpts_direct_call, hpts_domains, hpts_hopelessly_behind, hpts_loops, tcp_hpts_entry::hpts_root, hpts_sleep_max, hpts_slot(), HPTS_TICKS_PER_SLOT, hpts_timeout_swi(), hpts_use_assigned_cpu, hpts_wake_timeout, tcp_hpts_entry::ie, tcp_hpts_entry::ie_cookie, NUM_OF_HPTSI_SLOTS, tcp_hpts_entry::p_cpu, tcp_hpts_entry::p_cur_slot, tcp_hpts_entry::p_curtick, tcp_hpts_entry::p_hpts_active, tcp_hpts_entry::p_hpts_sleep_time, tcp_hpts_entry::p_hptss, tcp_hpts_entry::p_mtx, tcp_hpts_entry::p_mysleep, tcp_hpts_entry::p_num, tcp_hpts_entry::p_nxt_slot, tcp_hpts_entry::p_on_queue_cnt, tcp_hpts_entry::p_prev_slot, tcp_hpts_entry::p_runningslot, tcp_hptsi::rp_ent, tcp_hptsi::rp_num_hptss, tcp_hpts_entry::sleeping, tcp_hpts_entry::syscall_cnt, tcp_bind_threads, tcp_gethptstick(), tcp_hpts_precision, tcp_hpts_thread(), tcp_min_hptsi_time, tcp_pace, tcp_tv_to_usectick(), tick_to_wheel(), and wheel_wrap.
void tcp_run_hpts | ( | void | ) |
Definition at line 1642 of file tcp_hpts.c.
References __tcp_run_hpts(), and tcp_choose_hpts_to_run().
Referenced by tcp_lro_flush_all().
|
static |
Definition at line 484 of file tcp_hpts.c.
References conn_cnt_thresh, HPTS_MTX_ASSERT, tcp_hpts_entry::ie_cookie, tcp_hpts_entry::p_direct_wake, tcp_hpts_entry::p_hpts_wake_scheduled, tcp_hpts_entry::p_on_queue_cnt, and tcp_hpts_no_wake_over_thresh.
Referenced by tcp_hpts_insert_diag().
|
inlinestatic |
Definition at line 628 of file tcp_hpts.c.
References NUM_OF_HPTSI_SLOTS.
Referenced by tcp_hpts_insert_diag(), tcp_hptsi(), and tcp_init_hptsi().
TUNABLE_INT | ( | "net.inet.tcp.bind_hptss" | , |
& | tcp_bind_threads | ||
) |
TUNABLE_INT | ( | "net.inet.tcp.use_irq" | , |
& | tcp_use_irq_cpu | ||
) |
counter_u64_t back_tosleep |
Definition at line 299 of file tcp_hpts.c.
Referenced by tcp_init_hptsi().
counter_u64_t combined_wheel_wrap |
Definition at line 304 of file tcp_hpts.c.
Referenced by max_slots_available(), and tcp_init_hptsi().
|
static |
Definition at line 254 of file tcp_hpts.c.
Referenced by __tcp_run_hpts(), tcp_hpts_thread(), and tcp_wakehpts().
counter_u64_t cpu_uses_flowid |
Definition at line 333 of file tcp_hpts.c.
Referenced by hpts_cpuid(), and tcp_init_hptsi().
counter_u64_t cpu_uses_random |
Definition at line 334 of file tcp_hpts.c.
Referenced by hpts_cpuid(), and tcp_init_hptsi().
|
static |
Definition at line 244 of file tcp_hpts.c.
Referenced by tcp_choose_hpts_to_run(), tcp_hptsi(), and tcp_init_hptsi().
|
static |
Definition at line 256 of file tcp_hpts.c.
Referenced by __tcp_run_hpts(), and tcp_hpts_thread().
|
static |
Definition at line 255 of file tcp_hpts.c.
Referenced by __tcp_run_hpts(), sysctl_net_inet_tcp_hpts_max_sleep(), and tcp_hpts_thread().
counter_u64_t hpts_back_tosleep |
Definition at line 328 of file tcp_hpts.c.
Referenced by tcp_hpts_thread(), and tcp_init_hptsi().
struct sysctl_ctx_list hpts_ctx |
Definition at line 35 of file tcp_hpts.c.
counter_u64_t hpts_direct_awakening |
Definition at line 323 of file tcp_hpts.c.
Referenced by tcp_hpts_thread(), and tcp_init_hptsi().
counter_u64_t hpts_direct_call |
Definition at line 314 of file tcp_hpts.c.
Referenced by __tcp_run_hpts(), and tcp_init_hptsi().
|
static |
Definition at line 245 of file tcp_hpts.c.
Referenced by tcp_hptsi().
|
static |
Referenced by hpts_cpuid(), and tcp_init_hptsi().
counter_u64_t hpts_hopelessly_behind |
Definition at line 288 of file tcp_hpts.c.
Referenced by tcp_hptsi(), and tcp_init_hptsi().
counter_u64_t hpts_loops |
Definition at line 294 of file tcp_hpts.c.
Referenced by tcp_hptsi(), and tcp_init_hptsi().
struct sysctl_oid* hpts_root |
Definition at line 36 of file tcp_hpts.c.
|
static |
Definition at line 382 of file tcp_hpts.c.
Referenced by sysctl_net_inet_tcp_hpts_max_sleep(), tcp_hpts_set_max_sleep(), and tcp_init_hptsi().
|
static |
Definition at line 246 of file tcp_hpts.c.
Referenced by __tcp_run_hpts(), tcp_hpts_insert_diag(), tcp_hpts_thread(), and tcp_init_hptsi().
|
static |
Definition at line 247 of file tcp_hpts.c.
Referenced by tcp_choose_hpts_to_run().
counter_u64_t hpts_wake_timeout |
Definition at line 318 of file tcp_hpts.c.
Referenced by tcp_hpts_thread(), and tcp_init_hptsi().
struct intr_event* ie |
Definition at line 37 of file tcp_hpts.c.
void* ie_cookie |
Definition at line 38 of file tcp_hpts.c.
|
static |
Definition at line 375 of file tcp_hpts.c.
Referenced by tcp_hptsi().
uint32_t overidden_sleep |
Definition at line 28 of file tcp_hpts.c.
uint8_t p_avail |
Definition at line 18 of file tcp_hpts.c.
uint16_t p_cpu |
Definition at line 40 of file tcp_hpts.c.
uint32_t p_cur_slot |
Definition at line 10 of file tcp_hpts.c.
uint32_t p_curtick |
Definition at line 7 of file tcp_hpts.c.
uint32_t p_delayed_by |
Definition at line 33 of file tcp_hpts.c.
uint8_t p_direct_wake |
Definition at line 15 of file tcp_hpts.c.
uint8_t p_fill[3] |
Definition at line 19 of file tcp_hpts.c.
uint16_t p_hpts_active |
Definition at line 5 of file tcp_hpts.c.
uint32_t p_hpts_sleep_time |
Definition at line 26 of file tcp_hpts.c.
Referenced by tcp_hpts_insert_diag().
uint8_t p_hpts_wake_scheduled |
Definition at line 17 of file tcp_hpts.c.
struct hptsh * p_hptss |
uint32_t p_lasttick |
Definition at line 14 of file tcp_hpts.c.
struct mtx p_mtx |
Definition at line 1 of file tcp_hpts.c.
struct timeval p_mysleep |
Definition at line 2 of file tcp_hpts.c.
uint16_t p_num |
Definition at line 39 of file tcp_hpts.c.
uint32_t p_nxt_slot |
Definition at line 11 of file tcp_hpts.c.
uint8_t p_on_min_sleep |
Definition at line 16 of file tcp_hpts.c.
int32_t p_on_queue_cnt |
Definition at line 13 of file tcp_hpts.c.
uint32_t p_prev_slot |
Definition at line 9 of file tcp_hpts.c.
uint32_t p_runningslot |
Definition at line 8 of file tcp_hpts.c.
uint8_t p_wheel_complete |
Definition at line 6 of file tcp_hpts.c.
uint32_t saved_curslot |
Definition at line 31 of file tcp_hpts.c.
uint32_t saved_curtick |
Definition at line 30 of file tcp_hpts.c.
uint32_t saved_lasttick |
Definition at line 29 of file tcp_hpts.c.
uint32_t saved_prev_slot |
Definition at line 32 of file tcp_hpts.c.
uint64_t sleeping |
Definition at line 4 of file tcp_hpts.c.
uint64_t syscall_cnt |
Definition at line 3 of file tcp_hpts.c.
|
static |
Definition at line 241 of file tcp_hpts.c.
Referenced by __tcp_run_hpts(), hpts_cpuid(), tcp_hpts_insert_diag(), tcp_hpts_thread(), and tcp_init_hptsi().
|
static |
Definition at line 433 of file tcp_hpts.c.
Referenced by tcp_wakehpts().
|
static |
Definition at line 275 of file tcp_hpts.c.
Referenced by __tcp_run_hpts(), tcp_hpts_insert_diag(), tcp_hpts_thread(), and tcp_init_hptsi().
int32_t tcp_min_hptsi_time = DEFAULT_MIN_SLEEP |
Definition at line 253 of file tcp_hpts.c.
Referenced by bbr_log_type_tsosize(), sysctl_net_inet_tcp_hpts_min_sleep(), tcp_hpts_thread(), and tcp_init_hptsi().
|
static |
Referenced by hpts_random_cpu(), tcp_choose_hpts_to_run(), tcp_hpts_lock(), and tcp_init_hptsi().
|
static |
Definition at line 243 of file tcp_hpts.c.
Referenced by hpts_cpuid().
|
static |
Definition at line 432 of file tcp_hpts.c.
Referenced by __tcp_run_hpts(), and tcp_hpts_thread().
|
static |
Definition at line 431 of file tcp_hpts.c.
Referenced by __tcp_run_hpts(), and tcp_hpts_thread().
counter_u64_t wheel_wrap |
Definition at line 309 of file tcp_hpts.c.
Referenced by tcp_hptsi(), and tcp_init_hptsi().