#include <sys/cdefs.h>
#include "opt_inet.h"
#include "opt_inet6.h"
#include "opt_rss.h"
#include "opt_tcpdebug.h"
#include <sys/param.h>
#include <sys/bus.h>
#include <sys/interrupt.h>
#include <sys/module.h>
#include <sys/kernel.h>
#include <sys/hhook.h>
#include <sys/malloc.h>
#include <sys/mbuf.h>
#include <sys/proc.h>
#include <sys/socket.h>
#include <sys/socketvar.h>
#include <sys/sysctl.h>
#include <sys/systm.h>
#include <sys/refcount.h>
#include <sys/sched.h>
#include <sys/queue.h>
#include <sys/smp.h>
#include <sys/counter.h>
#include <sys/time.h>
#include <sys/kthread.h>
#include <sys/kern_prefetch.h>
#include <vm/uma.h>
#include <vm/vm.h>
#include <net/route.h>
#include <net/vnet.h>
#include <netinet/in.h>
#include <netinet/in_kdtrace.h>
#include <netinet/in_pcb.h>
#include <netinet/ip.h>
#include <netinet/ip_icmp.h>
#include <netinet/icmp_var.h>
#include <netinet/ip_var.h>
#include <netinet/ip6.h>
#include <netinet6/in6_pcb.h>
#include <netinet6/ip6_var.h>
#include <netinet/tcp.h>
#include <netinet/tcp_fsm.h>
#include <netinet/tcp_seq.h>
#include <netinet/tcp_timer.h>
#include <netinet/tcp_var.h>
#include <netinet/tcpip.h>
#include <netinet/cc/cc.h>
#include <netinet/tcp_hpts.h>
#include <netinet/tcp_log_buf.h>

Include dependency graph for tcp_hpts.c:

Data Structures
struct	tcp_hpts_entry

struct	tcp_hpts_entry::hptsh

struct	hptsh

struct	tcp_hptsi

struct	hpts_domain_info

Macros
#define	TCPSTATES /* for logging */

#define	NUM_OF_HPTSI_SLOTS 102400

#define	HPTS_MTX_ASSERT(hpts) mtx_assert(&(hpts)->p_mtx, MA_OWNED)

#define	HPTS_LOCK(hpts) mtx_lock(&(hpts)->p_mtx)

#define	HPTS_UNLOCK(hpts) mtx_unlock(&(hpts)->p_mtx)

#define	timersub(tvp, uvp, vvp)

#define	HPTS_MAX_SLEEP_ALLOWED (NUM_OF_HPTSI_SLOTS/2)

Enumerations
enum	{ IHPTS_NONE = 0 , IHPTS_ONQUEUE , IHPTS_MOVING }

Functions
	__FBSDID ("$FreeBSD$")

struct tcp_hpts_entry	__aligned (CACHE_LINE_SIZE)

	MALLOC_DEFINE (M_TCPHPTS, "tcp_hpts", "TCP hpts")

static int32_t	tcp_hptsi (struct tcp_hpts_entry *hpts, int from_callout)

static void	tcp_hpts_thread (void *ctx)

static void	tcp_init_hptsi (void *st)

	SYSCTL_NODE (_net_inet_tcp, OID_AUTO, hpts, CTLFLAG_RW\|CTLFLAG_MPSAFE, 0, "TCP Hpts controls")

	SYSCTL_NODE (_net_inet_tcp_hpts, OID_AUTO, stats, CTLFLAG_RD\|CTLFLAG_MPSAFE, 0, "TCP Hpts statistics")

	SYSCTL_COUNTER_U64 (_net_inet_tcp_hpts_stats, OID_AUTO, hopeless, CTLFLAG_RD, &hpts_hopelessly_behind, "Number of times hpts could not catch up and was behind hopelessly")

	SYSCTL_COUNTER_U64 (_net_inet_tcp_hpts_stats, OID_AUTO, loops, CTLFLAG_RD, &hpts_loops, "Number of times hpts had to loop to catch up")

	SYSCTL_COUNTER_U64 (_net_inet_tcp_hpts_stats, OID_AUTO, no_tcbsfound, CTLFLAG_RD, &back_tosleep, "Number of times hpts found no tcbs")

	SYSCTL_COUNTER_U64 (_net_inet_tcp_hpts_stats, OID_AUTO, comb_wheel_wrap, CTLFLAG_RD, &combined_wheel_wrap, "Number of times the wheel lagged enough to have an insert see wrap")

	SYSCTL_COUNTER_U64 (_net_inet_tcp_hpts_stats, OID_AUTO, wheel_wrap, CTLFLAG_RD, &wheel_wrap, "Number of times the wheel lagged enough to have an insert see wrap")

	SYSCTL_COUNTER_U64 (_net_inet_tcp_hpts_stats, OID_AUTO, direct_call, CTLFLAG_RD, &hpts_direct_call, "Number of times hpts was called by syscall/trap or other entry")

	SYSCTL_COUNTER_U64 (_net_inet_tcp_hpts_stats, OID_AUTO, timeout_wakeup, CTLFLAG_RD, &hpts_wake_timeout, "Number of times hpts threads woke up via the callout expiring")

	SYSCTL_COUNTER_U64 (_net_inet_tcp_hpts_stats, OID_AUTO, direct_awakening, CTLFLAG_RD, &hpts_direct_awakening, "Number of times hpts threads woke up via the callout expiring")

	SYSCTL_COUNTER_U64 (_net_inet_tcp_hpts_stats, OID_AUTO, back_tosleep, CTLFLAG_RD, &hpts_back_tosleep, "Number of times hpts threads woke up via the callout expiring and went back to sleep no work")

	SYSCTL_COUNTER_U64 (_net_inet_tcp_hpts_stats, OID_AUTO, cpusel_flowid, CTLFLAG_RD, &cpu_uses_flowid, "Number of times when setting cpuid we used the flowid field")

	SYSCTL_COUNTER_U64 (_net_inet_tcp_hpts_stats, OID_AUTO, cpusel_random, CTLFLAG_RD, &cpu_uses_random, "Number of times when setting cpuid we used the a random value")

	TUNABLE_INT ("net.inet.tcp.bind_hptss", &tcp_bind_threads)

	TUNABLE_INT ("net.inet.tcp.use_irq", &tcp_use_irq_cpu)

	SYSCTL_INT (_net_inet_tcp_hpts, OID_AUTO, bind_hptss, CTLFLAG_RD, &tcp_bind_threads, 2, "Thread Binding tunable")

	SYSCTL_INT (_net_inet_tcp_hpts, OID_AUTO, use_irq, CTLFLAG_RD, &tcp_use_irq_cpu, 0, "Use of irq CPU tunable")

	SYSCTL_INT (_net_inet_tcp_hpts, OID_AUTO, precision, CTLFLAG_RW, &tcp_hpts_precision, 120, "Value for PRE() precision of callout")

	SYSCTL_INT (_net_inet_tcp_hpts, OID_AUTO, cnt_thresh, CTLFLAG_RW, &conn_cnt_thresh, 0, "How many connections (below) make us use the callout based mechanism")

	SYSCTL_INT (_net_inet_tcp_hpts, OID_AUTO, logging, CTLFLAG_RW, &hpts_does_tp_logging, 0, "Do we add to any tp that has logging on pacer logs")

	SYSCTL_INT (_net_inet_tcp_hpts, OID_AUTO, use_assigned_cpu, CTLFLAG_RW, &hpts_use_assigned_cpu, 0, "Do we start any hpts timer on the assigned cpu?")

	SYSCTL_INT (_net_inet_tcp_hpts, OID_AUTO, use_oldest, CTLFLAG_RW, &hpts_uses_oldest, OLDEST_THRESHOLD, "Do syscalls look for the hpts that has been the longest since running (or just use cpu no if 0)?")

	SYSCTL_INT (_net_inet_tcp_hpts, OID_AUTO, dyn_minsleep, CTLFLAG_RW, &dynamic_min_sleep, 250, "What is the dynamic minsleep value?")

	SYSCTL_INT (_net_inet_tcp_hpts, OID_AUTO, dyn_maxsleep, CTLFLAG_RW, &dynamic_max_sleep, 5000, "What is the dynamic maxsleep value?")

	SYSCTL_INT (_net_inet_tcp_hpts, OID_AUTO, loopmax, CTLFLAG_RW, &max_pacer_loops, 10, "What is the maximum number of times the pacer will loop trying to catch up")

static int	sysctl_net_inet_tcp_hpts_max_sleep (SYSCTL_HANDLER_ARGS)

static int	sysctl_net_inet_tcp_hpts_min_sleep (SYSCTL_HANDLER_ARGS)

	SYSCTL_PROC (_net_inet_tcp_hpts, OID_AUTO, maxsleep, CTLTYPE_UINT\|CTLFLAG_RW\|CTLFLAG_NEEDGIANT, &hpts_sleep_max, 0, &sysctl_net_inet_tcp_hpts_max_sleep, "IU", "Maximum time hpts will sleep")

	SYSCTL_PROC (_net_inet_tcp_hpts, OID_AUTO, minsleep, CTLTYPE_UINT\|CTLFLAG_RW\|CTLFLAG_NEEDGIANT, &tcp_min_hptsi_time, 0, &sysctl_net_inet_tcp_hpts_min_sleep, "IU", "The minimum time the hpts must sleep before processing more slots")

	SYSCTL_INT (_net_inet_tcp_hpts, OID_AUTO, more_sleep, CTLFLAG_RW, &ticks_indicate_more_sleep, 0, "If we only process this many or less on a timeout, we need longer sleep on the next callout")

	SYSCTL_INT (_net_inet_tcp_hpts, OID_AUTO, less_sleep, CTLFLAG_RW, &ticks_indicate_less_sleep, 0, "If we process this many or more on a timeout, we need less sleep on the next callout")

	SYSCTL_INT (_net_inet_tcp_hpts, OID_AUTO, nowake_over_thresh, CTLFLAG_RW, &tcp_hpts_no_wake_over_thresh, 0, "When we are over the threshold on the pacer do we prohibit wakeups?")

static void	tcp_hpts_log (struct tcp_hpts_entry hpts, struct tcpcb tp, struct timeval *tv, int slots_to_run, int idx, int from_callout)

static void	tcp_wakehpts (struct tcp_hpts_entry *hpts)

static void	hpts_timeout_swi (void *arg)

static void	inp_hpts_insert (struct inpcb inp, struct tcp_hpts_entry hpts)

static struct tcp_hpts_entry *	tcp_hpts_lock (struct inpcb *inp)

static void	inp_hpts_release (struct inpcb *inp)

void	tcp_hpts_remove (struct inpcb *inp)

bool	tcp_in_hpts (struct inpcb *inp)

static int	hpts_slot (uint32_t wheel_slot, uint32_t plus)

static int	tick_to_wheel (uint32_t cts_in_wticks)

static int	hpts_slots_diff (int prev_slot, int slot_now)

static int32_t	max_slots_available (struct tcp_hpts_entry hpts, uint32_t wheel_slot, uint32_t target_slot)

uint32_t	tcp_hpts_insert_diag (struct inpcb inp, uint32_t slot, int32_t line, struct hpts_diag diag)

uint16_t	hpts_random_cpu (struct inpcb *inp)

static uint16_t	hpts_cpuid (struct inpcb inp, int failed)

static void	tcp_drop_in_pkts (struct tcpcb *tp)

static void	tcp_hpts_set_max_sleep (struct tcp_hpts_entry *hpts, int wrap_loop_cnt)

void	__tcp_set_hpts (struct inpcb *inp, int32_t line)

static void	__tcp_run_hpts (struct tcp_hpts_entry *hpts)

static struct tcp_hpts_entry *	tcp_choose_hpts_to_run ()

void	tcp_run_hpts (void)

	SYSINIT (tcphptsi, SI_SUB_SOFTINTR, SI_ORDER_ANY, tcp_init_hptsi, NULL)

	MODULE_VERSION (tcphpts, 1)

Variables
struct mtx	p_mtx

struct timeval	p_mysleep

uint64_t	syscall_cnt

uint64_t	sleeping

uint16_t	p_hpts_active

uint8_t	p_wheel_complete

uint32_t	p_curtick

uint32_t	p_runningslot

uint32_t	p_prev_slot

uint32_t	p_cur_slot

uint32_t	p_nxt_slot

int32_t	p_on_queue_cnt

uint32_t	p_lasttick

uint8_t	p_direct_wake

uint8_t	p_on_min_sleep

uint8_t	p_hpts_wake_scheduled

uint8_t	p_avail

uint8_t	p_fill [3]

struct hptsh *	p_hptss

uint32_t	p_hpts_sleep_time

uint32_t	overidden_sleep

uint32_t	saved_lasttick

uint32_t	saved_curtick

uint32_t	saved_curslot

uint32_t	saved_prev_slot

uint32_t	p_delayed_by

struct sysctl_ctx_list	hpts_ctx

struct sysctl_oid *	hpts_root

struct intr_event *	ie

void *	ie_cookie

uint16_t	p_num

uint16_t	p_cpu

static struct tcp_hptsi	tcp_pace

static int	tcp_bind_threads = 2

static int	tcp_use_irq_cpu = 0

static uint32_t *	cts_last_ran

static int	hpts_does_tp_logging = 0

static int	hpts_use_assigned_cpu = 1

static int32_t	hpts_uses_oldest = OLDEST_THRESHOLD

int32_t	tcp_min_hptsi_time = DEFAULT_MIN_SLEEP

static int	conn_cnt_thresh = DEFAULT_CONNECTION_THESHOLD

static int32_t	dynamic_min_sleep = DYNAMIC_MIN_SLEEP

static int32_t	dynamic_max_sleep = DYNAMIC_MAX_SLEEP

static int32_t	tcp_hpts_precision = 120

static struct hpts_domain_info	hpts_domains [MAXMEMDOM]

counter_u64_t	hpts_hopelessly_behind

counter_u64_t	hpts_loops

counter_u64_t	back_tosleep

counter_u64_t	combined_wheel_wrap

counter_u64_t	wheel_wrap

counter_u64_t	hpts_direct_call

counter_u64_t	hpts_wake_timeout

counter_u64_t	hpts_direct_awakening

counter_u64_t	hpts_back_tosleep

counter_u64_t	cpu_uses_flowid

counter_u64_t	cpu_uses_random

static int32_t	max_pacer_loops = 10

static uint32_t	hpts_sleep_max = HPTS_MAX_SLEEP_ALLOWED

static int	ticks_indicate_more_sleep = TICKS_INDICATE_MORE_SLEEP

static int	ticks_indicate_less_sleep = TICKS_INDICATE_LESS_SLEEP

static int	tcp_hpts_no_wake_over_thresh = 1

Macro Definition Documentation

◆ HPTS_LOCK

#define HPTS_LOCK ( hpts ) mtx_lock(&(hpts)->p_mtx)

Definition at line 182 of file tcp_hpts.c.

◆ HPTS_MAX_SLEEP_ALLOWED

#define HPTS_MAX_SLEEP_ALLOWED (NUM_OF_HPTSI_SLOTS/2)

Definition at line 380 of file tcp_hpts.c.

◆ HPTS_MTX_ASSERT

#define HPTS_MTX_ASSERT ( hpts ) mtx_assert(&(hpts)->p_mtx, MA_OWNED)

Definition at line 181 of file tcp_hpts.c.

◆ HPTS_UNLOCK

#define HPTS_UNLOCK ( hpts ) mtx_unlock(&(hpts)->p_mtx)

Definition at line 183 of file tcp_hpts.c.

◆ NUM_OF_HPTSI_SLOTS

#define NUM_OF_HPTSI_SLOTS 102400

Definition at line 178 of file tcp_hpts.c.

◆ TCPSTATES

#define TCPSTATES /* for logging */

Some notes about usage.

The tcp_hpts system is designed to provide a high precision timer system for tcp. Its main purpose is to provide a mechanism for pacing packets out onto the wire. It can be used in two ways by a given TCP stack (and those two methods can be used simultaneously).

First, and probably the main thing its used by Rack and BBR, it can be used to call tcp_output() of a transport stack at some time in the future. The normal way this is done is that tcp_output() of the stack schedules itself to be called again by calling tcp_hpts_insert(tcpcb, slot). The slot is the time from now that the stack wants to be called but it must be converted to tcp_hpts's notion of slot. This is done with one of the macros HPTS_MS_TO_SLOTS or HPTS_USEC_TO_SLOTS. So a typical call from the tcp_output() routine might look like:

tcp_hpts_insert(tp, HPTS_USEC_TO_SLOTS(550));

The above would schedule tcp_ouput() to be called in 550 useconds. Note that if using this mechanism the stack will want to add near its top a check to prevent unwanted calls (from user land or the arrival of incoming ack's). So it would add something like:

if (tcp_in_hpts(inp)) return;

to prevent output processing until the time alotted has gone by. Of course this is a bare bones example and the stack will probably have more consideration then just the above.

In order to run input queued segments from the HPTS context the tcp stack must define an input function for tfb_do_queued_segments(). This function understands how to dequeue a array of packets that were input and knows how to call the correct processing routine.

Locking in this is important as well so most likely the stack will need to define the tfb_do_segment_nounlock() splitting tfb_do_segment() into two parts. The main processing part that does not unlock the INP and returns a value of 1 or 0. It returns 0 if all is well and the lock was not released. It returns 1 if we had to destroy the TCB (a reset received etc). The remains of tfb_do_segment() then become just a simple call to the tfb_do_segment_nounlock() function and check the return code and possibly unlock.

The stack must also set the flag on the INP that it supports this feature i.e. INP_SUPPORTS_MBUFQ. The LRO code recoginizes this flag as well and will queue packets when it is set. There are other flags as well INP_MBUF_QUEUE_READY and INP_DONT_SACK_QUEUE. The first flag tells the LRO code that we are in the pacer for output so there is no need to wake up the hpts system to get immediate input. The second tells the LRO code that its okay if a SACK arrives you can still defer input and let the current hpts timer run (this is usually set when a rack timer is up so we know SACK's are happening on the connection already and don't want to wakeup yet).

There is a common functions within the rack_bbr_common code version i.e. ctf_do_queued_segments(). This function knows how to take the input queue of packets from tp->t_in_pkts and process them digging out all the arguments, calling any bpf tap and calling into tfb_do_segment_nounlock(). The common function (ctf_do_queued_segments()) requires that you have defined the tfb_do_segment_nounlock() as described above.

Definition at line 138 of file tcp_hpts.c.

◆ timersub

#define timersub	(	tvp,
		uvp,
		vvp
	)

Value:

        do {                                                            \
                (vvp)->tv_sec = (tvp)->tv_sec - (uvp)->tv_sec;          \
                (vvp)->tv_usec = (tvp)->tv_usec - (uvp)->tv_usec;       \
                if ((vvp)->tv_usec < 0) {                               \
                        (vvp)->tv_sec--;                                \
                        (vvp)->tv_usec += 1000000;                      \
                }                                                       \
        } while (0)

Definition at line 265 of file tcp_hpts.c.

Enumeration Type Documentation

◆ anonymous enum

anonymous enum

Enumerator
IHPTS_NONE
IHPTS_ONQUEUE
IHPTS_MOVING

Definition at line 282 of file tcp_hpts.c.

Function Documentation

◆ __aligned()

struct tcp_hpts_entry __aligned ( CACHE_LINE_SIZE )

◆ __FBSDID()

__FBSDID ( "$FreeBSD$" )

◆ __tcp_run_hpts()

static void __tcp_run_hpts ( struct tcp_hpts_entry * hpts )

static

Definition at line 1534 of file tcp_hpts.c.

References conn_cnt_thresh, dynamic_max_sleep, dynamic_min_sleep, hpts_direct_call, HPTS_MTX_ASSERT, HPTS_TICKS_PER_SLOT, hpts_timeout_swi(), hpts_use_assigned_cpu, tcp_hpts_entry::overidden_sleep, tcp_hpts_entry::p_cpu, tcp_hpts_entry::p_hpts_active, tcp_hpts_entry::p_hpts_sleep_time, tcp_hpts_entry::p_mtx, tcp_hpts_entry::p_mysleep, tcp_hpts_entry::p_on_min_sleep, tcp_hpts_entry::p_on_queue_cnt, tcp_hpts_entry::sleeping, tcp_hpts_entry::syscall_cnt, tcp_bind_threads, tcp_hpts_precision, tcp_hpts_set_max_sleep(), tcp_hptsi(), ticks_indicate_less_sleep, and ticks_indicate_more_sleep.

Referenced by tcp_run_hpts().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ __tcp_set_hpts()

void __tcp_set_hpts	(	struct inpcb *	inp,
		int32_t	line
	)

Definition at line 1517 of file tcp_hpts.c.

References hpts_cpuid(), inpcb::inp_hpts_cpu, inpcb::inp_hpts_cpu_set, inpcb::inp_in_hpts, INP_WLOCK_ASSERT, tcp_hpts_entry::p_mtx, and tcp_hpts_lock().

Here is the call graph for this function:

◆ hpts_cpuid()

static uint16_t hpts_cpuid	(	struct inpcb *	inp,
		int *	failed
	)

static

Definition at line 1014 of file tcp_hpts.c.

References hpts_domain_info::count, hpts_domain_info::cpu, cpu_uses_flowid, cpu_uses_random, hpts_domains, hpts_random_cpu(), inpcb::inp_flowid, inpcb::inp_flowtype, inpcb::inp_hpts_cpu, inpcb::inp_hpts_cpu_set, inpcb::inp_irq_cpu, inpcb::inp_irq_cpu_set, inpcb::inp_numa_domain, tcp_bind_threads, and tcp_use_irq_cpu.

Referenced by __tcp_set_hpts().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ hpts_random_cpu()

uint16_t hpts_random_cpu ( struct inpcb * inp )

Definition at line 994 of file tcp_hpts.c.

References inpcb::inp_hpts_cpu, inpcb::inp_hpts_cpu_set, tcp_hptsi::rp_num_hptss, and tcp_pace.

Referenced by hpts_cpuid(), and in_pcballoc().

Here is the caller graph for this function:

◆ hpts_slot()

static int hpts_slot	(	uint32_t	wheel_slot,
		uint32_t	plus
	)

inlinestatic

Definition at line 617 of file tcp_hpts.c.

References NUM_OF_HPTSI_SLOTS.

Referenced by tcp_hpts_insert_diag(), tcp_hpts_set_max_sleep(), tcp_hptsi(), and tcp_init_hptsi().

Here is the caller graph for this function:

◆ hpts_slots_diff()

static int hpts_slots_diff	(	int	prev_slot,
		int	slot_now
	)

inlinestatic

Definition at line 641 of file tcp_hpts.c.

References NUM_OF_HPTSI_SLOTS.

Referenced by max_slots_available(), tcp_hpts_insert_diag(), and tcp_hptsi().

Here is the caller graph for this function:

◆ hpts_timeout_swi()

static void hpts_timeout_swi ( void * arg )

static

Definition at line 499 of file tcp_hpts.c.

References tcp_hpts_entry::ie_cookie.

Referenced by __tcp_run_hpts(), tcp_hpts_insert_diag(), tcp_hpts_thread(), and tcp_init_hptsi().

Here is the caller graph for this function:

◆ inp_hpts_insert()

static void inp_hpts_insert	(	struct inpcb *	inp,
		struct tcp_hpts_entry *	hpts
	)

static

Definition at line 508 of file tcp_hpts.c.

References HPTS_MTX_ASSERT, IHPTS_MOVING, IHPTS_NONE, IHPTS_ONQUEUE, in_pcbref(), INP_DROPPED, inpcb::inp_flags, inpcb::inp_hpts_cpu, inpcb::inp_hpts_gencnt, inpcb::inp_hptsslot, inpcb::inp_in_hpts, INP_TIMEWAIT, INP_WLOCK_ASSERT, tcp_hpts_entry::p_cpu, tcp_hpts_entry::p_hptss, and tcp_hpts_entry::p_on_queue_cnt.

Referenced by tcp_hpts_insert_diag(), and tcp_hptsi().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ inp_hpts_release()

static void inp_hpts_release ( struct inpcb * inp )

static

Definition at line 547 of file tcp_hpts.c.

References IHPTS_NONE, in_pcbrele_wlocked(), and inpcb::inp_in_hpts.

Referenced by tcp_hpts_remove(), and tcp_hptsi().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ MALLOC_DEFINE()

MALLOC_DEFINE	(	M_TCPHPTS	,
		"tcp_hpts"	,
		"TCP hpts"
	)

◆ max_slots_available()

static int32_t max_slots_available	(	struct tcp_hpts_entry *	hpts,
		uint32_t	wheel_slot,
		uint32_t *	target_slot
	)

inlinestatic

Definition at line 673 of file tcp_hpts.c.

References combined_wheel_wrap, hpts_slots_diff(), NUM_OF_HPTSI_SLOTS, tcp_hpts_entry::p_cur_slot, tcp_hpts_entry::p_hpts_active, tcp_hpts_entry::p_nxt_slot, tcp_hpts_entry::p_prev_slot, tcp_hpts_entry::p_runningslot, and tcp_hpts_entry::p_wheel_complete.

Referenced by tcp_hpts_insert_diag(), and tcp_hptsi().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ MODULE_VERSION()

MODULE_VERSION	(	tcphpts	,
		1
	)

◆ SYSCTL_COUNTER_U64() [1/11]

SYSCTL_COUNTER_U64	(	_net_inet_tcp_hpts_stats	,
		OID_AUTO	,
		back_tosleep	,
		CTLFLAG_RD	,
		&	hpts_back_tosleep,
		"Number of times hpts threads woke up via the callout expiring and went back to sleep no work"
	)

◆ SYSCTL_COUNTER_U64() [2/11]

SYSCTL_COUNTER_U64	(	_net_inet_tcp_hpts_stats	,
		OID_AUTO	,
		comb_wheel_wrap	,
		CTLFLAG_RD	,
		&	combined_wheel_wrap,
		"Number of times the wheel lagged enough to have an insert see wrap"
	)

◆ SYSCTL_COUNTER_U64() [3/11]

SYSCTL_COUNTER_U64	(	_net_inet_tcp_hpts_stats	,
		OID_AUTO	,
		cpusel_flowid	,
		CTLFLAG_RD	,
		&	cpu_uses_flowid,
		"Number of times when setting cpuid we used the flowid field"
	)

◆ SYSCTL_COUNTER_U64() [4/11]

SYSCTL_COUNTER_U64	(	_net_inet_tcp_hpts_stats	,
		OID_AUTO	,
		cpusel_random	,
		CTLFLAG_RD	,
		&	cpu_uses_random,
		"Number of times when setting cpuid we used the a random value"
	)

◆ SYSCTL_COUNTER_U64() [5/11]

SYSCTL_COUNTER_U64	(	_net_inet_tcp_hpts_stats	,
		OID_AUTO	,
		direct_awakening	,
		CTLFLAG_RD	,
		&	hpts_direct_awakening,
		"Number of times hpts threads woke up via the callout expiring"
	)

◆ SYSCTL_COUNTER_U64() [6/11]

SYSCTL_COUNTER_U64	(	_net_inet_tcp_hpts_stats	,
		OID_AUTO	,
		direct_call	,
		CTLFLAG_RD	,
		&	hpts_direct_call,
		"Number of times hpts was called by syscall/trap or other entry"
	)

◆ SYSCTL_COUNTER_U64() [7/11]

SYSCTL_COUNTER_U64	(	_net_inet_tcp_hpts_stats	,
		OID_AUTO	,
		hopeless	,
		CTLFLAG_RD	,
		&	hpts_hopelessly_behind,
		"Number of times hpts could not catch up and was behind hopelessly"
	)

◆ SYSCTL_COUNTER_U64() [8/11]

SYSCTL_COUNTER_U64	(	_net_inet_tcp_hpts_stats	,
		OID_AUTO	,
		loops	,
		CTLFLAG_RD	,
		&	hpts_loops,
		"Number of times hpts had to loop to catch up"
	)

◆ SYSCTL_COUNTER_U64() [9/11]

SYSCTL_COUNTER_U64	(	_net_inet_tcp_hpts_stats	,
		OID_AUTO	,
		no_tcbsfound	,
		CTLFLAG_RD	,
		&	back_tosleep,
		"Number of times hpts found no tcbs"
	)

◆ SYSCTL_COUNTER_U64() [10/11]

SYSCTL_COUNTER_U64	(	_net_inet_tcp_hpts_stats	,
		OID_AUTO	,
		timeout_wakeup	,
		CTLFLAG_RD	,
		&	hpts_wake_timeout,
		"Number of times hpts threads woke up via the callout expiring"
	)

◆ SYSCTL_COUNTER_U64() [11/11]

SYSCTL_COUNTER_U64	(	_net_inet_tcp_hpts_stats	,
		OID_AUTO	,
		wheel_wrap	,
		CTLFLAG_RD	,
		&	wheel_wrap,
		"Number of times the wheel lagged enough to have an insert see wrap"
	)

◆ SYSCTL_INT() [1/13]

SYSCTL_INT	(	_net_inet_tcp_hpts	,
		OID_AUTO	,
		bind_hptss	,
		CTLFLAG_RD	,
		&	tcp_bind_threads,
		2	,
		"Thread Binding tunable"
	)

◆ SYSCTL_INT() [2/13]

SYSCTL_INT	(	_net_inet_tcp_hpts	,
		OID_AUTO	,
		cnt_thresh	,
		CTLFLAG_RW	,
		&	conn_cnt_thresh,
		0	,
		"How many connections (below) make us use the callout based mechanism"
	)

◆ SYSCTL_INT() [3/13]

SYSCTL_INT	(	_net_inet_tcp_hpts	,
		OID_AUTO	,
		dyn_maxsleep	,
		CTLFLAG_RW	,
		&	dynamic_max_sleep,
		5000	,
		"What is the dynamic maxsleep value?"
	)

◆ SYSCTL_INT() [4/13]

SYSCTL_INT	(	_net_inet_tcp_hpts	,
		OID_AUTO	,
		dyn_minsleep	,
		CTLFLAG_RW	,
		&	dynamic_min_sleep,
		250	,
		"What is the dynamic minsleep value?"
	)

◆ SYSCTL_INT() [5/13]

SYSCTL_INT	(	_net_inet_tcp_hpts	,
		OID_AUTO	,
		less_sleep	,
		CTLFLAG_RW	,
		&	ticks_indicate_less_sleep,
		0	,
		"If we process this many or more on a	timeout,
		we need less sleep on the next callout"
	)

◆ SYSCTL_INT() [6/13]

SYSCTL_INT	(	_net_inet_tcp_hpts	,
		OID_AUTO	,
		logging	,
		CTLFLAG_RW	,
		&	hpts_does_tp_logging,
		0	,
		"Do we add to any tp that has logging on pacer logs"
	)

◆ SYSCTL_INT() [7/13]

SYSCTL_INT	(	_net_inet_tcp_hpts	,
		OID_AUTO	,
		loopmax	,
		CTLFLAG_RW	,
		&	max_pacer_loops,
		10	,
		"What is the maximum number of times the pacer will loop trying to catch up"
	)

◆ SYSCTL_INT() [8/13]

SYSCTL_INT	(	_net_inet_tcp_hpts	,
		OID_AUTO	,
		more_sleep	,
		CTLFLAG_RW	,
		&	ticks_indicate_more_sleep,
		0	,
		"If we only process this many or less on a	timeout,
		we need longer sleep on the next callout"
	)

◆ SYSCTL_INT() [9/13]

SYSCTL_INT	(	_net_inet_tcp_hpts	,
		OID_AUTO	,
		nowake_over_thresh	,
		CTLFLAG_RW	,
		&	tcp_hpts_no_wake_over_thresh,
		0	,
		"When we are over the threshold on the pacer do we prohibit wakeups?"
	)

◆ SYSCTL_INT() [10/13]

SYSCTL_INT	(	_net_inet_tcp_hpts	,
		OID_AUTO	,
		precision	,
		CTLFLAG_RW	,
		&	tcp_hpts_precision,
		120	,
		"Value for PRE() precision of callout"
	)

◆ SYSCTL_INT() [11/13]

SYSCTL_INT	(	_net_inet_tcp_hpts	,
		OID_AUTO	,
		use_assigned_cpu	,
		CTLFLAG_RW	,
		&	hpts_use_assigned_cpu,
		0	,
		"Do we start any hpts timer on the assigned cpu?"
	)

◆ SYSCTL_INT() [12/13]

SYSCTL_INT	(	_net_inet_tcp_hpts	,
		OID_AUTO	,
		use_irq	,
		CTLFLAG_RD	,
		&	tcp_use_irq_cpu,
		0	,
		"Use of irq CPU tunable"
	)

◆ SYSCTL_INT() [13/13]

SYSCTL_INT	(	_net_inet_tcp_hpts	,
		OID_AUTO	,
		use_oldest	,
		CTLFLAG_RW	,
		&	hpts_uses_oldest,
		OLDEST_THRESHOLD	,
		"Do syscalls look for the hpts that has been the longest since running (or just use cpu no if 0)?"
	)

◆ sysctl_net_inet_tcp_hpts_max_sleep()

static int sysctl_net_inet_tcp_hpts_max_sleep ( SYSCTL_HANDLER_ARGS )

static

Definition at line 385 of file tcp_hpts.c.

References dynamic_min_sleep, HPTS_MAX_SLEEP_ALLOWED, and hpts_sleep_max.

◆ sysctl_net_inet_tcp_hpts_min_sleep()

static int sysctl_net_inet_tcp_hpts_min_sleep ( SYSCTL_HANDLER_ARGS )

static

Definition at line 403 of file tcp_hpts.c.

References LOWEST_SLEEP_ALLOWED, and tcp_min_hptsi_time.

◆ SYSCTL_NODE() [1/2]

SYSCTL_NODE	(	_net_inet_tcp	,
		OID_AUTO	,
		hpts	,
		CTLFLAG_RW\|	CTLFLAG_MPSAFE,
		0	,
		"TCP Hpts controls"
	)

◆ SYSCTL_NODE() [2/2]

SYSCTL_NODE	(	_net_inet_tcp_hpts	,
		OID_AUTO	,
		stats	,
		CTLFLAG_RD\|	CTLFLAG_MPSAFE,
		0	,
		"TCP Hpts statistics"
	)

◆ SYSCTL_PROC() [1/2]

SYSCTL_PROC	(	_net_inet_tcp_hpts	,
		OID_AUTO	,
		maxsleep	,
		CTLTYPE_UINT\|CTLFLAG_RW\|	CTLFLAG_NEEDGIANT,
		&	hpts_sleep_max,
		0	,
		&	sysctl_net_inet_tcp_hpts_max_sleep,
		"IU"	,
		"Maximum time hpts will sleep"
	)

◆ SYSCTL_PROC() [2/2]

SYSCTL_PROC	(	_net_inet_tcp_hpts	,
		OID_AUTO	,
		minsleep	,
		CTLTYPE_UINT\|CTLFLAG_RW\|	CTLFLAG_NEEDGIANT,
		&	tcp_min_hptsi_time,
		0	,
		&	sysctl_net_inet_tcp_hpts_min_sleep,
		"IU"	,
		"The minimum time the hpts must sleep before processing more slots"
	)

◆ SYSINIT()

SYSINIT	(	tcphptsi	,
		SI_SUB_SOFTINTR	,
		SI_ORDER_ANY	,
		tcp_init_hptsi	,
		NULL
	)

◆ tcp_choose_hpts_to_run()

static struct tcp_hpts_entry * tcp_choose_hpts_to_run ( )

static

Definition at line 1602 of file tcp_hpts.c.

References cts_last_ran, hpts_uses_oldest, tcp_hpts_entry::p_on_queue_cnt, tcp_hptsi::rp_ent, tcp_hptsi::rp_num_hptss, tcp_get_usecs(), tcp_pace, and TSTMP_GT.

Referenced by tcp_run_hpts().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ tcp_drop_in_pkts()

static void tcp_drop_in_pkts ( struct tcpcb * tp )

static

Definition at line 1069 of file tcp_hpts.c.

References tcpcb::t_in_pkt.

◆ tcp_hpts_insert_diag()

uint32_t tcp_hpts_insert_diag	(	struct inpcb *	inp,
		uint32_t	slot,
		int32_t	line,
		struct hpts_diag *	diag
	)

Definition at line 815 of file tcp_hpts.c.

Referenced by bbr_start_hpts_timer(), and rack_start_hpts_timer().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ tcp_hpts_lock()

static struct tcp_hpts_entry * tcp_hpts_lock ( struct inpcb * inp )

static

Definition at line 534 of file tcp_hpts.c.

References HPTS_LOCK, inpcb::inp_hpts_cpu, INP_LOCK_ASSERT, tcp_hptsi::rp_ent, and tcp_pace.

Referenced by __tcp_set_hpts(), tcp_hpts_insert_diag(), and tcp_hpts_remove().

Here is the caller graph for this function:

◆ tcp_hpts_log()

static void tcp_hpts_log	(	struct tcp_hpts_entry *	hpts,
		struct tcpcb *	tp,
		struct timeval *	tv,
		int	slots_to_run,
		int	idx,
		int	from_callout
	)

static

Definition at line 446 of file tcp_hpts.c.

References tcp_log_bbr::applimited, BBR_LOG_HPTSDIAG, tcp_log_bbr::delivered, tcp_log_bbr::epoch, tcp_log_bbr::flex1, tcp_log_bbr::flex2, tcp_log_bbr::flex3, tcp_log_bbr::flex4, tcp_log_bbr::flex5, tcp_log_bbr::flex6, tcp_log_bbr::flex7, tcp_log_bbr::flex8, tcp_log_bbr::inflight, inpcb::inp_socket, tcp_log_bbr::lost, tcp_log_bbr::lt_epoch, tcp_hpts_entry::overidden_sleep, tcp_hpts_entry::p_cpu, tcp_hpts_entry::p_cur_slot, tcp_hpts_entry::p_curtick, tcp_hpts_entry::p_delayed_by, tcp_hpts_entry::p_hpts_sleep_time, tcp_hpts_entry::p_nxt_slot, tcp_hpts_entry::p_on_queue_cnt, tcp_hpts_entry::p_prev_slot, tcp_hpts_entry::p_runningslot, tcp_log_bbr::pacing_gain, tcp_log_bbr::pkt_epoch, tcp_log_bbr::pkts_out, tcp_hpts_entry::saved_curslot, tcp_hpts_entry::saved_curtick, tcp_hpts_entry::saved_prev_slot, tcpcb::t_inpcb, TCP_LOG_EVENTP, tcp_tv_to_usectick(), tcp_log_bbr::timeStamp, tcp_log_stackspecific::u_bbr, and tcp_log_bbr::use_lt_bw.

Referenced by tcp_hptsi().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ tcp_hpts_remove()

void tcp_hpts_remove ( struct inpcb * inp )

Dynamic adjustment of sleeping times is done in "new" mode where we are depending on syscall returns and lro returns to push hpts forward mainly and the timer is only a backstop.

When we are in the "new" mode i.e. conn_cnt > conn_cnt_thresh then we do a dynamic adjustment on the time we sleep. Our threshold is if the lateness of the first client served (in ticks) is greater than or equal too ticks_indicate_more_sleep (10ms or 10000 ticks). If we were that late, the actual sleep time is adjusted down by 50%. If the ticks_ran is less than ticks_indicate_more_sleep (100 ticks or 1000usecs).

Definition at line 563 of file tcp_hpts.c.

References HPTS_UNLOCK, IHPTS_MOVING, IHPTS_ONQUEUE, inpcb::inp_hpts_gencnt, inp_hpts_release(), inpcb::inp_hpts_request, inpcb::inp_hptsslot, inpcb::inp_in_hpts, INP_WLOCK_ASSERT, tcp_hpts_entry::p_hptss, tcp_hpts_entry::p_on_queue_cnt, and tcp_hpts_lock().