tcp_hpts.h File Reference

This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Data Structures
struct	hpts_diag

Macros
#define	HPTS_TICKS_PER_SLOT   10
#define	HPTS_MS_TO_SLOTS(x)   ((x * 100) + 1)
#define	HPTS_USEC_TO_SLOTS(x)   ((x+9) /10)
#define	HPTS_USEC_IN_SEC   1000000
#define	HPTS_MSEC_IN_SEC   1000
#define	HPTS_USEC_IN_MSEC   1000
#define	PACE_TMR_DELACK   0x01 /* Delayed ack timer running */
#define	PACE_TMR_RACK   0x02 /* RACK timer running */
#define	PACE_TMR_TLP   0x04 /* TLP timer running */
#define	PACE_TMR_RXT   0x08 /* Retransmit timer running */
#define	PACE_TMR_PERSIT   0x10 /* Persists timer running */
#define	PACE_TMR_KEEP   0x20 /* Keep alive timer running */
#define	PACE_PKT_OUTPUT   0x40 /* Output Packets being paced */
#define	PACE_TMR_MASK   (PACE_TMR_KEEP|PACE_TMR_PERSIT|PACE_TMR_RXT|PACE_TMR_TLP|PACE_TMR_RACK|PACE_TMR_DELACK)
#define	DEFAULT_CONNECTION_THESHOLD   100
#define	LOWEST_SLEEP_ALLOWED   50
#define	DEFAULT_MIN_SLEEP
#define	DYNAMIC_MIN_SLEEP   DEFAULT_MIN_SLEEP
#define	DYNAMIC_MAX_SLEEP   100000 /* 100ms */
#define	OLDEST_THRESHOLD   1200
#define	TICKS_INDICATE_MORE_SLEEP   100 /* This would be 1ms */
#define	TICKS_INDICATE_LESS_SLEEP   1000 /* This would indicate 10ms */
#define	tcp_hpts_insert(inp, slot)    tcp_hpts_insert_diag((inp), (slot), __LINE__, NULL)
#define	tcp_set_hpts(a)   __tcp_set_hpts(a, __LINE__)

Functions
void	tcp_hpts_remove (struct inpcb *)
bool	tcp_in_hpts (struct inpcb *)
uint32_t	tcp_hpts_insert_diag (struct inpcb *inp, uint32_t slot, int32_t line, struct hpts_diag *diag)
void	__tcp_set_hpts (struct inpcb *inp, int32_t line)
void	tcp_set_inp_to_drop (struct inpcb *inp, uint16_t reason)
void	tcp_run_hpts (void)
uint16_t	hpts_random_cpu (struct inpcb *inp)
static __inline uint32_t	tcp_tv_to_hptstick (const struct timeval *sv)
static __inline uint32_t	tcp_tv_to_usectick (const struct timeval *sv)
static __inline uint32_t	tcp_tv_to_mssectick (const struct timeval *sv)
static __inline uint64_t	tcp_tv_to_lusectick (const struct timeval *sv)
static __inline uint32_t	tcp_gethptstick (struct timeval *sv)
static __inline uint32_t	tcp_get_usecs (struct timeval *tv)

Variables
int32_t	tcp_min_hptsi_time

Macro Definition Documentation

DEFAULT_CONNECTION_THESHOLD

#define DEFAULT_CONNECTION_THESHOLD   100

Definition at line 71 of file tcp_hpts.h.

DEFAULT_MIN_SLEEP

#define DEFAULT_MIN_SLEEP

Value:

                                 250    /* How many usec's is default for hpts sleep
                                 * this determines min granularity of the
                                 * hpts. If 1, granularity is 10useconds at
                                 * the cost of more CPU (context switching).
                                 * Note do not set this to 0.
                                 */

Definition at line 86 of file tcp_hpts.h.

DYNAMIC_MAX_SLEEP

#define DYNAMIC_MAX_SLEEP   100000 /* 100ms */

Definition at line 88 of file tcp_hpts.h.

DYNAMIC_MIN_SLEEP

#define DYNAMIC_MIN_SLEEP   DEFAULT_MIN_SLEEP

Definition at line 87 of file tcp_hpts.h.

HPTS_MS_TO_SLOTS

#define HPTS_MS_TO_SLOTS

(

x

)

((x * 100) + 1)

Definition at line 33 of file tcp_hpts.h.

HPTS_MSEC_IN_SEC

#define HPTS_MSEC_IN_SEC   1000

Definition at line 36 of file tcp_hpts.h.

HPTS_TICKS_PER_SLOT

#define HPTS_TICKS_PER_SLOT   10

Definition at line 32 of file tcp_hpts.h.

HPTS_USEC_IN_MSEC

#define HPTS_USEC_IN_MSEC   1000

Definition at line 37 of file tcp_hpts.h.

HPTS_USEC_IN_SEC

#define HPTS_USEC_IN_SEC   1000000

Definition at line 35 of file tcp_hpts.h.

HPTS_USEC_TO_SLOTS

#define HPTS_USEC_TO_SLOTS

(

x

)

((x+9) /10)

Definition at line 34 of file tcp_hpts.h.

LOWEST_SLEEP_ALLOWED

#define LOWEST_SLEEP_ALLOWED   50

Definition at line 85 of file tcp_hpts.h.

OLDEST_THRESHOLD

#define OLDEST_THRESHOLD   1200

Definition at line 90 of file tcp_hpts.h.

PACE_PKT_OUTPUT

#define PACE_PKT_OUTPUT   0x40 /* Output Packets being paced */

Definition at line 68 of file tcp_hpts.h.

PACE_TMR_DELACK

#define PACE_TMR_DELACK   0x01 /* Delayed ack timer running */

Definition at line 62 of file tcp_hpts.h.

PACE_TMR_KEEP

#define PACE_TMR_KEEP   0x20 /* Keep alive timer running */

Definition at line 67 of file tcp_hpts.h.

PACE_TMR_MASK

#define PACE_TMR_MASK   (PACE_TMR_KEEP|PACE_TMR_PERSIT|PACE_TMR_RXT|PACE_TMR_TLP|PACE_TMR_RACK|PACE_TMR_DELACK)

Definition at line 69 of file tcp_hpts.h.

PACE_TMR_PERSIT

#define PACE_TMR_PERSIT   0x10 /* Persists timer running */

Definition at line 66 of file tcp_hpts.h.

PACE_TMR_RACK

#define PACE_TMR_RACK   0x02 /* RACK timer running */

Definition at line 63 of file tcp_hpts.h.

PACE_TMR_RXT

#define PACE_TMR_RXT   0x08 /* Retransmit timer running */

Definition at line 65 of file tcp_hpts.h.

PACE_TMR_TLP

#define PACE_TMR_TLP   0x04 /* TLP timer running */

Definition at line 64 of file tcp_hpts.h.

tcp_hpts_insert

#define tcp_hpts_insert	(		inp,
			slot
	)		tcp_hpts_insert_diag((inp), (slot), __LINE__, NULL)

Definition at line 141 of file tcp_hpts.h.

tcp_set_hpts

#define tcp_set_hpts

(

a

)

__tcp_set_hpts(a, __LINE__)

Definition at line 145 of file tcp_hpts.h.

TICKS_INDICATE_LESS_SLEEP

#define TICKS_INDICATE_LESS_SLEEP   1000 /* This would indicate 10ms */

Definition at line 93 of file tcp_hpts.h.

TICKS_INDICATE_MORE_SLEEP

#define TICKS_INDICATE_MORE_SLEEP   100 /* This would be 1ms */

Definition at line 92 of file tcp_hpts.h.

Function Documentation

__tcp_set_hpts()

void __tcp_set_hpts	(	struct inpcb *	inp,
		int32_t	line
	)

Definition at line 1517 of file tcp_hpts.c.

References hpts_cpuid(), inpcb::inp_hpts_cpu, inpcb::inp_hpts_cpu_set, inpcb::inp_in_hpts, INP_WLOCK_ASSERT, tcp_hpts_entry::p_mtx, and tcp_hpts_lock().

Here is the call graph for this function:

hpts_random_cpu()

uint16_t hpts_random_cpu

(

struct inpcb *

inp

)

Definition at line 994 of file tcp_hpts.c.

References inpcb::inp_hpts_cpu, inpcb::inp_hpts_cpu_set, tcp_hptsi::rp_num_hptss, and tcp_pace.

Referenced by hpts_cpuid(), and in_pcballoc().

Here is the caller graph for this function:

tcp_get_usecs()

static __inline uint32_t tcp_get_usecs ( struct timeval *  tv )

static

Definition at line 198 of file tcp_hpts.h.

Referenced by bbr_do_segment(), bbr_do_segment_nounlock(), bbr_google_mode_off(), bbr_google_mode_on(), bbr_init(), bbr_log_ack_event(), bbr_log_flowend(), bbr_output(), bbr_remxt_tmr(), bbr_set_sockopt(), ctf_log_sack_filter(), ctf_process_inbound_raw(), cubic_log_hystart_event(), newreno_log_hystart_event(), rack_ack_received(), rack_do_compressed_ack_processing(), rack_do_segment(), rack_do_segment_nounlock(), rack_enter_persist(), rack_exit_persist(), rack_fastack(), rack_init(), rack_log_ack(), rack_log_alt_to_to_cancel(), rack_log_doseg_done(), rack_log_dsack_event(), rack_log_fsb(), rack_log_hdwr_pacing(), rack_log_input_packet(), rack_log_map_chg(), rack_log_pacing_delay_calc(), rack_log_progress_event(), rack_log_retran_reason(), rack_log_rtt_sample(), rack_log_rtt_sample_calc(), rack_log_rtt_shrinks(), rack_log_rtt_upd(), rack_log_timely(), rack_log_to_event(), rack_log_to_processing(), rack_log_to_prr(), rack_log_to_start(), rack_log_type_just_return(), rack_log_type_pacing_sizes(), rack_log_wakeup(), rack_output(), rack_post_recovery(), rack_process_ack(), rack_remxt_tmr(), rack_set_cc_pacing(), rack_start_gp_measurement(), rack_start_hpts_timer(), rack_strike_dupack(), rack_timeout_keepalive(), rack_timeout_persist(), rack_timer_audit(), rack_timer_cancel(), rack_undo_cc_pacing(), rack_update_rtt(), tcp_choose_hpts_to_run(), and tcp_respond().

tcp_gethptstick()

static __inline uint32_t tcp_gethptstick ( struct timeval *  sv )

static

Definition at line 187 of file tcp_hpts.h.

Referenced by tcp_hptsi(), and tcp_init_hptsi().

Here is the caller graph for this function:

tcp_hpts_insert_diag()

uint32_t tcp_hpts_insert_diag	(	struct inpcb *	inp,
		uint32_t	slot,
		int32_t	line,
		struct hpts_diag *	diag
	)

Definition at line 815 of file tcp_hpts.c.

References hpts_diag::co_ret, hpts_diag::have_slept, hpts_diag::hpts_sleep_time, hpts_slot(), hpts_slots_diff(), HPTS_TICKS_PER_SLOT, hpts_timeout_swi(), HPTS_UNLOCK, hpts_use_assigned_cpu, HPTS_USEC_IN_SEC, IHPTS_MOVING, INP_DROPPED, inpcb::inp_flags, inp_hpts_insert(), inpcb::inp_hpts_request, inpcb::inp_hptsslot, hpts_diag::inp_hptsslot, inpcb::inp_in_hpts, INP_TIMEWAIT, INP_WLOCK_ASSERT, max_slots_available(), hpts_diag::maxslots, hpts_diag::need_new_to, tcp_hpts_entry::p_cpu, tcp_hpts_entry::p_cur_slot, hpts_diag::p_cur_slot, tcp_hpts_entry::p_curtick, hpts_diag::p_curtick, tcp_hpts_entry::p_direct_wake, tcp_hpts_entry::p_hpts_active, hpts_diag::p_hpts_active, p_hpts_sleep_time, tcp_hpts_entry::p_hpts_sleep_time, tcp_hpts_entry::p_lasttick, hpts_diag::p_lasttick, tcp_hpts_entry::p_nxt_slot, hpts_diag::p_nxt_slot, tcp_hpts_entry::p_on_min_sleep, hpts_diag::p_on_min_sleep, tcp_hpts_entry::p_prev_slot, hpts_diag::p_prev_slot, tcp_hpts_entry::p_runningslot, hpts_diag::p_runningslot, tcp_hpts_entry::p_wheel_complete, hpts_diag::slot_remaining, hpts_diag::slot_req, tcp_bind_threads, tcp_hpts_lock(), tcp_hpts_precision, tcp_in_hpts(), tcp_tv_to_hptstick(), tcp_wakehpts(), tick_to_wheel(), hpts_diag::wheel_cts, hpts_diag::wheel_slot, and hpts_diag::yet_to_sleep.

Referenced by bbr_start_hpts_timer(), and rack_start_hpts_timer().

Here is the call graph for this function:

Here is the caller graph for this function:

tcp_hpts_remove()

void tcp_hpts_remove

(

struct inpcb *

inp

)

Dynamic adjustment of sleeping times is done in "new" mode where we are depending on syscall returns and lro returns to push hpts forward mainly and the timer is only a backstop.

When we are in the "new" mode i.e. conn_cnt > conn_cnt_thresh then we do a dynamic adjustment on the time we sleep. Our threshold is if the lateness of the first client served (in ticks) is greater than or equal too ticks_indicate_more_sleep (10ms or 10000 ticks). If we were that late, the actual sleep time is adjusted down by 50%. If the ticks_ran is less than ticks_indicate_more_sleep (100 ticks or 1000usecs).

Definition at line 563 of file tcp_hpts.c.

References HPTS_UNLOCK, IHPTS_MOVING, IHPTS_ONQUEUE, inpcb::inp_hpts_gencnt, inp_hpts_release(), inpcb::inp_hpts_request, inpcb::inp_hptsslot, inpcb::inp_in_hpts, INP_WLOCK_ASSERT, tcp_hpts_entry::p_hptss, tcp_hpts_entry::p_on_queue_cnt, and tcp_hpts_lock().

Referenced by bbr_do_segment_nounlock(), bbr_exit_persist(), bbr_output_wtime(), bbr_timer_audit(), bbr_timer_cancel(), rack_do_goodput_measurement(), rack_do_segment_nounlock(), rack_exit_persist(), rack_output(), rack_timer_audit(), rack_timer_cancel(), tcp_close(), tcp_ctloutput_set(), and tcp_twstart().

Here is the call graph for this function:

Here is the caller graph for this function:

tcp_in_hpts()

bool tcp_in_hpts

(

struct inpcb *

inp

)

Definition at line 610 of file tcp_hpts.c.

References IHPTS_ONQUEUE, and inpcb::inp_in_hpts.

Referenced by bbr_cong_signal(), bbr_do_segment_nounlock(), bbr_exit_persist(), bbr_fill_in_logging_data(), bbr_output_wtime(), bbr_start_hpts_timer(), bbr_timer_audit(), bbr_timer_cancel(), rack_do_goodput_measurement(), rack_do_segment_nounlock(), rack_exit_persist(), rack_fast_output(), rack_fast_rsm_output(), rack_log_ack(), rack_log_doseg_done(), rack_log_fsb(), rack_log_input_packet(), rack_log_map_chg(), rack_log_progress_event(), rack_log_retran_reason(), rack_log_rtt_upd(), rack_log_to_cancel(), rack_log_to_event(), rack_log_to_start(), rack_log_type_bbrsnd(), rack_log_type_just_return(), rack_output(), rack_start_hpts_timer(), rack_timer_audit(), rack_timer_cancel(), and tcp_hpts_insert_diag().

Here is the caller graph for this function:

tcp_run_hpts()

void tcp_run_hpts

(

void

)

Definition at line 1642 of file tcp_hpts.c.

References __tcp_run_hpts(), and tcp_choose_hpts_to_run().

Referenced by tcp_lro_flush_all().

Here is the call graph for this function:

Here is the caller graph for this function:

tcp_set_inp_to_drop()

void tcp_set_inp_to_drop	(	struct inpcb *	inp,
		uint16_t	reason
	)

tcp_tv_to_hptstick()

static __inline uint32_t tcp_tv_to_hptstick ( const struct timeval *  sv )

static

Definition at line 162 of file tcp_hpts.h.

Referenced by tcp_hpts_insert_diag().

Here is the caller graph for this function:

tcp_tv_to_lusectick()

static __inline uint64_t tcp_tv_to_lusectick ( const struct timeval *  sv )

static

Definition at line 180 of file tcp_hpts.h.

References HPTS_MSEC_IN_SEC, and HPTS_USEC_IN_MSEC.

tcp_tv_to_mssectick()

static __inline uint32_t tcp_tv_to_mssectick ( const struct timeval *  sv )

static

Definition at line 174 of file tcp_hpts.h.

References HPTS_USEC_IN_SEC.

Referenced by bbr_do_close_wait(), bbr_do_closing(), bbr_do_established(), bbr_do_fastnewdata(), bbr_do_fin_wait_1(), bbr_do_fin_wait_2(), bbr_do_lastack(), bbr_do_segment_nounlock(), bbr_do_syn_recv(), bbr_do_syn_sent(), bbr_log_ack(), bbr_log_rtt_sample(), bbr_output_wtime(), bbr_update_rtt(), rack_do_compressed_ack_processing(), rack_do_segment_nounlock(), rack_init(), and rack_output().

Here is the caller graph for this function:

tcp_tv_to_usectick()

static __inline uint32_t tcp_tv_to_usectick ( const struct timeval *  sv )

static

Definition at line 168 of file tcp_hpts.h.

References HPTS_TICKS_PER_SLOT.

Referenced by bbr_do_segment_nounlock(), bbr_log_ack_event(), bbr_output_wtime(), rack_do_compressed_ack_processing(), rack_do_goodput_measurement(), rack_do_segment_nounlock(), rack_enough_for_measurement(), rack_init(), rack_log_input_packet(), rack_log_rtt_upd(), rack_need_set_test(), rack_process_ack(), rack_start_gp_measurement(), rack_update_rtt(), tcp_hpts_log(), tcp_hptsi(), and tcp_init_hptsi().

Here is the caller graph for this function:

Variable Documentation

tcp_min_hptsi_time

int32_t tcp_min_hptsi_time

extern

Definition at line 253 of file tcp_hpts.c.

Referenced by bbr_log_type_tsosize(), sysctl_net_inet_tcp_hpts_min_sleep(), tcp_hpts_thread(), and tcp_init_hptsi().