36#include "opt_device_polling.h"
41#include <sys/limits.h>
47#include <sys/kernel.h>
50#include <sys/sysctl.h>
51#include <sys/timeet.h>
52#include <sys/timetc.h>
54#include <machine/atomic.h>
55#include <machine/clock.h>
56#include <machine/cpu.h>
57#include <machine/smp.h>
63static void loadtimer(sbintime_t now,
int first);
66static int round_freq(
struct eventtimer *et,
int freq);
74#define ET_HW_LOCK(state) \
76 if (timer->et_flags & ET_FLAGS_PERCPU) \
77 mtx_lock_spin(&(state)->et_hw_mtx); \
79 mtx_lock_spin(&et_hw_mtx); \
82#define ET_HW_UNLOCK(state) \
84 if (timer->et_flags & ET_FLAGS_PERCPU) \
85 mtx_unlock_spin(&(state)->et_hw_mtx); \
87 mtx_unlock_spin(&et_hw_mtx); \
90static struct eventtimer *
timer = NULL;
103 0,
"Multiplier for periodic mode");
107 0,
"Run periodic events when idle");
143 return (FILTER_HANDLED);
144 state = DPCPU_PTR(timerstate);
146 CTR3(KTR_SPARE2,
"ipi at %d: now %d.%08x",
147 curcpu, (
int)(
now >> 32), (u_int)(
now & 0xffffffff));
149 return (done ? FILTER_HANDLED : FILTER_STRAY);
159 struct trapframe *frame;
164 CTR3(KTR_SPARE2,
"handle at %d: now %d.%08x",
165 curcpu, (
int)(
now >> 32), (u_int)(
now & 0xffffffff));
171 frame = curthread->td_intr_frame;
172 usermode = TRAPF_USERMODE(frame);
175 state = DPCPU_PTR(timerstate);
183 hct = DPCPU_PTR(hardclocktime);
195 if (runs && fake < 2) {
206 profclock(runs, usermode, TRAPF_PC(frame));
222 (
timer->et_flags & ET_FLAGS_PERCPU));
238 state = DPCPU_PTR(timerstate);
244 && curcpu == CPU_FIRST()
281 state = DPCPU_PTR(timerstate);
284 if ((
timer->et_flags & ET_FLAGS_PERCPU) == 0) {
286 state = DPCPU_ID_PTR(cpu, timerstate);
296 CTR4(KTR_SPARE2,
"next at %d: next %d.%08x by %d",
297 curcpu, (
int)(event >> 32), (u_int)(event & 0xffffffff), c);
316 state = DPCPU_PTR(timerstate);
317 if (et->et_flags & ET_FLAGS_PERCPU) {
327 CTR3(KTR_SPARE2,
"intr at %d: now %d.%08x",
328 curcpu, (
int)(
now >> 32), (u_int)(
now & 0xffffffff));
331#ifdef EARLY_AP_STARTUP
336#ifdef EARLY_AP_STARTUP
337 if ((et->et_flags & ET_FLAGS_PERCPU) == 0) {
339 if ((et->et_flags & ET_FLAGS_PERCPU) == 0 &&
smp_started) {
342 state = DPCPU_ID_PTR(cpu, timerstate);
366 state = DPCPU_ID_PTR(cpu, timerstate);
369 ipi_cpu(cpu, IPI_HARDCLOCK);
388 if (
timer->et_flags & ET_FLAGS_PERCPU) {
389 state = DPCPU_PTR(timerstate);
403 CTR5(KTR_SPARE2,
"load p at %d: now %d.%08x first in %d.%08x",
404 curcpu, (
int)(
now >> 32), (u_int)(
now & 0xffffffff),
405 (
int)(
new >> 32), (u_int)(
new & 0xffffffff));
412 CTR4(KTR_SPARE2,
"load at %d: next %d.%08x eq %d",
413 curcpu, (
int)(
new >> 32), (u_int)(
new & 0xffffffff), eq);
431 else if (!
periodic && (
timer->et_flags & ET_FLAGS_ONESHOT) == 0)
450 state = DPCPU_PTR(timerstate);
451 switch (atomic_load_acq_int(&state->
action)) {
458 atomic_store_rel_int(&state->
action, 0);
465 atomic_store_rel_int(&state->
action, 0);
468 if (atomic_readandclear_int(&state->
handle) && !
busy) {
483 sbintime_t
now, next;
501#ifdef EARLY_AP_STARTUP
505 state = DPCPU_ID_PTR(cpu, timerstate);
507#ifndef EARLY_AP_STARTUP
534#ifdef EARLY_AP_STARTUP
536 if ((
timer->et_flags & ET_FLAGS_PERCPU) == 0) {
546 state = DPCPU_ID_PTR(cpu, timerstate);
547 atomic_store_rel_int(&state->
action,
548 (cpu == curcpu) ? 0 : (
start ? 1 : 2));
551 ipi_all_but_self(IPI_HARDCLOCK);
558 state = DPCPU_ID_PTR(cpu, timerstate);
559 if (atomic_load_acq_int(&state->
action))
574 if (et->et_frequency != 0) {
575 div = lmax((et->et_frequency + freq / 2) / freq, 1);
576 if (et->et_flags & ET_FLAGS_POW2DIV)
577 div = 1 << (flsl(div + div / 2) - 1);
578 freq = (et->et_frequency + div / 2) / div;
580 if (et->et_min_period > SBT_1S)
581 panic(
"Event timer \"%s\" doesn't support sub-second periods!",
583 else if (et->et_min_period != 0)
584 freq = min(freq, SBT2FREQ(et->et_min_period));
585 if (et->et_max_period < SBT_1S && et->et_max_period != 0)
586 freq = max(freq, SBT2FREQ(et->et_max_period));
599 mtx_init(&
et_hw_mtx,
"et_hw_mtx", NULL, MTX_SPIN);
601 state = DPCPU_ID_PTR(cpu, timerstate);
602 mtx_init(&state->
et_hw_mtx,
"et_hw_mtx", NULL, MTX_SPIN);
612 ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC);
616 ET_FLAGS_ONESHOT, ET_FLAGS_ONESHOT);
620 ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC);
623 panic(
"No usable event timer found!");
629 else if (!
periodic && (
timer->et_flags & ET_FLAGS_ONESHOT) == 0)
631 if (
timer->et_flags & ET_FLAGS_C3STOP)
639 if (singlemul <= 0 || singlemul > 20) {
640 if (
hz >= 1500 || (
hz % 128) == 0)
688 state = DPCPU_PTR(timerstate);
696 td->td_intr_nesting_level++;
698 td->td_intr_nesting_level--;
770 || curcpu == CPU_FIRST()
774 state = DPCPU_PTR(timerstate);
779 CTR3(KTR_SPARE2,
"idle at %d: now %d.%08x",
780 curcpu, (
int)(
now >> 32), (u_int)(
now & 0xffffffff));
788 return (MAX(t -
now, 0));
801 state = DPCPU_PTR(timerstate);
808 CTR3(KTR_SPARE2,
"active at %d: now %d.%08x",
809 curcpu, (
int)(now >> 32), (u_int)(now & 0xffffffff));
812 td->td_intr_nesting_level++;
814 td->td_intr_nesting_level--;
830 et->et_frequency = newfreq;
833 et->et_frequency = newfreq;
845 CTR6(KTR_SPARE2,
"new co at %d: on %d at %d.%08x - %d.%08x",
846 curcpu, cpu, (
int)(bt_opt >> 32), (u_int)(bt_opt & 0xffffffff),
847 (
int)(
bt >> 32), (u_int)(
bt & 0xffffffff));
849 KASSERT(!CPU_ABSENT(cpu), (
"Absent CPU %d", cpu));
850 state = DPCPU_ID_PTR(cpu, timerstate);
872 if ((
timer->et_flags & ET_FLAGS_PERCPU) == 0 || cpu == curcpu) {
882 ipi_cpu(cpu, IPI_HARDCLOCK);
893 struct eventtimer *et;
903 if (error != 0 || req->newptr == NULL ||
904 strcasecmp(
buf, et->et_name) == 0) {
915 if (et->et_flags & ET_FLAGS_C3STOP)
917 if (
timer->et_flags & ET_FLAGS_C3STOP)
927 CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE,
940 if (error != 0 || req->newptr == NULL)
950 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE,
958DB_SHOW_COMMAND(clocksource, db_show_clocksource)
964 st = DPCPU_ID_PTR(c, timerstate);
966 "CPU %2d: action %d handle %d ipi %d idle %d\n"
967 " now %#jx nevent %#jx (%jd)\n"
968 " ntick %#jx (%jd) nhard %#jx (%jd)\n"
969 " nstat %#jx (%jd) nprof %#jx (%jd)\n"
970 " ncall %#jx (%jd) ncallopt %#jx (%jd)\n",
971 c,
st->action,
st->handle,
st->ipi,
st->idle,
973 (uintmax_t)
st->nextevent,
975 (uintmax_t)
st->nexttick,
977 (uintmax_t)
st->nexthard,
979 (uintmax_t)
st->nextstat,
981 (uintmax_t)
st->nextprof,
983 (uintmax_t)
st->nextcall,
985 (uintmax_t)
st->nextcallopt,
static struct bt_table bt
static struct bt_table st
void hardclock_sync(int cpu)
void profclock(int cnt, int usermode, uintfptr_t pc)
void statclock(int cnt, int usermode)
void hardclock(int cnt, int usermode)
static sbintime_t nexttick
static void configtimer(int start)
SYSCTL_INT(_kern_eventtimer, OID_AUTO, singlemul, CTLFLAG_RWTUN, &singlemul, 0, "Multiplier for periodic mode")
void cpu_stopprofclock(void)
static int handleevents(sbintime_t now, int fake)
void cpu_new_callout(int cpu, sbintime_t bt, sbintime_t bt_opt)
sbintime_t cpu_idleclock(void)
static int doconfigtimer(void)
void cpu_et_frequency(struct eventtimer *et, uint64_t newfreq)
static void timercb(struct eventtimer *et, void *arg)
static int round_freq(struct eventtimer *et, int freq)
#define ET_HW_LOCK(state)
static sbintime_t getnextevent(void)
#define ET_HW_UNLOCK(state)
DPCPU_DEFINE_STATIC(struct pcpu_state, timerstate)
static char timername[32]
static int sysctl_kern_eventtimer_periodic(SYSCTL_HANDLER_ARGS)
static struct mtx et_hw_mtx
static void loadtimer(sbintime_t now, int first)
TUNABLE_INT("kern.eventtimer.periodic", &want_periodic)
static sbintime_t getnextcpuevent(int idle)
TUNABLE_STR("kern.eventtimer.timer", timername, sizeof(timername))
static int sysctl_kern_eventtimer_timer(SYSCTL_HANDLER_ARGS)
static sbintime_t profperiod
SYSCTL_UINT(_kern_eventtimer, OID_AUTO, idletick, CTLFLAG_RWTUN, &idletick, 0, "Run periodic events when idle")
static struct eventtimer * timer
void cpu_initclocks_ap(void)
static sbintime_t timerperiod
static void setuptimer(void)
SYSCTL_PROC(_kern_eventtimer, OID_AUTO, timer, CTLTYPE_STRING|CTLFLAG_RW|CTLFLAG_MPSAFE, 0, 0, sysctl_kern_eventtimer_timer, "A", "Chosen event timer")
void cpu_startprofclock(void)
void cpu_initclocks_bsp(void)
static sbintime_t statperiod
void cpu_activeclock(void)
DPCPU_DEFINE(sbintime_t, hardclocktime)
int et_init(struct eventtimer *et, et_event_cb_t *event, et_deregister_cb_t *deregister, void *arg)
int et_start(struct eventtimer *et, sbintime_t first, sbintime_t period)
struct eventtimer * et_find(const char *name, int check, int want)
int et_free(struct eventtimer *et)
int et_stop(struct eventtimer *et)
void panic(const char *fmt,...)
int sysctl_handle_int(SYSCTL_HANDLER_ARGS)
int sysctl_handle_string(SYSCTL_HANDLER_ARGS)
void callout_process(sbintime_t now)
int snprintf(char *str, size_t size, const char *format,...)