41#include <sys/kernel.h>
43#include <sys/limits.h>
45#include <sys/lock_profile.h>
46#include <sys/malloc.h>
53#include <sys/sysctl.h>
59#include <machine/cpufunc.h>
83lock_init(
struct lock_object *lock,
struct lock_class *
class,
const char *
name,
89 KASSERT(
flags & LO_NEW || !lock_initialized(lock),
90 (
"lock \"%s\" %p already initialized",
name, lock));
93 for (i = 0; i < LOCK_CLASS_MAX; i++)
95 lock->lo_flags = i << LO_CLASSSHIFT;
98 KASSERT(i < LOCK_CLASS_MAX, (
"unknown lock class %p",
class));
101 lock->lo_name =
name;
102 lock->lo_flags |=
flags | LO_INITIALIZED;
103 LOCK_LOG_INIT(lock, 0);
111 KASSERT(lock_initialized(lock), (
"lock %p is not initialized", lock));
112 WITNESS_DESTROY(lock);
113 LOCK_LOG_DESTROY(lock, 0);
114 lock->lo_flags &= ~LO_INITIALIZED;
117static SYSCTL_NODE(_debug, OID_AUTO, lock, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL,
120 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL,
134 struct lock_delay_config *
lc = la->config;
137 for (i = la->delay; i > 0; i--)
139 la->spin_cnt += la->delay;
142 if (__predict_false(la->delay >
lc->max))
146 SDT_PROBE1(lock, , , starvation, la->delay);
148 la->delay =
lc->base;
197DB_SHOW_COMMAND(lock, db_show_lock)
199 struct lock_object *lock;
200 struct lock_class *
class;
204 lock = (
struct lock_object *)
addr;
205 if (LO_CLASSINDEX(lock) > LOCK_CLASS_MAX) {
206 db_printf(
"Unknown lock class: %d\n", LO_CLASSINDEX(lock));
209 class = LOCK_CLASS(lock);
210 db_printf(
" class: %s\n", class->lc_name);
211 db_printf(
" name: %s\n", lock->lo_name);
212 class->lc_ddb_show(lock);
222struct lock_profile_object {
223 LIST_ENTRY(lock_profile_object) lpo_link;
224 struct lock_object *lpo_obj;
225 const char *lpo_file;
229 uint64_t lpo_acqtime;
230 uint64_t lpo_waittime;
231 u_int lpo_contest_locking;
238 SLIST_ENTRY(lock_prof) link;
239 struct lock_class *
class;
244 uintmax_t cnt_wait_max;
249 uintmax_t cnt_contest_locking;
254#define LPROF_HASH_SIZE 4096
255#define LPROF_HASH_MASK (LPROF_HASH_SIZE - 1)
256#define LPROF_CACHE_SIZE 4096
264struct lock_prof_type {
265 struct lphead lpt_lpalloc;
266 struct lpohead lpt_lpoalloc;
267 struct lphead lpt_hash[LPROF_HASH_SIZE];
268 struct lock_prof lpt_prof[LPROF_CACHE_SIZE];
269 struct lock_profile_object lpt_objs[LPROF_CACHE_SIZE];
272struct lock_prof_cpu {
273 struct lock_prof_type lpc_types[2];
277#define LP_CPU_SELF (DPCPU_PTR(lp))
278#define LP_CPU(cpu) (DPCPU_ID_PTR((cpu), lp))
282static volatile int lock_prof_resetting;
284#define LPROF_SBUF_SIZE 256
286static int lock_prof_rejected;
287static int lock_prof_skipspin;
289#ifndef USE_CPU_NANOSECONDS
298 ns =
bt.sec * (uint64_t)1000000000;
299 ns += ((uint64_t)1000000000 * (uint32_t)(
bt.frac >> 32)) >> 32;
305lock_prof_init_type(
struct lock_prof_type *
type)
309 SLIST_INIT(&
type->lpt_lpalloc);
310 LIST_INIT(&
type->lpt_lpoalloc);
311 for (i = 0; i < LPROF_CACHE_SIZE; i++) {
312 SLIST_INSERT_HEAD(&
type->lpt_lpalloc, &
type->lpt_prof[i],
314 LIST_INSERT_HEAD(&
type->lpt_lpoalloc, &
type->lpt_objs[i],
320lock_prof_init(
void *arg)
325 lock_prof_init_type(&LP_CPU(cpu)->lpc_types[0]);
326 lock_prof_init_type(&LP_CPU(cpu)->lpc_types[1]);
329SYSINIT(lockprof, SI_SUB_SMP, SI_ORDER_ANY, lock_prof_init, NULL);
332lock_prof_reset_wait(
void)
339 while (lock_prof_resetting)
346 struct lock_prof_cpu *lpc;
356 atomic_store_rel_int(&lock_prof_resetting, 1);
357 enabled = lock_prof_enable;
358 lock_prof_enable = 0;
373 for (i = 0; i < LPROF_CACHE_SIZE; i++) {
374 LIST_REMOVE(&lpc->lpc_types[0].lpt_objs[i], lpo_link);
375 LIST_REMOVE(&lpc->lpc_types[1].lpt_objs[i], lpo_link);
380 bzero(lpc,
sizeof(*lpc));
381 lock_prof_init_type(&lpc->lpc_types[0]);
382 lock_prof_init_type(&lpc->lpc_types[1]);
387 atomic_store_rel_int(&lock_prof_resetting, 0);
388 lock_prof_enable = enabled;
392lock_prof_output(
struct lock_prof *lp,
struct sbuf *sb)
396 for (p = lp->file; p != NULL && strncmp(p,
"../", 3) == 0; p += 3);
398 "%8ju %9ju %11ju %11ju %11ju %6ju %6ju %2ju %6ju %s:%d (%s:%s)\n",
399 lp->cnt_max / 1000, lp->cnt_wait_max / 1000, lp->cnt_tot / 1000,
400 lp->cnt_wait / 1000, lp->cnt_cur,
401 lp->cnt_cur == 0 ? (uintmax_t)0 :
402 lp->cnt_tot / (lp->cnt_cur * 1000),
403 lp->cnt_cur == 0 ? (uintmax_t)0 :
404 lp->cnt_wait / (lp->cnt_cur * 1000),
405 (uintmax_t)0, lp->cnt_contest_locking,
406 p, lp->line, lp->class->lc_name, lp->name);
410lock_prof_sum(
struct lock_prof *match,
struct lock_prof *dst,
int hash,
413 struct lock_prof_type *
type;
417 dst->file = match->file;
418 dst->line = match->line;
419 dst->class = match->class;
420 dst->name = match->name;
423 type = &LP_CPU(cpu)->lpc_types[spin];
424 SLIST_FOREACH(l, &
type->lpt_hash[hash], link) {
427 if (l->file != match->file || l->line != match->line ||
428 l->name != match->name)
431 if (l->cnt_max > dst->cnt_max)
432 dst->cnt_max = l->cnt_max;
433 if (l->cnt_wait_max > dst->cnt_wait_max)
434 dst->cnt_wait_max = l->cnt_wait_max;
435 dst->cnt_tot += l->cnt_tot;
436 dst->cnt_wait += l->cnt_wait;
437 dst->cnt_cur += l->cnt_cur;
438 dst->cnt_contest_locking += l->cnt_contest_locking;
444lock_prof_type_stats(
struct lock_prof_type *
type,
struct sbuf *sb,
int spin,
450 for (i = 0; i < LPROF_HASH_SIZE; ++i) {
451 SLIST_FOREACH(l, &
type->lpt_hash[i], link) {
452 struct lock_prof lp = {};
456 lock_prof_sum(l, &lp, i, spin, t);
457 lock_prof_output(&lp, sb);
463dump_lock_prof_stats(SYSCTL_HANDLER_ARGS)
473 sbuf_printf(sb,
"\n%8s %9s %11s %11s %11s %6s %6s %2s %6s %s\n",
474 "max",
"wait_max",
"total",
"wait_total",
"count",
"avg",
"wait_avg",
"cnt_hold",
"cnt_lock",
"name");
475 enabled = lock_prof_enable;
476 lock_prof_enable = 0;
484 lock_prof_type_stats(&LP_CPU(cpu)->lpc_types[0], sb, 0, t);
485 lock_prof_type_stats(&LP_CPU(cpu)->lpc_types[1], sb, 1, t);
487 atomic_thread_fence_rel();
488 lock_prof_enable = enabled;
493 error = SYSCTL_OUT(req,
"", 1);
499enable_lock_prof(SYSCTL_HANDLER_ARGS)
503 v = lock_prof_enable;
507 if (req->newptr == NULL)
509 if (v == lock_prof_enable)
513 lock_prof_enable = !!v;
519reset_lock_prof_stats(SYSCTL_HANDLER_ARGS)
527 if (req->newptr == NULL)
536static struct lock_prof *
537lock_profile_lookup(
struct lock_object *lo,
int spin,
const char *file,
540 const char *unknown =
"(unknown)";
541 struct lock_prof_type *
type;
542 struct lock_prof *lp;
548 if (p == NULL || *p ==
'\0')
550 hash = (uintptr_t)lo->lo_name * 31 + (uintptr_t)p * 31 + line;
551 hash &= LPROF_HASH_MASK;
552 type = &LP_CPU_SELF->lpc_types[spin];
553 head = &
type->lpt_hash[hash];
554 SLIST_FOREACH(lp, head, link) {
555 if (lp->line == line && lp->file == p &&
556 lp->name == lo->lo_name)
559 lp = SLIST_FIRST(&
type->lpt_lpalloc);
561 lock_prof_rejected++;
564 SLIST_REMOVE_HEAD(&
type->lpt_lpalloc, link);
567 lp->class = LOCK_CLASS(lo);
568 lp->name = lo->lo_name;
569 SLIST_INSERT_HEAD(&
type->lpt_hash[hash], lp, link);
573static struct lock_profile_object *
574lock_profile_object_lookup(
struct lock_object *lo,
int spin,
const char *file,
577 struct lock_profile_object *l;
578 struct lock_prof_type *
type;
579 struct lpohead *head;
581 head = &curthread->td_lprof[spin];
582 LIST_FOREACH(l, head, lpo_link)
583 if (l->lpo_obj == lo && l->lpo_file == file &&
586 type = &LP_CPU_SELF->lpc_types[spin];
587 l = LIST_FIRST(&
type->lpt_lpoalloc);
589 lock_prof_rejected++;
592 LIST_REMOVE(l, lpo_link);
597 LIST_INSERT_HEAD(head, l, lpo_link);
603lock_profile_obtain_lock_success(
struct lock_object *lo,
bool spin,
604 int contested, uint64_t waittime,
const char *file,
int line)
606 struct lock_profile_object *l;
608#ifdef LOCK_PROFILING_DEBUG_SPIN
609 bool is_spin = (LOCK_CLASS(lo)->lc_flags & LC_SPINLOCK);
610 if ((spin && !is_spin) || (!spin && is_spin))
611 printf(
"%s: lock %s spin mismatch (arg %d, flag %d)\n", __func__,
612 lo->lo_name, spin, is_spin);
616 if (!lock_prof_enable || (lo->lo_flags & LO_NOPROFILE))
618 if (lock_contested_only && !contested)
620 if (spin && lock_prof_skipspin == 1)
623 if (SCHEDULER_STOPPED())
628 if (lock_prof_enable == 0)
630 l = lock_profile_object_lookup(lo, spin, file, line);
634 if (++l->lpo_ref > 1)
636 l->lpo_contest_locking = contested;
637 l->lpo_acqtime = nanoseconds();
638 if (waittime && (l->lpo_acqtime > waittime))
639 l->lpo_waittime = l->lpo_acqtime - waittime;
646 atomic_thread_fence_rel();
651lock_profile_thread_exit(
struct thread *td)
654 struct lock_profile_object *l;
656 MPASS(curthread->td_critnest == 0);
662 lock_prof_reset_wait();
664 LIST_FOREACH(l, &td->td_lprof[0], lpo_link)
665 printf(
"thread still holds lock acquired at %s:%d\n",
666 l->lpo_file, l->lpo_line);
667 LIST_FOREACH(l, &td->td_lprof[1], lpo_link)
668 printf(
"thread still holds lock acquired at %s:%d\n",
669 l->lpo_file, l->lpo_line);
671 MPASS(LIST_FIRST(&td->td_lprof[0]) == NULL);
672 MPASS(LIST_FIRST(&td->td_lprof[1]) == NULL);
676lock_profile_release_lock(
struct lock_object *lo,
bool spin)
678 struct lock_profile_object *l;
679 struct lock_prof_type *
type;
680 struct lock_prof *lp;
681 uint64_t curtime, holdtime;
682 struct lpohead *head;
684#ifdef LOCK_PROFILING_DEBUG_SPIN
685 bool is_spin = (LOCK_CLASS(lo)->lc_flags & LC_SPINLOCK);
686 if ((spin && !is_spin) || (!spin && is_spin))
687 printf(
"%s: lock %s spin mismatch (arg %d, flag %d)\n", __func__,
688 lo->lo_name, spin, is_spin);
691 if (lo->lo_flags & LO_NOPROFILE)
693 head = &curthread->td_lprof[spin];
694 if (LIST_FIRST(head) == NULL)
696 if (SCHEDULER_STOPPED())
700 if (lock_prof_enable == 0 && lock_prof_resetting == 1)
706 LIST_FOREACH(l, head, lpo_link)
707 if (l->lpo_obj == lo)
711 if (--l->lpo_ref > 0)
713 lp = lock_profile_lookup(lo, spin, l->lpo_file, l->lpo_line);
716 curtime = nanoseconds();
717 if (curtime < l->lpo_acqtime)
719 holdtime = curtime - l->lpo_acqtime;
725 if (holdtime > lp->cnt_max)
726 lp->cnt_max = holdtime;
727 if (l->lpo_waittime > lp->cnt_wait_max)
728 lp->cnt_wait_max = l->lpo_waittime;
729 lp->cnt_tot += holdtime;
730 lp->cnt_wait += l->lpo_waittime;
731 lp->cnt_contest_locking += l->lpo_contest_locking;
732 lp->cnt_cur += l->lpo_cnt;
734 LIST_REMOVE(l, lpo_link);
735 type = &LP_CPU_SELF->lpc_types[spin];
736 LIST_INSERT_HEAD(&
type->lpt_lpoalloc, l, lpo_link);
741 atomic_thread_fence_rel();
746 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL,
748SYSCTL_INT(_debug_lock_prof, OID_AUTO, skipspin, CTLFLAG_RW,
749 &lock_prof_skipspin, 0,
"Skip profiling on spinlocks.");
750SYSCTL_INT(_debug_lock_prof, OID_AUTO, rejected, CTLFLAG_RD,
751 &lock_prof_rejected, 0,
"Number of rejected profiling records");
752SYSCTL_INT(_debug_lock_prof, OID_AUTO, contested_only, CTLFLAG_RW,
753 &lock_contested_only, 0,
"Only profile contested acquires");
755 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0,
756 dump_lock_prof_stats,
"A",
757 "Lock profiling statistics");
759 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, NULL, 0,
760 reset_lock_prof_stats,
"I",
761 "Reset lock profiling statistics");
763 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, 0,
764 enable_lock_prof,
"I",
765 "Enable lock profiling");
device_property_type_t type
SYSCTL_PROC(_kern_binmisc, OID_AUTO, add, CTLFLAG_MPSAFE|CTLTYPE_STRUCT|CTLFLAG_WR, NULL, IBC_ADD, sysctl_kern_binmisc, "S,ximgact_binmisc_entry", "Add an activator entry")
SYSINIT(imgact_binmisc, SI_SUB_EXEC, SI_ORDER_MIDDLE, imgact_binmisc_init, NULL)
static struct bt_table bt
DPCPU_DEFINE_STATIC(int, pcputicks)
SLIST_HEAD(et_eventtimers_list, eventtimer)
struct lock_class lock_class_lockmgr
struct lock_class lock_class_mtx_sleep
struct lock_class lock_class_mtx_spin
struct lock_class lock_class_rm_sleepable
struct lock_class lock_class_rm
struct lock_class lock_class_rw
struct lock_class lock_class_sx
int sysctl_wire_old_buffer(struct sysctl_req *req, size_t len)
int sysctl_handle_int(SYSCTL_HANDLER_ARGS)
struct sbuf * sbuf_new_for_sysctl(struct sbuf *s, char *buf, int length, struct sysctl_req *req)
void binuptime(struct bintime *bt)
void bintime(struct bintime *bt)
void sched_relinquish(struct thread *td)
static bool kasan_enabled __read_mostly
CTASSERT(LOCK_CLASS_MAX==15)
SYSCTL_U16(_debug_lock, OID_AUTO, delay_base, CTLFLAG_RW, &locks_delay.base, 0, "")
void lock_delay_default_init(struct lock_delay_config *lc)
SYSCTL_INT(_debug_lock_delay, OID_AUTO, starvation_limit, CTLFLAG_RW, &starvation_limit, 0, "")
SDT_PROVIDER_DEFINE(lock)
void lock_destroy(struct lock_object *lock)
u_short __read_frequently locks_delay_retries
static SYSCTL_NODE(_debug, OID_AUTO, lock, CTLFLAG_RD|CTLFLAG_MPSAFE, NULL, "lock debugging")
u_short __read_frequently locks_delay_loops
struct lock_delay_config __read_frequently locks_delay
void lock_init(struct lock_object *lock, struct lock_class *class, const char *name, const char *type, int flags)
static u_int lock_roundup_2(u_int val)
struct lock_class * lock_classes[LOCK_CLASS_MAX+1]
void lock_delay(struct lock_delay_arg *la)
LOCK_DELAY_SYSINIT(locks_delay_init)
SDT_PROBE_DEFINE1(lock,,, starvation, "u_int")
static u_int __read_mostly starvation_limit
static void locks_delay_init(void *arg __unused)
static u_int __read_mostly restrict_starvation
int printf(const char *fmt,...)
int sbuf_finish(struct sbuf *s)
void sbuf_delete(struct sbuf *s)
int sbuf_printf(struct sbuf *s, const char *fmt,...)
void cpus_fence_seq_cst(void)
void quiesce_all_critical(void)