93#include "opt_hwpmc_hooks.h"
95#include "opt_witness.h"
100#include <sys/kernel.h>
103#include <sys/malloc.h>
104#include <sys/mutex.h>
108#include <sys/sched.h>
109#include <sys/stack.h>
110#include <sys/sysctl.h>
111#include <sys/syslog.h>
112#include <sys/systm.h>
118#include <machine/stdarg.h>
120#if !defined(DDB) && !defined(STACK)
121#error "DDB or STACK options are required for WITNESS"
126#define KTR_WITNESS KTR_SUBSYS
131#define LI_RECURSEMASK 0x0000ffff
132#define LI_EXCLUSIVE 0x00010000
133#define LI_NORELEASE 0x00020000
134#define LI_SLEEPABLE 0x00040000
137#define WITNESS_COUNT 1536
139#define WITNESS_HASH_SIZE 251
140#define WITNESS_PENDLIST (512 + (MAXCPU * 4))
143#define WITNESS_LO_DATA_COUNT 2048
146#define WITNESS_LO_HASH_SIZE 1021
153#define LOCK_NCHILDREN 5
154#define LOCK_CHILDCOUNT 2048
158#define FULLGRAPH_SBUF_SIZE 512
164#define WITNESS_UNRELATED 0x00
165#define WITNESS_PARENT 0x01
166#define WITNESS_ANCESTOR 0x02
167#define WITNESS_CHILD 0x04
168#define WITNESS_DESCENDANT 0x08
169#define WITNESS_ANCESTOR_MASK (WITNESS_PARENT | WITNESS_ANCESTOR)
170#define WITNESS_DESCENDANT_MASK (WITNESS_CHILD | WITNESS_DESCENDANT)
171#define WITNESS_RELATED_MASK \
172 (WITNESS_ANCESTOR_MASK | WITNESS_DESCENDANT_MASK)
173#define WITNESS_REVERSAL 0x10
175#define WITNESS_RESERVED1 0x20
176#define WITNESS_RESERVED2 0x40
177#define WITNESS_LOCK_ORDER_KNOWN 0x80
180#define WITNESS_DTOA(x) (((x) & WITNESS_RELATED_MASK) >> 2)
183#define WITNESS_ATOD(x) (((x) & WITNESS_RELATED_MASK) << 2)
185#define WITNESS_INDEX_ASSERT(i) \
186 MPASS((i) > 0 && (i) <= w_max_used_index && (i) < witness_count)
228 STAILQ_ENTRY(
witness) w_typelist;
233 uint16_t w_num_ancestors;
235 uint16_t w_num_descendants;
238 unsigned w_displayed:1;
239 unsigned w_reversed:1;
302 return ((w1->
w_class->lc_flags & (LC_SLEEPLOCK | LC_SPINLOCK)) ==
303 (w2->
w_class->lc_flags & (LC_SLEEPLOCK | LC_SPINLOCK)));
320 struct lock_class *lock_class);
322 const struct lock_object *lock);
332static void witness_ddb_compute_levels(
void);
333static void witness_ddb_display(
int(*)(
const char *fmt, ...));
334static void witness_ddb_display_descendants(
int(*)(
const char *fmt, ...),
336static void witness_ddb_display_list(
int(*prnt)(
const char *fmt, ...),
337 struct witness_list *list);
338static void witness_ddb_level_descendants(
struct witness *
parent,
int l);
339static void witness_ddb_list(
struct thread *td);
360 int (*prnt)(
const char *fmt, ...));
361static int witness_output(
const char *fmt, ...) __printflike(1, 2);
364static
int witness_voutput(const
char *fmt, va_list ap) __printflike(1, 0);
381 CTLFLAG_RWTUN | CTLTYPE_INT | CTLFLAG_MPSAFE, NULL, 0,
383 "
witness is watching lock operations");
397SYSCTL_INT(_debug_witness, OID_AUTO, kdb, CTLFLAG_RWTUN, &witness_kdb, 0,
"");
400#if defined(DDB) || defined(KDB)
407int witness_trace = 1;
408SYSCTL_INT(_debug_witness, OID_AUTO, trace, CTLFLAG_RWTUN, &witness_trace, 0,
"");
411#ifdef WITNESS_SKIPSPIN
434SYSCTL_PROC(_debug_witness, OID_AUTO, output_channel,
435 CTLTYPE_STRING | CTLFLAG_RWTUN | CTLFLAG_MPSAFE, NULL, 0,
437 "Output channel for warnings");
443 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0,
445 "Show locks relation graphs");
451 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0,
453 "Show bad witness stacks");
459static struct witness_list
w_all = STAILQ_HEAD_INITIALIZER(
w_all);
463static struct witness_list
w_sleep = STAILQ_HEAD_INITIALIZER(
w_sleep);
488static const char w_stillcold[] =
"Witness is still cold\n";
490static const char w_notallowed[] =
"The sysctl is disabled on the arch\n";
708#if defined(__i386__) || defined(__amd64__)
736 {
"dirhash",
"bufwait" },
742 {
"ufs",
"bufwait" },
763 while (strncmp(file,
"../", 3) == 0)
792 struct lock_object *lock;
816 mtx_assert(&
Giant, MA_NOTOWNED);
818 CTR1(
KTR_WITNESS,
"%s: initializing witness", __func__);
819 mtx_init(&
w_mtx,
"witness lock", NULL, MTX_SPIN | MTX_QUIET |
820 MTX_NOWITNESS | MTX_NOPROFILE);
823 memset(w, 0,
sizeof(*w));
828 (
"%s: Invalid list of free witness objects", __func__));
831 STAILQ_REMOVE_HEAD(&
w_free, w_list);
848 w->w_file =
"order list";
849 for (order++; order->
w_name != NULL; order++) {
853 w1->w_file =
"order list";
863 KASSERT(lock->lo_flags & LO_WITNESS,
864 (
"%s: lock %s is on pending list but not LO_WITNESS",
865 __func__, lock->lo_name));
879 struct lock_class *
class;
882 class = LOCK_CLASS(lock);
883 if ((lock->lo_flags & LO_RECURSABLE) != 0 &&
884 (class->lc_flags & LC_RECURSABLE) == 0)
885 kassert_panic(
"%s: lock (%s) %s can not be recursable",
886 __func__, class->lc_name, lock->lo_name);
887 if ((lock->lo_flags & LO_SLEEPABLE) != 0 &&
888 (class->lc_flags & LC_SLEEPABLE) == 0)
889 kassert_panic(
"%s: lock (%s) %s can not be sleepable",
890 __func__, class->lc_name, lock->lo_name);
891 if ((lock->lo_flags & LO_UPGRADABLE) != 0 &&
892 (class->lc_flags & LC_UPGRADABLE) == 0)
893 kassert_panic(
"%s: lock (%s) %s can not be upgradable",
894 __func__, class->lc_name, lock->lo_name);
904 (lock->lo_flags & LO_WITNESS) == 0)
905 lock->lo_witness = NULL;
910 panic(
"%s: pending locks list is too small, "
911 "increase WITNESS_PENDLIST\n",
920 struct lock_class *
class;
923 class = LOCK_CLASS(lock);
926 panic(
"lock (%s) %s destroyed while witness_cold",
927 class->lc_name, lock->lo_name);
930 if ((lock->lo_flags & LO_WITNESS) == 0 || lock->lo_witness == NULL)
932 w = lock->lo_witness;
934 mtx_lock_spin(&
w_mtx);
935 MPASS(w->w_refcount > 0);
938 if (w->w_refcount == 0)
940 mtx_unlock_spin(&
w_mtx);
945witness_ddb_compute_levels(
void)
952 STAILQ_FOREACH(w, &
w_all, w_list)
958 STAILQ_FOREACH(w, &
w_all, w_list) {
960 if (w->w_num_ancestors > 0)
962 witness_ddb_level_descendants(w, 0);
967witness_ddb_level_descendants(
struct witness *w,
int l)
971 if (w->w_ddb_level >= l)
979 witness_ddb_level_descendants(&
w_data[i], l);
984witness_ddb_display_descendants(
int(*prnt)(
const char *fmt, ...),
989 for (i = 0; i < indent; i++)
991 prnt(
"%s (type: %s, depth: %d, active refs: %d)",
993 w->w_ddb_level, w->w_refcount);
994 if (w->w_displayed) {
995 prnt(
" -- (already displayed)\n");
999 if (w->w_file != NULL && w->w_line != 0)
1003 prnt(
" -- never acquired\n");
1010 witness_ddb_display_descendants(prnt, &
w_data[i],
1016witness_ddb_display_list(
int(*prnt)(
const char *fmt, ...),
1017 struct witness_list *list)
1021 STAILQ_FOREACH(w, list, w_typelist) {
1022 if (w->w_file == NULL || w->w_ddb_level > 0)
1026 witness_ddb_display_descendants(prnt, w, 0);
1033witness_ddb_display(
int(*prnt)(
const char *fmt, ...))
1037 KASSERT(
witness_cold == 0, (
"%s: witness_cold", __func__));
1038 witness_ddb_compute_levels();
1041 STAILQ_FOREACH(w, &
w_all, w_list)
1048 prnt(
"Sleep locks:\n");
1049 witness_ddb_display_list(prnt, &
w_sleep);
1056 prnt(
"\nSpin locks:\n");
1057 witness_ddb_display_list(prnt, &
w_spin);
1064 prnt(
"\nLocks which were never acquired:\n");
1065 STAILQ_FOREACH(w, &
w_all, w_list) {
1066 if (w->w_file != NULL || w->w_refcount == 0)
1068 prnt(
"%s (type: %s, depth: %d)\n", w->
w_name,
1069 w->
w_class->lc_name, w->w_ddb_level);
1084 if (lock1 == NULL || lock1->lo_witness == NULL || lock2 == NULL ||
1085 lock2->lo_witness == NULL)
1088 mtx_assert(&
w_mtx, MA_NOTOWNED);
1089 mtx_lock_spin(&
w_mtx);
1097 mtx_unlock_spin(&
w_mtx);
1102 CTR3(
KTR_WITNESS,
"%s: adding %s as a child of %s", __func__,
1103 lock2->lo_witness->w_name, lock1->lo_witness->w_name);
1104 itismychild(lock1->lo_witness, lock2->lo_witness);
1105 mtx_unlock_spin(&
w_mtx);
1111 int line,
struct lock_object *interlock)
1115 struct lock_class *
class, *iclass;
1120 if (
witness_cold || witness_watch < 1 || lock->lo_witness == NULL ||
1124 w = lock->lo_witness;
1125 class = LOCK_CLASS(lock);
1128 if (class->lc_flags & LC_SLEEPLOCK) {
1135 kassert_panic(
"acquiring blockable sleep lock with "
1136 "spinlock or critical section held (%s) %s @ %s:%d",
1137 class->lc_name, lock->lo_name,
1144 lock_list = td->td_sleeplocks;
1145 if (lock_list == NULL || lock_list->
ll_count == 0)
1157 lock_list = PCPU_GET(spinlocks);
1158 if (lock_list == NULL || lock_list->
ll_count == 0) {
1171 if (lock1 != NULL) {
1173 (
flags & LOP_EXCLUSIVE) == 0) {
1175 class->lc_name, lock->lo_name,
1179 kassert_panic(
"excl->share");
1182 (
flags & LOP_EXCLUSIVE) != 0) {
1184 class->lc_name, lock->lo_name,
1188 kassert_panic(
"share->excl");
1194 if (interlock != NULL) {
1195 iclass = LOCK_CLASS(interlock);
1198 kassert_panic(
"interlock (%s) %s not locked @ %s:%d",
1199 iclass->lc_name, interlock->lo_name,
1202 kassert_panic(
"interlock (%s) %s recursed @ %s:%d",
1203 iclass->lc_name, interlock->lo_name,
1211 if (interlock != NULL && plock->
li_lock == interlock) {
1233 w1 = plock->
li_lock->lo_witness;
1237 mtx_lock_spin(&
w_mtx);
1239 mtx_unlock_spin(&
w_mtx);
1251 if (!(lock->lo_flags & LO_DUPOK) && !(
flags & LOP_DUPOK) &&
1255 mtx_unlock_spin(&
w_mtx);
1257 "acquiring duplicate lock of same type: \"%s\"\n",
1265 mtx_unlock_spin(&
w_mtx);
1268 mtx_assert(&
w_mtx, MA_OWNED);
1278 for (j = 0, lle = lock_list; lle != NULL; lle = lle->
ll_next) {
1279 for (i = lle->
ll_count - 1; i >= 0; i--, j++) {
1280 struct stack pstack;
1281 bool pstackv, trace;
1289 if (interlock == lock1->
li_lock)
1296 w1 = lock1->
li_lock->lo_witness;
1298 KASSERT((lock1->
li_lock->lo_flags & LO_WITNESS) == 0,
1299 (
"lock missing witness structure"));
1308 lock == &
Giant.lock_object)
1315 if ((lock->lo_flags & LO_SLEEPABLE) != 0 &&
1316 (
flags & LOP_NOSLEEP) == 0 &&
1326 if ((lock->lo_flags & LO_SLEEPABLE) != 0 &&
1327 (
flags & LOP_NOSLEEP) == 0 &&
1336 lock == &
Giant.lock_object)
1358 w->w_reversed = w1->w_reversed = 1;
1369 trace = atomic_load_int(&witness_trace);
1381 mtx_unlock_spin(&
w_mtx);
1383#ifdef WITNESS_NO_VNODE
1390 if ((lock->lo_flags & LO_IS_VNODE) != 0 &&
1391 (lock1->
li_lock->lo_flags & LO_IS_VNODE) != 0)
1398 if ((lock->lo_flags & LO_SLEEPABLE) != 0 &&
1399 (
flags & LOP_NOSLEEP) == 0 &&
1402 "lock order reversal: (sleepable after non-sleepable)\n");
1404 && lock == &
Giant.lock_object)
1406 "lock order reversal: (Giant after non-sleepable)\n");
1416 MPASS(lock2->
li_lock != NULL);
1417 if (lock2->
li_lock->lo_witness == w)
1419 if (i == 0 && lle->
ll_next != NULL) {
1433 lock, lock->lo_name, w->
w_name,
1450 lock->lo_name, w->
w_name,
1464 "lock order %s -> %s established at:\n",
1467 M_NOWAIT, STACK_SBUF_FMT_LONG);
1471 "lock order %s -> %s attempted at:\n",
1473 stack_save(&pstack);
1475 STACK_SBUF_FMT_LONG);
1491 if (
flags & LOP_NEWORDER &&
1493 (lock->lo_flags & LO_SLEEPABLE) != 0 &&
1494 (
flags & LOP_NOSLEEP) == 0)) {
1495 CTR3(
KTR_WITNESS,
"%s: adding %s as a child of %s", __func__,
1500 mtx_unlock_spin(&
w_mtx);
1514 w = lock->lo_witness;
1518 if (LOCK_CLASS(lock)->lc_flags & LC_SLEEPLOCK)
1519 lock_list = &td->td_sleeplocks;
1521 lock_list = PCPU_PTR(spinlocks);
1525 if (instance != NULL) {
1527 CTR4(
KTR_WITNESS,
"%s: pid %d recursed on %s r=%d", __func__,
1528 td->td_proc->p_pid, lock->lo_name,
1546 CTR3(
KTR_WITNESS,
"%s: pid %d added lle %p", __func__,
1547 td->td_proc->p_pid, lle);
1555 if ((
flags & LOP_EXCLUSIVE) != 0)
1557 if ((lock->lo_flags & LO_SLEEPABLE) != 0 && (
flags & LOP_NOSLEEP) == 0)
1559 CTR4(
KTR_WITNESS,
"%s: pid %d added %s as lle[%d]", __func__,
1560 td->td_proc->p_pid, lock->lo_name, lle->
ll_count - 1);
1567 struct lock_class *
class;
1569 KASSERT(
witness_cold == 0, (
"%s: witness_cold", __func__));
1570 if (lock->lo_witness == NULL ||
witness_watch == -1 || KERNEL_PANICKED())
1572 class = LOCK_CLASS(lock);
1574 if ((lock->lo_flags & LO_UPGRADABLE) == 0)
1576 "upgrade of non-upgradable lock (%s) %s @ %s:%d",
1577 class->lc_name, lock->lo_name,
1579 if ((class->lc_flags & LC_SLEEPLOCK) == 0)
1581 "upgrade of non-sleep lock (%s) %s @ %s:%d",
1582 class->lc_name, lock->lo_name,
1586 if (instance == NULL) {
1587 kassert_panic(
"upgrade of unlocked lock (%s) %s @ %s:%d",
1588 class->lc_name, lock->lo_name,
1595 "upgrade of exclusive lock (%s) %s @ %s:%d",
1596 class->lc_name, lock->lo_name,
1600 "upgrade of recursed lock (%s) %s r=%d @ %s:%d",
1601 class->lc_name, lock->lo_name,
1613 struct lock_class *
class;
1615 KASSERT(
witness_cold == 0, (
"%s: witness_cold", __func__));
1616 if (lock->lo_witness == NULL ||
witness_watch == -1 || KERNEL_PANICKED())
1618 class = LOCK_CLASS(lock);
1620 if ((lock->lo_flags & LO_UPGRADABLE) == 0)
1622 "downgrade of non-upgradable lock (%s) %s @ %s:%d",
1623 class->lc_name, lock->lo_name,
1625 if ((class->lc_flags & LC_SLEEPLOCK) == 0)
1627 "downgrade of non-sleep lock (%s) %s @ %s:%d",
1628 class->lc_name, lock->lo_name,
1632 if (instance == NULL) {
1633 kassert_panic(
"downgrade of unlocked lock (%s) %s @ %s:%d",
1634 class->lc_name, lock->lo_name,
1641 "downgrade of shared lock (%s) %s @ %s:%d",
1642 class->lc_name, lock->lo_name,
1646 "downgrade of recursed lock (%s) %s r=%d @ %s:%d",
1647 class->lc_name, lock->lo_name,
1651 instance->
li_flags &= ~LI_EXCLUSIVE;
1659 struct lock_class *
class;
1664 if (
witness_cold || lock->lo_witness == NULL || KERNEL_PANICKED())
1667 class = LOCK_CLASS(lock);
1670 if (class->lc_flags & LC_SLEEPLOCK)
1671 lock_list = &td->td_sleeplocks;
1673 lock_list = PCPU_PTR(spinlocks);
1675 for (; *lock_list != NULL; lock_list = &(*lock_list)->
ll_next)
1676 for (i = 0; i < (*lock_list)->ll_count; i++) {
1677 instance = &(*lock_list)->ll_children[i];
1678 if (instance->
li_lock == lock)
1689 kassert_panic(
"lock (%s) %s not locked @ %s:%d", class->lc_name,
1699 (
flags & LOP_EXCLUSIVE) == 0) {
1704 kassert_panic(
"excl->ushare");
1707 (
flags & LOP_EXCLUSIVE) != 0) {
1713 kassert_panic(
"share->uexcl");
1717 CTR4(
KTR_WITNESS,
"%s: pid %d unrecursed on %s r=%d", __func__,
1718 td->td_proc->p_pid, instance->
li_lock->lo_name,
1727 kassert_panic(
"lock marked norelease");
1732 CTR4(
KTR_WITNESS,
"%s: pid %d removed %s from lle[%d]", __func__,
1733 td->td_proc->p_pid, instance->
li_lock->lo_name,
1734 (*lock_list)->ll_count - 1);
1735 for (j = i; j < (*lock_list)->ll_count - 1; j++)
1736 (*lock_list)->ll_children[j] =
1737 (*lock_list)->ll_children[j + 1];
1738 (*lock_list)->ll_count--;
1750 if ((*lock_list)->ll_count == 0) {
1751 if (*lock_list == lle) {
1757 CTR3(
KTR_WITNESS,
"%s: pid %d removed lle %p", __func__,
1758 td->td_proc->p_pid, lle);
1769 lle = td->td_sleeplocks;
1770 if (lle == NULL || KERNEL_PANICKED())
1773 for (n = 0; lle != NULL; lle = lle->
ll_next)
1774 for (i = lle->
ll_count - 1; i >= 0; i--) {
1777 "Thread %p exiting with the following locks held:\n", td);
1784 "Thread %p cannot exit while holding sleeplocks\n", td);
1809 for (lle = td->td_sleeplocks; lle != NULL; lle = lle->
ll_next)
1810 for (i = lle->
ll_count - 1; i >= 0; i--) {
1814 if (
flags & WARN_GIANTOK &&
1817 if (
flags & WARN_SLEEPOK &&
1824 printf(
" with the following %slocks held:\n",
1825 (
flags & WARN_SLEEPOK) != 0 ?
1826 "non-sleepable " :
"");
1838 lock_list = PCPU_GET(spinlocks);
1839 if (lock_list != NULL && lock_list->
ll_count != 0) {
1850 lock1->
li_lock == lock && n == 0)
1856 printf(
" with the following %slocks held:\n",
1857 (
flags & WARN_SLEEPOK) != 0 ?
"non-sleepable " :
"");
1861 if (
flags & WARN_PANIC && n)
1862 kassert_panic(
"%s", __func__);
1873 if (
witness_cold || witness_watch < 1 || lock->lo_witness == NULL)
1875 w = lock->lo_witness;
1884 if (
witness_cold || witness_watch < 1 || lock->lo_witness == NULL)
1886 w = lock->lo_witness;
1891enroll(
const char *description,
struct lock_class *lock_class)
1895 MPASS(description != NULL);
1899 if ((lock_class->lc_flags & LC_SPINLOCK)) {
1902 }
else if ((lock_class->lc_flags & LC_SLEEPLOCK) == 0) {
1903 kassert_panic(
"lock class %s is not sleep or spin",
1904 lock_class->lc_name);
1908 mtx_lock_spin(&
w_mtx);
1915 strcpy(w->
w_name, description);
1918 STAILQ_INSERT_HEAD(&
w_all, w, w_list);
1919 if (lock_class->lc_flags & LC_SPINLOCK) {
1920 STAILQ_INSERT_HEAD(&
w_spin, w, w_typelist);
1922 }
else if (lock_class->lc_flags & LC_SLEEPLOCK) {
1923 STAILQ_INSERT_HEAD(&
w_sleep, w, w_typelist);
1930 mtx_unlock_spin(&
w_mtx);
1934 if (w->w_refcount == 1)
1936 mtx_unlock_spin(&
w_mtx);
1939 "lock (%s) %s does not match earlier (%s) lock",
1940 description, lock_class->lc_name,
1949 MPASS(w->w_refcount == 0);
1950 if (w->
w_class->lc_flags & LC_SLEEPLOCK) {
1969 mtx_assert(&
w_mtx, MA_OWNED);
1983 ci =
child->w_index;
1995 parent->w_num_descendants++;
1996 child->w_num_ancestors++;
2026 w_data[i].w_num_descendants++;
2027 w_data[j].w_num_ancestors++;
2036 printf(
"witness rmatrix paradox! [%d][%d]=%d "
2037 "both ancestor and descendant\n",
2040 printf(
"Witness disabled.\n");
2045 printf(
"witness rmatrix paradox! [%d][%d]=%d "
2046 "both ancestor and descendant\n",
2049 printf(
"Witness disabled.\n");
2063 mtx_assert(&
w_mtx, MA_OWNED);
2068 mtx_unlock_spin(&
w_mtx);
2073 "%s: parent \"%s\" (%s) and child \"%s\" (%s) are not "
2074 "the same lock type", __func__,
parent->w_name,
2076 child->w_class->lc_name);
2078 mtx_lock_spin(&
w_mtx);
2090 unsigned char r1, r2;
2104 if (!mtx_owned(&
w_mtx))
2106 printf(
"%s: rmatrix mismatch between %s (index %d) and %s "
2107 "(index %d): w_rmatrix[%d][%d] == %hhx but "
2108 "w_rmatrix[%d][%d] == %hhx\n",
2112 printf(
"Witness disabled.\n");
2115 return (r1 & rmask);
2166 mtx_assert(&
w_mtx, MA_OWNED);
2169 mtx_unlock_spin(&
w_mtx);
2172 if (STAILQ_EMPTY(&
w_free)) {
2174 mtx_unlock_spin(&
w_mtx);
2175 printf(
"WITNESS: unable to allocate a new witness object\n");
2178 w = STAILQ_FIRST(&
w_free);
2179 STAILQ_REMOVE_HEAD(&
w_free, w_list);
2184 bzero(w,
sizeof(*w));
2195 STAILQ_INSERT_HEAD(&
w_free, w, w_list);
2206 mtx_lock_spin(&
w_mtx);
2210 mtx_unlock_spin(&
w_mtx);
2211 printf(
"%s: witness exhausted\n", __func__);
2215 mtx_unlock_spin(&
w_mtx);
2216 bzero(lle,
sizeof(*lle));
2224 mtx_lock_spin(&
w_mtx);
2227 mtx_unlock_spin(&
w_mtx);
2237 for (lle = list; lle != NULL; lle = lle->
ll_next)
2238 for (i = lle->
ll_count - 1; i >= 0; i--) {
2240 if (instance->
li_lock == lock)
2248 int (*prnt)(
const char *fmt, ...))
2250 struct lock_object *lock;
2254 "exclusive" :
"shared", LOCK_CLASS(lock)->lc_name, lock->lo_name);
2255 if (lock->lo_witness->w_name != lock->lo_name)
2256 prnt(
" (%s)", lock->lo_witness->w_name);
2257 prnt(
" r = %d (%p) locked @ %s:%d\n",
2285 vlog(LOG_NOTICE, fmt, ap);
2295witness_thread_has_locks(
struct thread *td)
2298 if (td->td_sleeplocks == NULL)
2300 return (td->td_sleeplocks->ll_count != 0);
2304witness_proc_has_locks(
struct proc *p)
2308 FOREACH_THREAD_IN_PROC(p, td) {
2309 if (witness_thread_has_locks(td))
2318 int (*prnt)(
const char *fmt, ...))
2324 for (lle = *lock_list; lle != NULL; lle = lle->
ll_next)
2325 for (i = lle->
ll_count - 1; i >= 0; i--) {
2341 int (*prnt)(
const char *fmt, ...))
2346 if (owner->td_critnest == 0 || owner->td_oncpu == NOCPU)
2350 if (instance != NULL)
2355witness_save(
struct lock_object *lock,
const char **filep,
int *linep)
2359 struct lock_class *
class;
2366 if (SCHEDULER_STOPPED())
2368 KASSERT(
witness_cold == 0, (
"%s: witness_cold", __func__));
2369 if (lock->lo_witness == NULL ||
witness_watch == -1 || KERNEL_PANICKED())
2371 class = LOCK_CLASS(lock);
2372 if (class->lc_flags & LC_SLEEPLOCK)
2373 lock_list = curthread->td_sleeplocks;
2377 lock_list = PCPU_GET(spinlocks);
2380 if (instance == NULL) {
2381 kassert_panic(
"%s: lock (%s) %s not locked", __func__,
2382 class->lc_name, lock->lo_name);
2394 struct lock_class *
class;
2401 if (SCHEDULER_STOPPED())
2403 KASSERT(
witness_cold == 0, (
"%s: witness_cold", __func__));
2404 if (lock->lo_witness == NULL ||
witness_watch == -1 || KERNEL_PANICKED())
2406 class = LOCK_CLASS(lock);
2407 if (class->lc_flags & LC_SLEEPLOCK)
2408 lock_list = curthread->td_sleeplocks;
2412 lock_list = PCPU_GET(spinlocks);
2415 if (instance == NULL)
2416 kassert_panic(
"%s: lock (%s) %s not locked", __func__,
2417 class->lc_name, lock->lo_name);
2418 lock->lo_witness->w_file = file;
2419 lock->lo_witness->w_line = line;
2420 if (instance == NULL)
2430#ifdef INVARIANT_SUPPORT
2432 struct lock_class *
class;
2434 if (lock->lo_witness == NULL ||
witness_watch < 1 || KERNEL_PANICKED())
2436 class = LOCK_CLASS(lock);
2437 if ((class->lc_flags & LC_SLEEPLOCK) != 0)
2439 else if ((class->lc_flags & LC_SPINLOCK) != 0)
2442 kassert_panic(
"Lock (%s) %s is not sleep or spin!",
2443 class->lc_name, lock->lo_name);
2448 if (instance != NULL)
2449 kassert_panic(
"Lock (%s) %s locked @ %s:%d.",
2450 class->lc_name, lock->lo_name,
2454 case LA_LOCKED | LA_RECURSED:
2455 case LA_LOCKED | LA_NOTRECURSED:
2457 case LA_SLOCKED | LA_RECURSED:
2458 case LA_SLOCKED | LA_NOTRECURSED:
2460 case LA_XLOCKED | LA_RECURSED:
2461 case LA_XLOCKED | LA_NOTRECURSED:
2462 if (instance == NULL) {
2463 kassert_panic(
"Lock (%s) %s not locked @ %s:%d.",
2464 class->lc_name, lock->lo_name,
2468 if ((
flags & LA_XLOCKED) != 0 &&
2471 "Lock (%s) %s not exclusively locked @ %s:%d.",
2472 class->lc_name, lock->lo_name,
2474 if ((
flags & LA_SLOCKED) != 0 &&
2477 "Lock (%s) %s exclusively locked @ %s:%d.",
2478 class->lc_name, lock->lo_name,
2480 if ((
flags & LA_RECURSED) != 0 &&
2482 kassert_panic(
"Lock (%s) %s not recursed @ %s:%d.",
2483 class->lc_name, lock->lo_name,
2485 if ((
flags & LA_NOTRECURSED) != 0 &&
2487 kassert_panic(
"Lock (%s) %s recursed @ %s:%d.",
2488 class->lc_name, lock->lo_name,
2492 kassert_panic(
"Invalid lock assertion at %s:%d.",
2503 struct lock_class *
class;
2505 if (lock->lo_witness == NULL ||
witness_watch == -1 || KERNEL_PANICKED())
2507 class = LOCK_CLASS(lock);
2508 if (class->lc_flags & LC_SLEEPLOCK)
2509 lock_list = curthread->td_sleeplocks;
2513 lock_list = PCPU_GET(spinlocks);
2516 if (instance == NULL) {
2517 kassert_panic(
"%s: lock (%s) %s not locked", __func__,
2518 class->lc_name, lock->lo_name);
2544witness_ddb_list(
struct thread *td)
2547 KASSERT(
witness_cold == 0, (
"%s: witness_cold", __func__));
2548 KASSERT(
kdb_active, (
"%s: not in the debugger", __func__));
2568 if (td == curthread && PCPU_GET(spinlocks) != NULL)
2572DB_SHOW_COMMAND(locks, db_witness_list)
2577 td = db_lookup_thread(
addr,
true);
2580 witness_ddb_list(td);
2583DB_SHOW_ALL_COMMAND(locks, db_witness_list_all)
2593 FOREACH_PROC_IN_SYSTEM(p) {
2594 if (!witness_proc_has_locks(p))
2596 FOREACH_THREAD_IN_PROC(p, td) {
2597 if (!witness_thread_has_locks(td))
2599 db_printf(
"Process %d (%s) thread %p (%d)\n", p->p_pid,
2600 p->p_comm, td, td->td_tid);
2601 witness_ddb_list(td);
2607DB_SHOW_ALIAS(alllocks, db_witness_list_all)
2609DB_SHOW_COMMAND(
witness, db_witness_display)
2612 witness_ddb_display(db_printf);
2620 struct witness *tmp_w1, *tmp_w2, *w1, *w2;
2621 int generation, i, j;
2629 tmp_w1 =
malloc(
sizeof(
struct witness), M_TEMP, M_WAITOK | M_ZERO);
2630 tmp_w2 =
malloc(
sizeof(
struct witness), M_TEMP, M_WAITOK | M_ZERO);
2639 mtx_lock_spin(&
w_mtx);
2641 mtx_unlock_spin(&
w_mtx);
2642 sbuf_printf(sb,
"Number of known direct relationships is %d\n",
2645 mtx_lock_spin(&
w_mtx);
2647 mtx_unlock_spin(&
w_mtx);
2656 if (w1->w_reversed == 0) {
2657 mtx_unlock_spin(&
w_mtx);
2663 mtx_unlock_spin(&
w_mtx);
2665 if (tmp_w1->w_reversed == 0)
2671 mtx_lock_spin(&
w_mtx);
2673 mtx_unlock_spin(&
w_mtx);
2696 if (data2 && data2 != data1) {
2701 mtx_unlock_spin(&
w_mtx);
2707 "\nLock order reversal between \"%s\"(%s) and \"%s\"(%s)!\n",
2712 "Lock order \"%s\"(%s) -> \"%s\"(%s) first seen at:\n",
2718 if (data2 && data2 != data1) {
2720 "Lock order \"%s\"(%s) -> \"%s\"(%s) first seen at:\n",
2728 mtx_lock_spin(&
w_mtx);
2730 mtx_unlock_spin(&
w_mtx);
2740 mtx_unlock_spin(&
w_mtx);
2743 free(tmp_data1, M_TEMP);
2744 free(tmp_data2, M_TEMP);
2745 free(tmp_w1, M_TEMP);
2746 free(tmp_w2, M_TEMP);
2779sbuf_db_printf_drain(
void *arg __unused,
const char *
data,
int len)
2782 return (db_printf(
"%.*s", len,
data));
2785DB_SHOW_COMMAND(badstacks, db_witness_badstacks)
2791 sbuf_new(&sb, buffer,
sizeof(buffer), SBUF_FIXEDLEN);
2801 static const struct {
2814 for (i = 0; i < nitems(channels); i++)
2821 if (error != 0 || req->newptr == NULL)
2825 for (i = 0; i < nitems(channels); i++)
2826 if (strcmp(channels[i].
name,
buf) == 0) {
2842 error = SYSCTL_OUT(req, w_notallowed,
sizeof(w_notallowed));
2864 mtx_lock_spin(&
w_mtx);
2865 STAILQ_FOREACH(w, &
w_all, w_list)
2867 STAILQ_FOREACH(w, &
w_all, w_list)
2869 mtx_unlock_spin(&
w_mtx);
2887 if (error != 0 || req->newptr == NULL)
2901 if (w->w_displayed != 0 || (w->w_file == NULL && w->w_line == 0))
2924 unsigned int hash = 5381;
2929 for (i = 0; i < size; i++)
2930 hash = ((hash << 5) + hash) + (
unsigned int)key[i];
2932 for (i = 0; key[i] != 0; i++)
2933 hash = ((hash << 5) + hash) + (
unsigned int)key[i];
2977 mtx_assert(&
w_mtx, MA_OWNED);
2981 if (strcmp(w->
w_name, key) == 0)
2996 MPASS(w->
w_name != NULL);
2998 mtx_assert(&
w_mtx, MA_OWNED);
3000 (
"%s: trying to add a hash entry that already exists!", __func__));
3001 KASSERT(w->w_hash_next == NULL,
3002 (
"%s: w->w_hash_next != NULL", __func__));
3029 while (
data != NULL) {
3081 data->wlod_key = key;
3085 stack_save(&
data->wlod_stack);
3095 mtx_assert(&
w_mtx, MA_OWNED);
3117 if (witness_trace) {
3124 stack_sbuf_print_ddb(&sb, &
st);
device_property_type_t type
static struct bt_table st
void *() malloc(size_t size, struct malloc_type *mtp, int flags)
void free(void *addr, struct malloc_type *mtp)
struct lock_class lock_class_mtx_sleep
struct mtx __exclusive_cache_line Giant
struct lock_class lock_class_mtx_spin
struct lock_class lock_class_rm
struct lock_class lock_class_rw
void panic(const char *fmt,...)
struct lock_class lock_class_sx
int sysctl_wire_old_buffer(struct sysctl_req *req, size_t len)
int sysctl_handle_int(SYSCTL_HANDLER_ARGS)
int sysctl_handle_string(SYSCTL_HANDLER_ARGS)
struct sbuf * sbuf_new_for_sysctl(struct sbuf *s, char *buf, int length, struct sysctl_req *req)
struct lock_object * li_lock
struct lock_instance ll_children[LOCK_NCHILDREN]
struct lock_list_entry * ll_next
struct witness * wh_array[WITNESS_HASH_SIZE]
struct witness_lock_order_key wlod_key
struct witness_lock_order_data * wlod_next
struct witness_lock_order_data * wloh_array[WITNESS_LO_HASH_SIZE]
struct lock_class * w_class
struct lock_object * wh_lock
struct lock_class * w_class
u_char __read_frequently kdb_active
struct thread * kdb_thread
void kdb_enter(const char *why, const char *msg)
struct pcpu * pcpu_find(u_int cpuid)
int printf(const char *fmt,...)
void vlog(int level, const char *fmt, va_list ap)
int vprintf(const char *fmt, va_list ap)
int snprintf(char *str, size_t size, const char *format,...)
int sbuf_finish(struct sbuf *s)
void sbuf_delete(struct sbuf *s)
int sbuf_printf(struct sbuf *s, const char *fmt,...)
void sbuf_set_drain(struct sbuf *s, sbuf_drain_func *func, void *ctx)
ssize_t sbuf_len(struct sbuf *s)
char * sbuf_data(struct sbuf *s)
void sbuf_clear(struct sbuf *s)
struct sbuf * sbuf_new(struct sbuf *s, char *buf, int length, int flags)
int stack_sbuf_print_flags(struct sbuf *sb, const struct stack *st, int flags, enum stack_sbuf_fmt format)
void stack_zero(struct stack *st)
void stack_copy(const struct stack *src, struct stack *dst)
void stack_sbuf_print(struct sbuf *sb, const struct stack *st)
static int witness_spin_warn
static struct witness_list w_free
#define WITNESS_LOCK_ORDER_KNOWN
void witness_destroy(struct lock_object *lock)
#define WITNESS_DESCENDANT
static struct witness_lock_order_data * witness_lock_order_get(struct witness *parent, struct witness *child)
const char * witness_file(struct lock_object *lock)
static int sysctl_debug_witness_channel(SYSCTL_HANDLER_ARGS)
static void witness_hash_put(struct witness *w)
#define WITNESS_HASH_SIZE
static struct witness_list w_all
static struct lock_list_entry w_locklistdata[LOCK_CHILDCOUNT]
#define WITNESS_LO_HASH_SIZE
void witness_downgrade(struct lock_object *lock, int flags, const char *file, int line)
#define WITNESS_ANCESTOR_MASK
int witness_line(struct lock_object *lock)
void witness_assert(const struct lock_object *lock, int flags, const char *file, int line)
FEATURE(witness, "kernel has witness(9) support")
static void itismychild(struct witness *parent, struct witness *child)
static unsigned int w_generation
static __inline int witness_lock_type_equal(struct witness *w1, struct witness *w2)
#define WITNESS_LO_DATA_COUNT
static int sysctl_debug_witness_fullgraph(SYSCTL_HANDLER_ARGS)
void witness_unlock(struct lock_object *lock, int flags, const char *file, int line)
static struct witness * w_data
static void witness_list_lock(struct lock_instance *instance, int(*prnt)(const char *fmt,...))
static SYSCTL_NODE(_debug, OID_AUTO, witness, CTLFLAG_RW|CTLFLAG_MPSAFE, NULL, "Witness Locking")
static struct witness_list w_spin
void witness_startup(void *mem)
void witness_restore(struct lock_object *lock, const char *file, int line)
static const char w_notrunning[]
#define WITNESS_RELATED_MASK
static uint8_t ** w_rmatrix
static void sbuf_print_witness_badstacks(struct sbuf *sb, size_t *oldidx)
static int w_max_used_index
int witness_defineorder(struct lock_object *lock1, struct lock_object *lock2)
static int sysctl_debug_witness_watch(SYSCTL_HANDLER_ARGS)
void witness_lock(struct lock_object *lock, int flags, const char *file, int line)
static int isitmychild(struct witness *parent, struct witness *child)
static int static int witness_output_drain(void *arg __unused, const char *data, int len)
SYSCTL_PROC(_debug_witness, OID_AUTO, watch, CTLFLAG_RWTUN|CTLTYPE_INT|CTLFLAG_MPSAFE, NULL, 0, sysctl_debug_witness_watch, "I", "witness is watching lock operations")
static void adopt(struct witness *parent, struct witness *child)
static const char w_stillcold[]
int witness_warn(int flags, struct lock_object *lock, const char *fmt,...)
static const char * fixup_filename(const char *file)
static struct lock_list_entry * w_lock_list_free
static struct witness_lock_order_hash w_lohash
void witness_releaseok(struct lock_object *lock)
#define FULLGRAPH_SBUF_SIZE
SYSCTL_INT(_debug_witness, OID_AUTO, skipspin, CTLFLAG_RDTUN, &witness_skipspin, 0, "")
void witness_init(struct lock_object *lock, const char *type)
static struct witness * witness_get(void)
static struct witness * enroll(const char *description, struct lock_class *lock_class)
static void witness_init_hash_tables(void)
static void witness_free(struct witness *m)
static void depart(struct witness *w)
static __inline int witness_lock_order_key_equal(const struct witness_lock_order_key *a, const struct witness_lock_order_key *b)
static struct lock_list_entry * witness_lock_list_get(void)
static void witness_add_fullgraph(struct sbuf *sb, struct witness *parent)
static void witness_debugger(int cond, const char *msg)
static struct witness_list w_sleep
static int witness_lock_order_check(struct witness *parent, struct witness *child)
static void witness_lock_list_free(struct lock_list_entry *lle)
static void witness_increment_graph_generation(void)
#define WITNESS_DESCENDANT_MASK
void witness_upgrade(struct lock_object *lock, int flags, const char *file, int line)
static struct witness_lock_order_data * w_lofree
void witness_checkorder(struct lock_object *lock, int flags, const char *file, int line, struct lock_object *interlock)
static MALLOC_DEFINE(M_WITNESS, "Witness", "Witness")
static struct witness_pendhelp pending_locks[WITNESS_PENDLIST]
static int _isitmyx(struct witness *w1, struct witness *w2, int rmask, const char *fname)
static int sysctl_debug_witness_badstacks(SYSCTL_HANDLER_ARGS)
static int blessed(struct witness *, struct witness *)
static struct witness_hash w_hash
static struct lock_instance * find_instance(struct lock_list_entry *list, const struct lock_object *lock)
void witness_norelease(struct lock_object *lock)
void witness_display_spinlock(struct lock_object *lock, struct thread *owner, int(*prnt)(const char *fmt,...))
static int witness_lock_order_add(struct witness *parent, struct witness *child)
void witness_thread_exit(struct thread *td)
void witness_save(struct lock_object *lock, const char **filep, int *linep)
static int witness_output(const char *fmt,...) __printflike(1
int witness_list_locks(struct lock_list_entry **lock_list, int(*prnt)(const char *fmt,...))
static uint32_t witness_hash_djb2(const uint8_t *key, uint32_t size)
static struct witness_blessed blessed_list[]
int witness_startup_count(void)
static struct witness_order_list_entry order_lists[]
static int isitmydescendant(struct witness *parent, struct witness *child)
static int witness_voutput(const char *fmt, va_list ap) __printflike(1
#define WITNESS_INDEX_ASSERT(i)
STAILQ_HEAD(witness_list, witness)
static struct witness * witness_hash_get(const char *key)
static void witness_enter_debugger(const char *msg)
static int static void witness_setflag(struct lock_object *lock, int flag, int set)
static struct witness_lock_order_data w_lodata[WITNESS_LO_DATA_COUNT]