31#include "opt_witness.h"
32#include "opt_hwpmc_hooks.h"
39#include <sys/kernel.h>
44#include <sys/bitstring.h>
46#include <sys/rangelock.h>
47#include <sys/resourcevar.h>
51#include <sys/sleepqueue.h>
52#include <sys/selinfo.h>
53#include <sys/syscallsubr.h>
54#include <sys/dtrace_bsd.h>
55#include <sys/sysent.h>
56#include <sys/turnstile.h>
57#include <sys/taskqueue.h>
59#include <sys/rwlock.h>
60#include <sys/umtxvar.h>
61#include <sys/vmmeter.h>
62#include <sys/cpuset.h>
64#include <sys/pmckern.h>
68#include <security/audit/audit.h>
72#include <vm/vm_extern.h>
74#include <vm/vm_phys.h>
75#include <sys/eventhandler.h>
89 "struct thread KBI td_flags");
91 "struct thread KBI td_pflags");
93 "struct thread KBI td_frame");
95 "struct thread KBI td_emuldata");
97 "struct proc KBI p_flag");
99 "struct proc KBI p_pid");
101 "struct proc KBI p_filemon");
103 "struct proc KBI p_comm");
105 "struct proc KBI p_emuldata");
109 "struct thread KBI td_flags");
111 "struct thread KBI td_pflags");
113 "struct thread KBI td_frame");
115 "struct thread KBI td_emuldata");
117 "struct proc KBI p_flag");
119 "struct proc KBI p_pid");
121 "struct proc KBI p_filemon");
123 "struct proc KBI p_comm");
125 "struct proc KBI p_emuldata");
162 &
maxthread, 0,
"Maximum number of threads");
167static u_long tidhash;
168static u_long tidhashlock;
169static struct rwlock *tidhashtbl_lock;
170#define TIDHASH(tid) (&tidhashtbl[(tid) & tidhash])
171#define TIDHASHLOCK(tid) (&tidhashtbl_lock[(tid) & tidhashlock])
179thread_count_inc_try(
void)
183 nthreads_new = atomic_fetchadd_int(&
nthreads, 1) + 1;
197 static struct timeval lastfail;
201 if (thread_count_inc_try()) {
206 if (thread_count_inc_try()) {
211 printf(
"maxthread limit exceeded by uid %u "
212 "(pid %d); consider increasing kern.maxthread\n",
213 curthread->td_ucred->cr_ruid, curproc->p_pid);
235 static lwpid_t trytid;
247 KASSERT(trytid != 0, (
"unexpectedly ran out of IDs"));
250 KASSERT(tid != -1, (
"unexpectedly ran out of IDs"));
255 return (tid + NO_PID);
264 KASSERT(rtid >= NO_PID,
265 (
"%s: invalid tid %d\n", __func__, rtid));
268 (
"thread ID %d not allocated\n", rtid));
287 for (i = 0; i < n; i++) {
312 KASSERT(tb->
n < nitems(tb->
tab),
313 (
"%s: count too high %d", __func__, tb->
n));
314 tb->
tab[tb->
n] = td->td_tid;
322 KASSERT(tb->
n <= nitems(tb->
tab),
323 (
"%s: count too high %d", __func__, tb->
n));
324 if (tb->
n == nitems(tb->
tab)) {
334 KASSERT(tb->
n <= nitems(tb->
tab),
335 (
"%s: count too high %d", __func__, tb->
n));
349 td = (
struct thread *)mem;
350 TD_SET_STATE(td, TDS_INACTIVE);
351 td->td_lastcpu = td->td_oncpu = NOCPU;
359 td->td_lend_user_pri = PRI_MAX;
361 audit_thread_alloc(td);
367 MPASS(td->td_sel == NULL);
379 td = (
struct thread *)mem;
383 switch (TD_GET_STATE(td)) {
392 panic(
"bad state for thread unlinking");
397 panic(
"bad thread state");
402 audit_thread_free(td);
409 td_softdep_cleanup(td);
410 MPASS(td->td_su == NULL);
422 td = (
struct thread *)mem;
424 td->td_allocdomain = vm_phys_domain(vtophys(td));
443 td = (
struct thread *)mem;
449 MPASS(td->td_sel == NULL);
463 TAILQ_INIT(&p->p_threads);
473 if (p->p_ksi != NULL) {
475 p->p_ksi->ksi_flags = KSI_EXT | KSI_INS;
477 LIST_INIT(&p->p_mqnotifier);
510 mtx_init(&
tid_lock,
"TID lock", NULL, MTX_DEF);
517 if (tid0 != THREAD0_TID)
518 panic(
"tid0 %d != %d\n", tid0, THREAD0_TID);
520 flags = UMA_ZONE_NOFREE;
529 flags |= UMA_ZONE_CONTIG;
535 tidhashlock = (tidhash + 1) / 64;
538 tidhashtbl_lock =
malloc(
sizeof(*tidhashtbl_lock) * (tidhashlock + 1),
539 M_TIDHASH, M_WAITOK | M_ZERO);
540 for (i = 0; i < tidhashlock + 1; i++)
541 rw_init(&tidhashtbl_lock[i],
"tidhash");
562 if (atomic_fcmpset_rel_ptr((uintptr_t *)&tdd->
tdd_zombies,
563 (uintptr_t *)&ztd, (uintptr_t)td))
575 atomic_subtract_rel_int(&td->td_proc->p_exitthreads, 1);
585 struct thread *itd, *ntd;
587 struct credbatch credbatch;
599 itd = (
struct thread *)atomic_swap_ptr((uintptr_t *)&tdd->
tdd_zombies,
611 credbatch_prep(&credbatch);
616 while (itd != NULL) {
617 ntd = itd->td_zombie;
621 MPASS(itd->td_limit != NULL);
622 if (lim != itd->td_limit) {
632 credbatch_process(&credbatch);
646 MPASS(limcount != 0);
660 for (i = 0; i < vm_ndomains; i++) {
692 int i, cticks, lticks;
696 cticks = atomic_load_int(&
ticks);
697 for (i = 0; i < vm_ndomains; i++) {
701 (u_int)(cticks - lticks) > 5 *
hz) {
735 t =
malloc(
sizeof(*t), M_TEMP, M_WAITOK);
757 KASSERT(td->td_kstack == 0, (
"thread_alloc got thread with kstack"));
758 if (!vm_thread_new(td, pages)) {
765 bzero(&td->td_sa.args,
sizeof(td->td_sa.args));
767 cpu_thread_alloc(td);
776 KASSERT(td->td_kstack == 0,
777 (
"thread_alloc_stack called on a thread with kstack"));
778 if (!vm_thread_new(td, pages))
780 cpu_thread_alloc(td);
791 lock_profile_thread_exit(td);
794 td->td_cpuset = NULL;
796 if (td->td_kstack != 0)
797 vm_thread_dispose(td);
798 callout_drain(&td->td_slpcallout);
823 PROC_LOCK_ASSERT(p, MA_OWNED);
824 newtd->td_realucred =
crcowget(p->p_ucred);
825 newtd->td_ucred = newtd->td_realucred;
826 newtd->td_limit =
lim_hold(p->p_limit);
827 newtd->td_cowgen = p->p_cowgen;
834 MPASS(td->td_realucred == td->td_ucred);
835 newtd->td_realucred =
crcowget(td->td_realucred);
836 newtd->td_ucred = newtd->td_realucred;
837 newtd->td_limit =
lim_hold(td->td_limit);
838 newtd->td_cowgen = td->td_cowgen;
845 if (td->td_realucred != NULL)
847 if (td->td_limit != NULL)
855 struct ucred *oldcred;
856 struct plimit *oldlimit;
862 td->td_cowgen = p->p_cowgen;
866 if (oldlimit != NULL)
876 PROC_LOCK_ASSERT(p, MA_OWNED);
877 MPASS(td->td_cowgen != p->p_cowgen);
878 MPASS(td->td_ucred == p->p_ucred);
879 MPASS(td->td_limit == p->p_limit);
880 td->td_cowgen = p->p_cowgen;
895 uint64_t runtime, new_switchtime;
904 PROC_SLOCK_ASSERT(p, MA_OWNED);
905 mtx_assert(&
Giant, MA_NOTOWNED);
907 PROC_LOCK_ASSERT(p, MA_OWNED);
908 KASSERT(p != NULL, (
"thread exiting without a process"));
909 CTR3(KTR_PROC,
"thread_exit: thread %p (pid %ld, %s)", td,
910 (
long)p->p_pid, td->td_name);
911 SDT_PROBE0(proc, , , lwp__exit);
912 KASSERT(TAILQ_EMPTY(&td->td_sigqueue.sq_list), (
"signal pending"));
913 MPASS(td->td_realucred == td->td_ucred);
930 if (p->p_flag & P_HADTHREADS) {
931 if (p->p_numthreads > 1) {
932 atomic_add_int(&td->td_proc->p_exitthreads, 1);
934 td2 = FIRST_THREAD_IN_PROC(p);
942 if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) {
943 if (p->p_numthreads == p->p_suspcount) {
944 thread_lock(p->p_singlethread);
946 p->p_singlethread, p,
false);
952 PCPU_SET(deadthread, td);
957 panic (
"thread_exit: Last thread exiting on its own");
965 if (PMC_PROC_IS_USING_PMCS(td->td_proc)) {
966 PMC_SWITCH_CONTEXT(td, PMC_FN_CSW_OUT);
967 PMC_CALL_HOOK_UNLOCKED(td, PMC_FN_THR_EXIT, NULL);
968 }
else if (PMC_SYSTEM_SAMPLING_ACTIVE())
969 PMC_CALL_HOOK_UNLOCKED(td, PMC_FN_THR_EXIT_LOG, NULL);
978 runtime = new_switchtime - PCPU_GET(switchtime);
979 td->td_runtime += runtime;
980 td->td_incruntime += runtime;
981 PCPU_SET(switchtime, new_switchtime);
982 PCPU_SET(switchticks,
ticks);
986 td->td_ru.ru_nvcsw++;
991 TD_SET_STATE(td, TDS_INACTIVE);
995 CTR1(KTR_PROC,
"thread_exit: cpu_throw() thread %p", td);
997 panic(
"I'm a teapot!");
1010 mtx_assert(&
Giant, MA_NOTOWNED);
1011 KASSERT(p->p_numthreads == 1, (
"multiple threads in thread_wait()"));
1012 KASSERT(p->p_exitthreads == 0, (
"p_exitthreads leaking"));
1013 td = FIRST_THREAD_IN_PROC(p);
1017 lock_profile_thread_exit(td);
1019 td->td_cpuset = NULL;
1020 cpu_thread_clean(td);
1022 callout_drain(&td->td_slpcallout);
1040 TD_SET_STATE(td, TDS_INACTIVE);
1042 td->td_flags = TDF_INMEM;
1044 LIST_INIT(&td->td_contested);
1045 LIST_INIT(&td->td_lprof[0]);
1046 LIST_INIT(&td->td_lprof[1]);
1048 SLIST_INIT(&td->td_epochs);
1052 TAILQ_INSERT_TAIL(&p->p_threads, td, td_plist);
1063 struct proc *p = td->td_proc;
1065 PROC_LOCK_ASSERT(p, MA_OWNED);
1067 MPASS(SLIST_EMPTY(&td->td_epochs));
1070 TAILQ_REMOVE(&p->p_threads, td, td_plist);
1081 PROC_LOCK_ASSERT(p, MA_OWNED);
1082 PROC_SLOCK_ASSERT(p, MA_OWNED);
1083 if (
mode == SINGLE_EXIT)
1084 remaining = p->p_numthreads;
1085 else if (
mode == SINGLE_BOUNDARY)
1086 remaining = p->p_numthreads - p->p_boundary_count;
1087 else if (
mode == SINGLE_NO_EXIT ||
mode == SINGLE_ALLPROC)
1088 remaining = p->p_numthreads - p->p_suspcount;
1090 panic(
"calc_remaining: wrong mode %d",
mode);
1098 return (
mode == SINGLE_ALLPROC ? 0 : 1);
1106 PROC_LOCK_ASSERT(p, MA_OWNED);
1107 PROC_SLOCK_ASSERT(p, MA_OWNED);
1108 THREAD_LOCK_ASSERT(td2, MA_OWNED);
1119 if (TD_IS_SUSPENDED(td2)) {
1124 if (TD_CAN_ABORT(td2)) {
1126 return (wakeup_swapper);
1129 case SINGLE_BOUNDARY:
1130 case SINGLE_NO_EXIT:
1131 if (TD_IS_SUSPENDED(td2) &&
1132 (td2->td_flags & TDF_BOUNDARY) == 0) {
1137 if (TD_CAN_ABORT(td2)) {
1139 return (wakeup_swapper);
1142 case SINGLE_ALLPROC:
1151 if (TD_IS_SUSPENDED(td2) && (td2->td_flags & (TDF_BOUNDARY |
1152 TDF_ALLPROCSUSP)) == 0) {
1157 if (TD_CAN_ABORT(td2)) {
1158 if ((td2->td_flags & TDF_SBDRY) == 0) {
1160 td2->td_flags |= TDF_ALLPROCSUSP;
1163 return (wakeup_swapper);
1171 return (wakeup_swapper);
1192 int remaining, wakeup_swapper;
1195 KASSERT(
mode == SINGLE_EXIT ||
mode == SINGLE_BOUNDARY ||
1196 mode == SINGLE_ALLPROC ||
mode == SINGLE_NO_EXIT,
1197 (
"invalid mode %d",
mode));
1204 KASSERT((
mode == SINGLE_ALLPROC && td->td_proc != p) ||
1205 (
mode != SINGLE_ALLPROC && td->td_proc == p),
1206 (
"mode %d proc %p curproc %p",
mode, p, td->td_proc));
1207 mtx_assert(&
Giant, MA_NOTOWNED);
1208 PROC_LOCK_ASSERT(p, MA_OWNED);
1210 if ((p->p_flag & P_HADTHREADS) == 0 &&
mode != SINGLE_ALLPROC)
1214 if (p->p_singlethread != NULL && p->p_singlethread != td)
1217 if (
mode == SINGLE_EXIT) {
1218 p->p_flag |= P_SINGLE_EXIT;
1219 p->p_flag &= ~P_SINGLE_BOUNDARY;
1221 p->p_flag &= ~P_SINGLE_EXIT;
1222 if (
mode == SINGLE_BOUNDARY)
1223 p->p_flag |= P_SINGLE_BOUNDARY;
1225 p->p_flag &= ~P_SINGLE_BOUNDARY;
1227 if (
mode == SINGLE_ALLPROC)
1228 p->p_flag |= P_TOTAL_STOP;
1229 p->p_flag |= P_STOPPED_SINGLE;
1231 p->p_singlethread = td;
1234 if (P_SHOULDSTOP(p) != P_STOPPED_SINGLE)
1237 FOREACH_THREAD_IN_PROC(p, td2) {
1241 td2->td_flags |= TDF_ASTPENDING | TDF_NEEDSUSPCHK;
1242 if (TD_IS_INHIBITED(td2)) {
1245 }
else if (TD_IS_RUNNING(td2) && td != td2) {
1246 forward_signal(td2);
1270 if (
mode == SINGLE_EXIT) {
1276 KASSERT(p->p_numthreads == 1, (
"Unthreading with >1 threads"));
1277 p->p_singlethread = NULL;
1278 p->p_flag &= ~(P_STOPPED_SINGLE | P_SINGLE_EXIT | P_HADTHREADS);
1283 while (p->p_exitthreads != 0) {
1290 }
else if (
mode == SINGLE_BOUNDARY) {
1303 FOREACH_THREAD_IN_PROC(p, td2) {
1307 KASSERT((td2->td_flags & TDF_BOUNDARY) != 0,
1308 (
"td %p not on boundary", td2));
1309 KASSERT(TD_IS_SUSPENDED(td2),
1310 (
"td %p is not suspended", td2));
1326 PROC_LOCK_ASSERT(p, MA_OWNED);
1327 return (P_SHOULDSTOP(p) || ((p->p_flag & P_TRACED) != 0 &&
1328 (td->td_dbgflags & TDB_SUSPEND) != 0));
1373 mtx_assert(&
Giant, MA_NOTOWNED);
1374 PROC_LOCK_ASSERT(p, MA_OWNED);
1376 if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) {
1377 KASSERT(p->p_singlethread != NULL,
1378 (
"singlethread not set"));
1385 if (p->p_singlethread == td)
1388 if ((p->p_flag & P_SINGLE_EXIT) && return_instead)
1392 if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE &&
1393 (p->p_flag & P_SINGLE_BOUNDARY) && return_instead)
1399 if ((td->td_flags & TDF_SBDRY) != 0) {
1400 KASSERT(return_instead,
1401 (
"TDF_SBDRY set for unsafe thread_suspend_check"));
1402 KASSERT((td->td_flags & (TDF_SEINTR | TDF_SERESTART)) !=
1403 (TDF_SEINTR | TDF_SERESTART),
1404 (
"both TDF_SEINTR and TDF_SERESTART"));
1405 return (TD_SBDRY_INTR(td) ? TD_SBDRY_ERRNO(td) : 0);
1413 if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td)) {
1420 if (__predict_false(p->p_sysent->sv_thread_detach != NULL))
1421 (p->p_sysent->sv_thread_detach)(td);
1424 panic(
"stopped thread did not exit");
1429 if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) {
1430 if (p->p_numthreads == p->p_suspcount + 1) {
1431 thread_lock(p->p_singlethread);
1433 p->p_singlethread, p,
false);
1445 if (return_instead == 0) {
1446 p->p_boundary_count++;
1447 td->td_flags |= TDF_BOUNDARY;
1485 if ((td->td_flags & TDF_NEEDSUSPCHK) == 0)
1490 if (p->p_flag & P_SINGLE_EXIT)
1492 else if (P_SHOULDSTOP(p) ||
1493 ((p->p_flag & P_TRACED) && (td->td_dbgflags & TDB_SUSPEND)))
1503 KASSERT(!TD_IS_SUSPENDED(td), (
"already suspended"));
1504 PROC_LOCK_ASSERT(p, MA_OWNED);
1505 PROC_SLOCK_ASSERT(p, MA_OWNED);
1510 if (p == td->td_proc) {
1516 td->td_flags &= ~TDF_NEEDSUSPCHK;
1517 TD_SET_SUSPENDED(td);
1533 PROC_SLOCK_ASSERT(p, MA_OWNED);
1534 THREAD_LOCK_ASSERT(td, MA_OWNED);
1535 KASSERT(!TD_IS_SUSPENDED(td), (
"already suspended"));
1537 td->td_flags &= ~TDF_NEEDSUSPCHK;
1538 TD_SET_SUSPENDED(td);
1546 THREAD_LOCK_ASSERT(td, MA_OWNED);
1547 KASSERT(TD_IS_SUSPENDED(td), (
"Thread not suspended"));
1548 TD_CLR_SUSPENDED(td);
1549 td->td_flags &= ~TDF_ALLPROCSUSP;
1550 if (td->td_proc == p) {
1551 PROC_SLOCK_ASSERT(p, MA_OWNED);
1553 if (boundary && (td->td_flags & TDF_BOUNDARY) != 0) {
1554 td->td_flags &= ~TDF_BOUNDARY;
1555 p->p_boundary_count--;
1567 PROC_LOCK_ASSERT(p, MA_OWNED);
1569 if (TD_ON_SLEEPQ(td))
1574 THREAD_LOCK_ASSERT(td, MA_OWNED);
1575 KASSERT(TD_IS_SUSPENDED(td), (
"Thread not suspended"));
1577 TD_CLR_SUSPENDED(td);
1579 MPASS(p->p_suspcount > 0);
1595 PROC_LOCK_ASSERT(p, MA_OWNED);
1596 PROC_SLOCK_ASSERT(p, MA_OWNED);
1598 if (!P_SHOULDSTOP(p)) {
1599 FOREACH_THREAD_IN_PROC(p, td) {
1601 if (TD_IS_SUSPENDED(td)) {
1607 }
else if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE &&
1608 p->p_numthreads == p->p_suspcount) {
1614 if (p->p_singlethread->td_proc == p) {
1615 thread_lock(p->p_singlethread);
1617 p->p_singlethread, p,
false);
1633 KASSERT(
mode == SINGLE_EXIT ||
mode == SINGLE_BOUNDARY ||
1634 mode == SINGLE_ALLPROC ||
mode == SINGLE_NO_EXIT,
1635 (
"invalid mode %d",
mode));
1636 PROC_LOCK_ASSERT(p, MA_OWNED);
1637 KASSERT((
mode == SINGLE_ALLPROC && (p->p_flag & P_TOTAL_STOP) != 0) ||
1638 (
mode != SINGLE_ALLPROC && (p->p_flag & P_TOTAL_STOP) == 0),
1639 (
"mode %d does not match P_TOTAL_STOP",
mode));
1640 KASSERT(
mode == SINGLE_ALLPROC || p->p_singlethread == curthread,
1641 (
"thread_single_end from other thread %p %p",
1642 curthread, p->p_singlethread));
1643 KASSERT(
mode != SINGLE_BOUNDARY ||
1644 (p->p_flag & P_SINGLE_BOUNDARY) != 0,
1645 (
"mis-matched SINGLE_BOUNDARY flags %x", p->p_flag));
1646 p->p_flag &= ~(P_STOPPED_SINGLE | P_SINGLE_EXIT | P_SINGLE_BOUNDARY |
1649 p->p_singlethread = NULL;
1658 FOREACH_THREAD_IN_PROC(p, td) {
1660 if (TD_IS_SUSPENDED(td)) {
1662 mode == SINGLE_BOUNDARY);
1667 KASSERT(
mode != SINGLE_BOUNDARY || p->p_boundary_count == 0,
1668 (
"inconsistent boundary count %d", p->p_boundary_count));
1686tdfind_hash(lwpid_t tid, pid_t pid,
struct proc **pp,
struct thread **tdp)
1688#define RUN_THRESH 16
1697 LIST_FOREACH(td,
TIDHASH(tid), td_hash) {
1698 if (td->td_tid != tid) {
1703 if (pid != -1 && p->p_pid != pid) {
1709 LIST_REMOVE(td, td_hash);
1710 LIST_INSERT_HEAD(
TIDHASH(td->td_tid),
1735 if (td->td_tid == tid) {
1736 if (pid != -1 && td->td_proc->p_pid != pid)
1738 PROC_LOCK(td->td_proc);
1746 if (td->td_tid != tid) {
1750 if (td->td_proc != p) {
1754 if (p->p_state == PRS_NEW) {
1766 LIST_INSERT_HEAD(
TIDHASH(td->td_tid), td, td_hash);
1775 LIST_REMOVE(td, td_hash);
_Static_assert(sizeof(struct acctv3) - offsetof(struct acctv3, ac_trailer)==sizeof(struct acctv2) - offsetof(struct acctv2, ac_trailer), "trailer")
void cpuset_rel(struct cpuset *set)
void kdtrace_thread_dtor(struct thread *td)
void kdtrace_thread_ctor(struct thread *td)
void *() malloc(size_t size, struct malloc_type *mtp, int flags)
void free(void *addr, struct malloc_type *mtp)
struct mtx __exclusive_cache_line Giant
int priv_check_cred(struct ucred *cred, int priv)
EVENTHANDLER_LIST_DEFINE(process_ctor)
struct ucred * crcowget(struct ucred *cr)
void credbatch_add(struct credbatch *crb, struct thread *td)
void credbatch_final(struct credbatch *crb)
struct ucred * crcowsync(void)
void crfree(struct ucred *cr)
void crcowfree(struct thread *td)
void rlqentry_free(struct rl_q_entry *rleq)
void rucollect(struct rusage *ru, struct rusage *ru2)
void lim_free(struct plimit *limp)
struct plimit * lim_hold(struct plimit *limp)
struct plimit * lim_cowsync(void)
void ruxagg_locked(struct proc *p, struct thread *td)
void lim_freen(struct plimit *limp, int n)
void panic(const char *fmt,...)
void sigqueue_init(sigqueue_t *list, struct proc *p)
ksiginfo_t * ksiginfo_alloc(int wait)
void thread_stopped(struct proc *p)
void mi_switch(int flags)
int setrunnable(struct thread *td, int srqflags)
int kern_thr_exit(struct thread *td)
static int calc_remaining(struct proc *p, int mode)
static void thread_reap_all(void)
struct tidbatch __aligned
static void thread_fini(void *mem, int size)
static LIST_HEAD(tidhashhead, thread)
static void thread_dtor(void *mem, int size, void *arg)
static void tidbatch_add(struct tidbatch *tb, struct thread *td)
static void thread_reap_domain(struct thread_domain_data *tdd)
void tidhash_add(struct thread *td)
void thread_cow_synced(struct thread *td)
void thread_run_flash(struct thread *td)
void thread_stash(struct thread *td)
static lwpid_t tid_alloc(void)
void thread_cow_update(struct thread *td)
void thread_cow_get_proc(struct thread *newtd, struct proc *p)
static __exclusive_cache_line struct mtx tid_lock
void proc_linkup(struct proc *p, struct thread *td)
static void thread_free_batched(struct thread *td)
void thread_cow_get(struct thread *newtd, struct thread *td)
static int thread_init(void *mem, int size, int flags)
void thread_reap_barrier(void)
static bitstr_t * tid_bitmap
static void thread_reap(void)
static uma_zone_t thread_zone
static void tid_free_locked(lwpid_t rtid)
SDT_PROBE_DEFINE(proc,,, lwp__exit)
void tidhash_remove(struct thread *td)
static void tid_free(lwpid_t rtid)
static int thread_unsuspend_one(struct thread *td, struct proc *p, bool boundary)
static void tidbatch_final(struct tidbatch *tb)
static void thread_count_sub(int n)
static void tid_free_batch(lwpid_t *batch, int n)
static void thread_zombie(struct thread *)
static struct task thread_reap_task
static void thread_count_dec(void)
static bool tdfind_hash(lwpid_t tid, pid_t pid, struct proc **pp, struct thread **tdp)
static bool thread_count_inc(void)
static int weed_inhib(int mode, struct thread *td2, struct proc *p)
void thread_link(struct thread *td, struct proc *p)
int thread_check_susp(struct thread *td, bool sleep)
int thread_suspend_check(int return_instead)
static void tidbatch_process(struct tidbatch *tb)
void thread_single_end(struct proc *p, int mode)
struct thread * thread_alloc(int pages)
struct thread * tdfind(lwpid_t tid, pid_t pid)
int thread_single(struct proc *p, int mode)
SYSCTL_INT(_kern, OID_AUTO, maxthread, CTLFLAG_RDTUN, &maxthread, 0, "Maximum number of threads")
static __exclusive_cache_line int nthreads
static struct callout thread_reap_callout
void proc_linkup0(struct proc *p, struct thread *td)
int thread_alloc_stack(struct thread *td, int pages)
static void thread_reap_task_cb(void *, int)
void thread_cow_free(struct thread *td)
bool thread_suspend_check_needed(void)
void thread_wait(struct proc *p)
static MALLOC_DEFINE(M_TIDHASH, "tidhash", "thread hash")
SDT_PROVIDER_DECLARE(proc)
void thread_unsuspend(struct proc *p)
void thread_free(struct thread *td)
void thread_suspend_one(struct thread *td)
static int remain_for_mode(int mode)
static void thread_reap_callout_cb(void *)
static void tidbatch_prep(struct tidbatch *tb)
static int thread_ctor(void *mem, int size, void *arg, int flags)
void thread_suspend_switch(struct thread *td, struct proc *p)
void thread_unlink(struct thread *td)
int ppsratecheck(struct timeval *lasttime, int *curpps, int maxpps)
void callout_init(struct callout *c, int mpsafe)
void umtx_thread_alloc(struct thread *td)
void umtx_thread_init(struct thread *td)
void umtx_thread_exit(struct thread *td)
void umtx_thread_fini(struct thread *td)
struct iommu_domain ** domain
void sched_exit_thread(struct thread *td, struct thread *child)
void sched_relinquish(struct thread *td)
void sched_throw(struct thread *td)
int sched_sizeof_thread(void)
void sched_sleep(struct thread *td, int pri)
struct thread * tdd_zombies
void * hashinit(int elements, struct malloc_type *type, u_long *hashmask)
void kmsan_thread_free(struct thread *td)
void kmsan_thread_alloc(struct thread *td)
int printf(const char *fmt,...)
void sleepq_remove_nested(struct thread *td)
struct sleepqueue * sleepq_alloc(void)
int sleepq_abort(struct thread *td, int intrval)
void sleepq_free(struct sleepqueue *sq)
int quiesce_all_cpus(const char *wmesg, int prio)
int taskqueue_enqueue(struct taskqueue *queue, struct task *task)
void taskqueue_drain(struct taskqueue *queue, struct task *task)
void turnstile_free(struct turnstile *ts)
struct turnstile * turnstile_alloc(void)
void witness_thread_exit(struct thread *td)
void seltdfini(struct thread *td)