40#include <sys/sysctl.h>
42#include <sys/sysproto.h>
44#include <sys/kernel.h>
46#include <sys/malloc.h>
50#include <sys/refcount.h>
53#include <sys/syscallsubr.h>
54#include <sys/capsicum.h>
55#include <sys/cpuset.h>
56#include <sys/domainset.h>
59#include <sys/libkern.h>
60#include <sys/limits.h>
62#include <sys/interrupt.h>
63#include <sys/vmmeter.h>
67#include <vm/vm_object.h>
68#include <vm/vm_page.h>
69#include <vm/vm_pageout.h>
70#include <vm/vm_extern.h>
71#include <vm/vm_param.h>
72#include <vm/vm_phys.h>
73#include <vm/vm_pagequeue.h>
138SYSCTL_INT(_kern_sched, OID_AUTO, cpusetsize, CTLFLAG_RD | CTLFLAG_CAPRD,
139 SYSCTL_NULL_INT_PTR,
sizeof(cpuset_t),
"sizeof(cpuset_t)");
144static int domainset_valid(
const struct domainset *,
const struct domainset *);
149static struct cpuset *
153 if (
set->cs_id == CPUSET_INVALID)
161static struct cpuset *
165 while ((
set->cs_flags & CPU_SET_ROOT) == 0 &&
set->cs_parent != NULL)
177 refcount_acquire(&
set->cs_ref);
185static struct cpuset *
197static struct cpuset *
212 if (refcount_release_if_not_last(&
set->cs_ref))
215 if (!refcount_release(&
set->cs_ref)) {
219 LIST_REMOVE(
set, cs_siblings);
221 if (
id != CPUSET_INVALID)
222 LIST_REMOVE(
set, cs_link);
226 if (
id != CPUSET_INVALID)
238 if (refcount_release_if_not_last(&
set->cs_ref))
241 if (!refcount_release(&
set->cs_ref)) {
245 LIST_REMOVE(
set, cs_siblings);
246 if (
set->cs_id != CPUSET_INVALID)
247 LIST_REMOVE(
set, cs_link);
248 LIST_INSERT_HEAD(head,
set, cs_link);
262 LIST_REMOVE(
set, cs_link);
265 if (
id != CPUSET_INVALID)
272static struct cpuset *
277 if (setid == CPUSET_INVALID)
281 if (
set->cs_id == setid)
287 KASSERT(td != NULL, (
"[%s:%d] td is NULL", __func__, __LINE__));
288 if (
set != NULL && jailed(td->td_ucred)) {
289 struct cpuset *jset, *tset;
291 jset = td->td_ucred->cr_prison->pr_cpuset;
292 for (tset =
set; tset != NULL; tset = tset->cs_parent)
311 const cpuset_t *
mask,
struct domainset *
domain, cpusetid_t
id)
324 LIST_INIT(&
set->cs_children);
325 refcount_init(&
set->cs_ref, 1);
329 CPU_AND(&
set->cs_mask, &
set->cs_mask, &
parent->cs_mask);
332 LIST_INSERT_HEAD(&
parent->cs_children,
set, cs_siblings);
333 if (
set->cs_id != CPUSET_INVALID)
360 dofree = (*setp == NULL);
381 for (i = 0; i <
count; i++) {
383 LIST_INSERT_HEAD(list,
set, cs_link);
400 while ((
set = LIST_FIRST(list)) != NULL) {
401 LIST_REMOVE(
set, cs_link);
409 struct domainset *
set;
412 for (i = 0; i <
count; i++) {
414 LIST_INSERT_HEAD(list,
set, ds_link);
429 struct domainset *
set;
431 while ((
set = LIST_FIRST(list)) != NULL) {
432 LIST_REMOVE(
set, ds_link);
442 DOMAINSET_COPY(&from->ds_mask, &to->ds_mask);
443 to->ds_policy = from->ds_policy;
444 to->ds_prefer = from->ds_prefer;
452 return (DOMAINSET_CMP(&one->ds_mask, &two->ds_mask) == 0 &&
453 one->ds_policy == two->ds_policy &&
454 one->ds_prefer == two->ds_prefer);
461 if (
child->ds_policy != DOMAINSET_POLICY_PREFER)
462 return (DOMAINSET_SUBSET(&
parent->ds_mask, &
child->ds_mask));
463 return (DOMAINSET_ISSET(
child->ds_prefer, &
parent->ds_mask));
468 const struct domainset *
child)
470 if (
child->ds_policy != DOMAINSET_POLICY_PREFER)
471 return (DOMAINSET_OVERLAP(&
parent->ds_mask, &
child->ds_mask));
472 return (DOMAINSET_ISSET(
child->ds_prefer, &
parent->ds_mask));
481static struct domainset *
484 struct domainset *ndomain;
487 KASSERT(
domain->ds_cnt <= vm_ndomains,
488 (
"invalid domain count in domainset %p",
domain));
489 KASSERT(
domain->ds_policy != DOMAINSET_POLICY_PREFER ||
490 domain->ds_prefer < vm_ndomains,
491 (
"invalid preferred domain in domains %p",
domain));
501 if (ndomain == NULL) {
504 for (i = 0, j = 0; i < DOMAINSET_FLS(&
domain->ds_mask); i++)
505 if (DOMAINSET_ISSET(i, &
domain->ds_mask))
506 domain->ds_order[j++] = i;
511 if (freelist != NULL)
512 LIST_INSERT_HEAD(freelist,
domain, ds_link);
530 DOMAINSET_ZERO(&
empty);
531 for (i = 0; i < vm_ndomains; i++)
532 if (VM_DOMAIN_EMPTY(i))
533 DOMAINSET_SET(i, &
empty);
540 for (i = j = 0; i < DOMAINSET_FLS(&
domain->ds_mask); i++)
541 if (DOMAINSET_ISSET(i, &
domain->ds_mask))
542 domain->ds_order[j++] = i;
545 if (
domain->ds_policy == DOMAINSET_POLICY_PREFER &&
547 domain->ds_policy = DOMAINSET_POLICY_ROUNDROBIN;
560 struct domainset *ndomain;
567 if (
domain->ds_policy <= DOMAINSET_POLICY_INVALID ||
568 domain->ds_policy > DOMAINSET_POLICY_MAX)
570 if (
domain->ds_policy == DOMAINSET_POLICY_PREFER &&
590 FOREACH_PROC_IN_SYSTEM(p) {
592 if (p->p_state == PRS_NEW) {
596 FOREACH_THREAD_IN_PROC(p, td) {
598 td->td_domain.dr_policy = td->td_cpuset->cs_domain;
610static struct domainset *
612 const struct domainset *
domain,
struct domainlist *freelist)
614 struct domainset *ndomain;
616 ndomain = LIST_FIRST(freelist);
617 LIST_REMOVE(ndomain, ds_link);
627 DOMAINSET_AND(&ndomain->ds_mask, &pdomain->ds_mask);
645 if (
set->cs_flags & CPU_SET_RDONLY)
648 CPU_AND(&newmask, &
set->cs_mask,
mask);
650 CPU_COPY(
mask, &newmask);
652 if (CPU_EMPTY(&newmask))
655 LIST_FOREACH(nset, &
set->cs_children, cs_siblings)
671 LIST_FOREACH(nset, &
set->cs_children, cs_siblings)
688 error =
priv_check(curthread, PRIV_SCHED_CPUSET);
698 if ((
set->cs_flags & CPU_SET_ROOT) != 0 &&
699 jailed(curthread->td_ucred) &&
700 set == curthread->td_ucred->cr_prison->pr_cpuset)
706 if ((
set->cs_flags & (CPU_SET_ROOT | CPU_SET_RDONLY)) == CPU_SET_ROOT) {
707 KASSERT(
set->cs_parent != NULL,
708 (
"jail.cpuset=%d is not a proper child of parent jail's root.",
716 root =
set->cs_parent;
721 if (root && !CPU_SUBSET(&root->cs_mask,
mask)) {
743 struct domainset *orig,
int *
count,
int augment_mask __unused)
747 struct domainset newset;
751 if (
set->cs_flags & CPU_SET_RDONLY)
758 DOMAINSET_AND(&newset.ds_mask, &dset->ds_mask);
763 LIST_FOREACH(nset, &
set->cs_children, cs_siblings)
775 struct domainset *orig,
struct domainlist *domains)
787 if (
set->cs_domain != orig) {
788 orig =
set->cs_domain;
792 LIST_FOREACH(nset, &
set->cs_children, cs_siblings)
806 struct domainlist domains;
807 struct domainset temp;
808 struct domainset *dset;
810 int ndomains, needed;
813 error =
priv_check(curthread, PRIV_SCHED_CPUSET);
822 if (jailed(curthread->td_ucred) &&
823 set->cs_flags & CPU_SET_ROOT)
832 dset = root->cs_domain;
843 if (
domain->ds_policy == DOMAINSET_POLICY_PREFER)
844 DOMAINSET_COPY(&
set->cs_domain->ds_mask,
856 if (ndomains >= needed)
865 dset =
set->cs_domain;
888cpuset_which(cpuwhich_t which, id_t
id,
struct proc **pp,
struct thread **tdp,
889 struct cpuset **setp)
906 if ((p =
pfind(
id)) == NULL)
921 case CPU_WHICH_CPUSET:
923 thread_lock(curthread);
925 thread_unlock(curthread);
944 *setp =
pr->pr_cpuset;
945 mtx_unlock(&
pr->pr_mtx);
949 case CPU_WHICH_DOMAIN:
960 td = FIRST_THREAD_IN_PROC(p);
968 const struct domainset *
domain)
971 struct domainset *dset;
999 const cpuset_t *
mask,
const struct domainset *
domain,
1000 struct setlist *cpusets,
struct domainlist *domains)
1003 struct cpuset *nset;
1004 struct domainset *dset;
1005 struct domainset *d;
1013 dset =
parent->cs_domain;
1020 nset = LIST_FIRST(cpusets);
1023 LIST_REMOVE(nset, cs_link);
1029static struct cpuset *
1032 struct cpuset *tdset;
1034 tdset = td->td_cpuset;
1035 td->td_cpuset = nset;
1036 td->td_domain.dr_policy = nset->cs_domain;
1044 struct domainset *
domain)
1050 mask = &tdset->cs_mask;
1052 domain = tdset->cs_domain;
1058 struct domainset *
domain,
struct cpuset **nsetp,
1059 struct setlist *freelist,
struct domainlist *domainlist)
1065 mask = &tdset->cs_mask;
1067 domain = tdset->cs_domain;
1084 if (CPU_CMP(&tdset->cs_mask, &
parent->cs_mask) != 0) {
1085 CPU_AND(
mask, &tdset->cs_mask, &
set->cs_mask);
1087 CPU_COPY(&
set->cs_mask,
mask);
1093 if (tdset->cs_domain !=
parent->cs_domain) {
1095 DOMAINSET_AND(&
domain->ds_mask, &
set->cs_domain->ds_mask);
1099 if (CPU_EMPTY(
mask) || DOMAINSET_EMPTY(&
domain->ds_mask))
1111 if (tdset->cs_id != CPUSET_INVALID)
1118 struct cpuset **nsetp,
struct setlist *freelist,
1119 struct domainlist *domainlist)
1129 if (tdset->cs_id != CPUSET_INVALID) {
1143 struct cpuset *nroot,
struct cpuset **nsetp,
1144 struct setlist *cpusets,
struct domainlist *domainlist)
1146 struct domainset ndomain;
1148 struct cpuset *pbase;
1154 CPU_AND(&nmask, &pbase->cs_mask, &nroot->cs_mask);
1157 DOMAINSET_AND(&ndomain.ds_mask, &
set->cs_domain->ds_mask);
1160 if (CPU_EMPTY(&nmask) || DOMAINSET_EMPTY(&ndomain.ds_mask))
1169 pbase = LIST_FIRST(cpusets);
1170 LIST_REMOVE(pbase, cs_link);
1173 LIST_INSERT_HEAD(cpusets, pbase, cs_link);
1205 struct domainset *
domain,
bool rebase)
1207 struct setlist freelist;
1208 struct setlist droplist;
1209 struct domainlist domainlist;
1210 struct cpuset *base, *nset, *nroot, *tdroot;
1227 LIST_INIT(&droplist);
1234 error =
cpuset_which(CPU_WHICH_PID, pid, &p, &td, &nset);
1238 needed = p->p_numthreads;
1239 if (
set != NULL && rebase && tdroot != nroot)
1241 if (nfree >= needed)
1244 if (nfree < needed) {
1250 PROC_LOCK_ASSERT(p, MA_OWNED);
1263 if (
set != NULL && rebase && nroot != tdroot) {
1264 cpusetid_t base_id, root_id;
1266 root_id = td->td_ucred->cr_prison->pr_cpuset->cs_id;
1269 if (base_id != root_id) {
1271 &freelist, &domainlist);
1272 if (error == EDEADLK &&
1286 FOREACH_THREAD_IN_PROC(p, td) {
1303 FOREACH_THREAD_IN_PROC(p, td) {
1307 &nset, &freelist, &domainlist);
1310 domain, &nset, &freelist, &domainlist);
1321 if (base != NULL && base !=
set)
1323 while ((nset = LIST_FIRST(&droplist)) != NULL)
1339 for (i = 0; i < __bitset_words(setlen); i++) {
1348 if (bufsiz <
sizeof(__STRING(ULONG_MAX)))
1350 bytes =
snprintf(p, bufsiz,
"%lx",
set->__bits[i]);
1363 BIT_ZERO(setlen,
set);
1365 for (i = 0; i < __bitset_words(setlen); i++) {
1370 ret =
sscanf(p,
"%lx", &
set->__bits[i]);
1371 if (ret == 0 || ret == -1)
1373 while (isxdigit(*p))
1401 if (strlen(
buf) > CPUSETBUFSIZ - 1)
1422 char buf[DOMAINSETBUFSIZ];
1423 struct domainset *dset;
1424 struct domainset key;
1425 int policy, prefer, error;
1428 dset = *(
struct domainset **)arg1;
1433 (
const struct bitset *)&dset->ds_mask, DOMAINSET_SETSIZE);
1434 sprintf(p,
":%d:%d", dset->ds_policy, dset->ds_prefer);
1438 if (error != 0 || req->newptr == NULL)
1444 memset(&key, 0,
sizeof(key));
1446 DOMAINSET_SETSIZE,
buf)];
1449 if (
sscanf(p,
":%d:%d", &policy, &prefer) != 2)
1451 key.ds_policy = policy;
1452 key.ds_prefer = prefer;
1458 *(
struct domainset **)arg1 = dset;
1469 struct setlist cpusets;
1470 struct domainlist domainlist;
1471 struct cpuset *nset;
1485 &cpusets, &domainlist);
1520 CPU_SET(cpu, &
mask);
1531 struct domainset *dset;
1535 DOMAINSET_COPY(&all_domains, &dset->ds_mask);
1536 dset->ds_policy = DOMAINSET_POLICY_FIRSTTOUCH;
1537 dset->ds_prefer = -1;
1541 DOMAINSET_COPY(&all_domains, &dset->ds_mask);
1542 dset->ds_policy = DOMAINSET_POLICY_INTERLEAVE;
1543 dset->ds_prefer = -1;
1547 DOMAINSET_COPY(&all_domains, &dset->ds_mask);
1548 dset->ds_policy = DOMAINSET_POLICY_ROUNDROBIN;
1549 dset->ds_prefer = -1;
1552 for (i = 0; i < vm_ndomains; i++) {
1554 DOMAINSET_ZERO(&dset->ds_mask);
1555 DOMAINSET_SET(i, &dset->ds_mask);
1556 dset->ds_policy = DOMAINSET_POLICY_ROUNDROBIN;
1560 DOMAINSET_COPY(&all_domains, &dset->ds_mask);
1561 dset->ds_policy = DOMAINSET_POLICY_PREFER;
1562 dset->ds_prefer = i;
1573 struct domainset *dset, *tmp;
1575 mtx_init(&
cpuset_lock,
"cpuset", NULL, MTX_SPIN | MTX_RECURSE);
1581 kernel_object->domain.dr_policy =
domainset2;
1586 LIST_REMOVE(dset, ds_link);
1608 cpuset_zone = uma_zcreate(
"cpuset",
sizeof(
struct cpuset), NULL, NULL,
1609 NULL, NULL, UMA_ALIGN_CACHE, 0);
1610 domainset_zone = uma_zcreate(
"domainset",
sizeof(
struct domainset),
1611 NULL, NULL, NULL, NULL, UMA_ALIGN_CACHE, 0);
1619 LIST_INIT(&
set->cs_children);
1621 refcount_init(&
set->cs_ref, 1);
1622 set->cs_flags = CPU_SET_ROOT | CPU_SET_RDONLY;
1632 KASSERT(error == 0, (
"Error creating default set: %d\n", error));
1639 KASSERT(error == 0, (
"Error creating kernel set: %d\n", error));
1652 for (i = 0; i < MAXMEMDOM; i++)
1667 set = td->td_cpuset;
1688 KASSERT(
pr != NULL, (
"[%s:%d] invalid pr", __func__, __LINE__));
1689 KASSERT(setp != NULL, (
"[%s:%d] invalid setp", __func__, __LINE__));
1696 KASSERT(
set != NULL, (
"[%s:%d] cpuset_create returned invalid data",
1697 __func__, __LINE__));
1700 set->cs_flags |= CPU_SET_ROOT;
1711 KASSERT(p != NULL, (
"[%s:%d] invalid proc", __func__, __LINE__));
1712 KASSERT(
set != NULL, (
"[%s:%d] invalid set", __func__, __LINE__));
1730 if (IN_CAPABILITY_MODE(td)) {
1731 if (
level != CPU_LEVEL_WHICH)
1733 if (which != CPU_WHICH_TID && which != CPU_WHICH_PID)
1736 !(which == CPU_WHICH_TID &&
id == td->td_tid) &&
1737 !(which == CPU_WHICH_PID &&
id == td->td_proc->p_pid))
1743#ifndef _SYS_SYSPROTO_H_
1751 struct cpuset *root;
1763 error = copyout(&
set->cs_id, uap->
setid,
sizeof(
set->cs_id));
1770#ifndef _SYS_SYSPROTO_H_
1786 id_t
id, cpusetid_t setid)
1794 if (which != CPU_WHICH_PID)
1804#ifndef _SYS_SYSPROTO_H_
1822 id_t
id, cpusetid_t *setid)
1824 struct cpuset *nset;
1831 if (
level == CPU_LEVEL_WHICH && which != CPU_WHICH_CPUSET)
1844 case CPU_WHICH_CPUSET:
1845 case CPU_WHICH_JAIL:
1848 case CPU_WHICH_DOMAIN:
1852 case CPU_LEVEL_ROOT:
1857 case CPU_LEVEL_CPUSET:
1859 case CPU_LEVEL_WHICH:
1865 error = copyout(&tmpid, setid,
sizeof(tmpid));
1870#ifndef _SYS_SYSPROTO_H_
1889 id_t
id,
size_t cpusetsize, cpuset_t *maskp)
1892 struct cpuset *nset;
1899 if (cpusetsize <
sizeof(cpuset_t) || cpusetsize > CPU_MAXSIZE / NBBY)
1905 mask =
malloc(size, M_TEMP, M_WAITOK | M_ZERO);
1910 case CPU_LEVEL_ROOT:
1911 case CPU_LEVEL_CPUSET:
1919 case CPU_WHICH_CPUSET:
1920 case CPU_WHICH_JAIL:
1923 case CPU_WHICH_INTRHANDLER:
1924 case CPU_WHICH_ITHREAD:
1925 case CPU_WHICH_DOMAIN:
1929 if (
level == CPU_LEVEL_ROOT)
1933 CPU_COPY(&nset->cs_mask,
mask);
1936 case CPU_LEVEL_WHICH:
1940 CPU_COPY(&ttd->td_cpuset->cs_mask,
mask);
1944 FOREACH_THREAD_IN_PROC(p, ttd) {
1946 CPU_OR(
mask,
mask, &ttd->td_cpuset->cs_mask);
1950 case CPU_WHICH_CPUSET:
1951 case CPU_WHICH_JAIL:
1952 CPU_COPY(&
set->cs_mask,
mask);
1955 case CPU_WHICH_INTRHANDLER:
1956 case CPU_WHICH_ITHREAD:
1959 case CPU_WHICH_DOMAIN:
1960 if (id < 0 || id >= MAXMEMDOM)
1976 error = copyout(
mask, maskp, size);
1982#ifndef _SYS_SYSPROTO_H_
2001 id_t
id,
size_t cpusetsize,
const cpuset_t *maskp)
2003 struct cpuset *nset;
2010 if (cpusetsize <
sizeof(cpuset_t) || cpusetsize > CPU_MAXSIZE / NBBY)
2015 mask =
malloc(cpusetsize, M_TEMP, M_WAITOK | M_ZERO);
2016 error = copyin(maskp,
mask, cpusetsize);
2022 if (cpusetsize >
sizeof(cpuset_t)) {
2026 end = cp = (
char *)&
mask->__bits;
2028 cp +=
sizeof(cpuset_t);
2035 if (CPU_EMPTY(
mask)) {
2040 case CPU_LEVEL_ROOT:
2041 case CPU_LEVEL_CPUSET:
2053 case CPU_WHICH_CPUSET:
2054 case CPU_WHICH_JAIL:
2057 case CPU_WHICH_INTRHANDLER:
2058 case CPU_WHICH_ITHREAD:
2059 case CPU_WHICH_DOMAIN:
2063 if (
level == CPU_LEVEL_ROOT)
2071 case CPU_LEVEL_WHICH:
2079 case CPU_WHICH_CPUSET:
2080 case CPU_WHICH_JAIL:
2088 case CPU_WHICH_INTRHANDLER:
2089 case CPU_WHICH_ITHREAD:
2106#ifndef _SYS_SYSPROTO_H_
2126 id_t
id,
size_t domainsetsize, domainset_t *maskp,
int *policyp)
2128 struct domainset outset;
2130 struct cpuset *nset;
2132 struct domainset *dset;
2137 if (domainsetsize <
sizeof(domainset_t) ||
2138 domainsetsize > DOMAINSET_MAXSIZE / NBBY)
2143 mask =
malloc(domainsetsize, M_TEMP, M_WAITOK | M_ZERO);
2144 bzero(&outset,
sizeof(outset));
2149 case CPU_LEVEL_ROOT:
2150 case CPU_LEVEL_CPUSET:
2158 case CPU_WHICH_CPUSET:
2159 case CPU_WHICH_JAIL:
2162 case CPU_WHICH_INTRHANDLER:
2163 case CPU_WHICH_ITHREAD:
2164 case CPU_WHICH_DOMAIN:
2168 if (
level == CPU_LEVEL_ROOT)
2175 case CPU_LEVEL_WHICH:
2183 FOREACH_THREAD_IN_PROC(p, ttd) {
2185 dset = ttd->td_cpuset->cs_domain;
2187 DOMAINSET_OR(&outset.ds_mask, &dset->ds_mask);
2189 outset.ds_policy = dset->ds_policy;
2190 outset.ds_prefer = dset->ds_prefer;
2194 case CPU_WHICH_CPUSET:
2195 case CPU_WHICH_JAIL:
2199 case CPU_WHICH_INTRHANDLER:
2200 case CPU_WHICH_ITHREAD:
2201 case CPU_WHICH_DOMAIN:
2218 if (outset.ds_policy == DOMAINSET_POLICY_PREFER) {
2219 DOMAINSET_ZERO(&outset.ds_mask);
2220 DOMAINSET_SET(outset.ds_prefer, &outset.ds_mask);
2222 DOMAINSET_COPY(&outset.ds_mask,
mask);
2224 error = copyout(
mask, maskp, domainsetsize);
2226 if (suword32(policyp, outset.ds_policy) != 0)
2233#ifndef _SYS_SYSPROTO_H_
2253 id_t
id,
size_t domainsetsize,
const domainset_t *maskp,
int policy)
2255 struct cpuset *nset;
2263 if (domainsetsize <
sizeof(domainset_t) ||
2264 domainsetsize > DOMAINSET_MAXSIZE / NBBY)
2266 if (policy <= DOMAINSET_POLICY_INVALID ||
2267 policy > DOMAINSET_POLICY_MAX)
2273 mask =
malloc(domainsetsize, M_TEMP, M_WAITOK | M_ZERO);
2274 error = copyin(maskp,
mask, domainsetsize);
2280 if (domainsetsize >
sizeof(domainset_t)) {
2284 end = cp = (
char *)&
mask->__bits;
2285 end += domainsetsize;
2286 cp +=
sizeof(domainset_t);
2293 if (DOMAINSET_EMPTY(
mask)) {
2298 domain.ds_policy = policy;
2303 if (!DOMAINSET_SUBSET(&all_domains, &
domain.ds_mask)) {
2309 if (policy == DOMAINSET_POLICY_PREFER) {
2311 if (DOMAINSET_COUNT(&
domain.ds_mask) != 1) {
2317 DOMAINSET_COPY(&all_domains, &
domain.ds_mask);
2328 case CPU_LEVEL_ROOT:
2329 case CPU_LEVEL_CPUSET:
2341 case CPU_WHICH_CPUSET:
2342 case CPU_WHICH_JAIL:
2345 case CPU_WHICH_INTRHANDLER:
2346 case CPU_WHICH_ITHREAD:
2347 case CPU_WHICH_DOMAIN:
2351 if (
level == CPU_LEVEL_ROOT)
2359 case CPU_LEVEL_WHICH:
2367 case CPU_WHICH_CPUSET:
2368 case CPU_WHICH_JAIL:
2376 case CPU_WHICH_INTRHANDLER:
2377 case CPU_WHICH_ITHREAD:
2395ddb_display_bitset(
const struct bitset *
set,
int size)
2399 for (once = 0, bit = 0; bit < size; bit++) {
2400 if (CPU_ISSET(bit,
set)) {
2402 db_printf(
"%d", bit);
2405 db_printf(
",%d", bit);
2409 db_printf(
"<none>");
2413ddb_display_cpuset(
const cpuset_t *
set)
2415 ddb_display_bitset((
const struct bitset *)
set, CPU_SETSIZE);
2419ddb_display_domainset(
const domainset_t *
set)
2421 ddb_display_bitset((
const struct bitset *)
set, DOMAINSET_SETSIZE);
2424DB_SHOW_COMMAND(cpusets, db_show_cpusets)
2429 db_printf(
"set=%p id=%-6u ref=%-6d flags=0x%04x parent id=%d\n",
2430 set,
set->cs_id, refcount_load(&
set->cs_ref),
set->cs_flags,
2431 (
set->cs_parent != NULL) ?
set->cs_parent->cs_id : 0);
2432 db_printf(
" cpu mask=");
2433 ddb_display_cpuset(&
set->cs_mask);
2435 db_printf(
" domain policy %d prefer %d mask=",
2436 set->cs_domain->ds_policy,
set->cs_domain->ds_prefer);
2437 ddb_display_domainset(&
set->cs_domain->ds_mask);
2444DB_SHOW_COMMAND(domainsets, db_show_domainsets)
2446 struct domainset *
set;
2449 db_printf(
"set=%p policy %d prefer %d cnt %d\n",
2451 db_printf(
" mask =");
2452 ddb_display_domainset(&
set->ds_mask);
const struct cf_level * level
static long empty[CPUSTATES]
int sys_cpuset_getdomain(struct thread *td, struct cpuset_getdomain_args *uap)
static int cpuset_setproc_test_maskthread(struct cpuset *tdset, cpuset_t *mask, struct domainset *domain)
static int cpuset_shadow(struct cpuset *set, struct cpuset **nsetp, const cpuset_t *mask, const struct domainset *domain, struct setlist *cpusets, struct domainlist *domains)
int kern_cpuset_getaffinity(struct thread *td, cpulevel_t level, cpuwhich_t which, id_t id, size_t cpusetsize, cpuset_t *maskp)
int cpuset_setithread(lwpid_t id, int cpu)
static struct cpuset * cpuset_refroot(struct cpuset *set)
static int cpuset_create(struct cpuset **setp, struct cpuset *parent, const cpuset_t *mask)
static uma_zone_t cpuset_zone
static int cpuset_testupdate(struct cpuset *set, cpuset_t *mask, int augment_mask)
static struct cpuset * cpuset_kernel
static struct domainlist cpuset_domains
static struct domainset * domainset_shadow(const struct domainset *pdomain, const struct domainset *domain, struct domainlist *freelist)
struct domainset __read_mostly domainset_roundrobin
int sys_cpuset(struct thread *td, struct cpuset_args *uap)
static void domainset_notify(void)
static void cpuset_rel_defer(struct setlist *head, struct cpuset *set)
static int cpuset_testshadow(struct cpuset *set, const cpuset_t *mask, const struct domainset *domain)
int kern_cpuset_setdomain(struct thread *td, cpulevel_t level, cpuwhich_t which, id_t id, size_t domainsetsize, const domainset_t *maskp, int policy)
static uma_zone_t domainset_zone
static void cpuset_freelist_init(struct setlist *list, int count)
static int cpuset_testupdate_domain(struct cpuset *set, struct domainset *dset, struct domainset *orig, int *count, int augment_mask __unused)
int sys_cpuset_getid(struct thread *td, struct cpuset_getid_args *uap)
static int domainset_valid(const struct domainset *, const struct domainset *)
static void domainset_copy(const struct domainset *from, struct domainset *to)
int sys_cpuset_setaffinity(struct thread *td, struct cpuset_setaffinity_args *uap)
struct cpuset * cpuset_ref(struct cpuset *set)
void cpuset_kernthread(struct thread *td)
struct domainset __read_mostly domainset_firsttouch
int cpusetobj_strscan(cpuset_t *set, const char *buf)
static struct cpuset * cpuset_refbase(struct cpuset *set)
struct domainset __read_mostly domainset_interleave
static struct mtx cpuset_lock
static int cpuset_check_capabilities(struct thread *td, cpulevel_t level, cpuwhich_t which, id_t id)
struct domainset __read_mostly domainset_prefer[MAXMEMDOM]
void domainset_init(void)
static struct cpuset * cpuset_zero
void cpuset_rel(struct cpuset *set)
static struct cpuset * cpuset_getbase(struct cpuset *set)
int sys_cpuset_getaffinity(struct thread *td, struct cpuset_getaffinity_args *uap)
static void domainset_freelist_add(struct domainlist *list, int count)
struct domainset __read_mostly domainset_fixed[MAXMEMDOM]
static void cpuset_update_domain(struct cpuset *set, struct domainset *domain, struct domainset *orig, struct domainlist *domains)
static int cpuset_setproc_setthread(struct cpuset *tdset, struct cpuset *set, struct cpuset **nsetp, struct setlist *freelist, struct domainlist *domainlist)
static void cpuset_rel_complete(struct cpuset *set)
int kern_cpuset_setaffinity(struct thread *td, cpulevel_t level, cpuwhich_t which, id_t id, size_t cpusetsize, const cpuset_t *maskp)
int sys_cpuset_setid(struct thread *td, struct cpuset_setid_args *uap)
static void domainset_freelist_free(struct domainlist *list)
int kern_cpuset_getdomain(struct thread *td, cpulevel_t level, cpuwhich_t which, id_t id, size_t domainsetsize, domainset_t *maskp, int *policyp)
int sysctl_handle_domainset(SYSCTL_HANDLER_ARGS)
static struct domainset * domainset2
static bool domainset_empty_vm(struct domainset *domain)
char * cpusetobj_strprint(char *buf, const cpuset_t *set)
static int _cpuset_setthread(lwpid_t id, cpuset_t *mask, struct domainset *domain)
static struct domainset * domainset0
static struct domainset * _domainset_create(struct domainset *domain, struct domainlist *freelist)
static struct unrhdr * cpuset_unr
static int cpuset_setproc_maskthread(struct cpuset *tdset, cpuset_t *mask, struct domainset *domain, struct cpuset **nsetp, struct setlist *freelist, struct domainlist *domainlist)
static int domainset_restrict(const struct domainset *parent, const struct domainset *child)
int sys_cpuset_setdomain(struct thread *td, struct cpuset_setdomain_args *uap)
static int cpuset_modify_domain(struct cpuset *set, struct domainset *domain)
static void cpuset_freelist_add(struct setlist *list, int count)
static int bitset_strscan(struct bitset *set, int setlen, const char *buf)
struct domainset * domainset_create(const struct domainset *domain)
struct cpuset * cpuset_thread0(void)
static int cpuset_setproc_setthread_mask(struct cpuset *tdset, struct cpuset *set, cpuset_t *mask, struct domainset *domain)
static int cpuset_setproc_newbase(struct thread *td, struct cpuset *set, struct cpuset *nroot, struct cpuset **nsetp, struct setlist *cpusets, struct domainlist *domainlist)
void domainset_zero(void)
int cpuset_which(cpuwhich_t which, id_t id, struct proc **pp, struct thread **tdp, struct cpuset **setp)
static struct setlist cpuset_ids
static int cpuset_setproc(pid_t pid, struct cpuset *set, cpuset_t *mask, struct domainset *domain, bool rebase)
int cpuset_setproc_update_set(struct proc *p, struct cpuset *set)
int kern_cpuset_setid(struct thread *td, cpuwhich_t which, id_t id, cpusetid_t setid)
static int cpuset_init(struct cpuset *set, struct cpuset *parent, const cpuset_t *mask, struct domainset *domain, cpusetid_t id)
static int bitset_strprint(char *buf, size_t bufsiz, const struct bitset *set, int setlen)
static void cpuset_update(struct cpuset *set, cpuset_t *mask)
int kern_cpuset_getid(struct thread *td, cpulevel_t level, cpuwhich_t which, id_t id, cpusetid_t *setid)
int cpuset_create_root(struct prison *pr, struct cpuset **setp)
static void cpuset_freelist_free(struct setlist *list)
static int cpuset_modify(struct cpuset *set, cpuset_t *mask)
cpuset_t cpuset_domain[MAXMEMDOM]
static struct cpuset * cpuset_lookup(cpusetid_t setid, struct thread *td)
static struct cpuset * cpuset_default
static int cpuset_setproc_test_setthread(struct cpuset *tdset, struct cpuset *set)
static void domainset_freelist_init(struct domainlist *list, int count)
static int domainset_equal(const struct domainset *one, const struct domainset *two)
static struct cpuset * cpuset_update_thread(struct thread *td, struct cpuset *nset)
LIST_HEAD(domainlist, domainset)
int cpuset_setthread(lwpid_t id, cpuset_t *mask)
SYSCTL_INT(_kern_sched, OID_AUTO, cpusetsize, CTLFLAG_RD|CTLFLAG_CAPRD, SYSCTL_NULL_INT_PTR, sizeof(cpuset_t), "sizeof(cpuset_t)")
static struct cpuset * cpuset_getroot(struct cpuset *set)
int intr_getaffinity(int irq, int mode, void *m)
int intr_setaffinity(int irq, int mode, void *m)
struct prison * prison_find_child(struct prison *mypr, int prid)
void *() malloc(size_t size, struct malloc_type *mtp, int flags)
void free(void *addr, struct malloc_type *mtp)
static struct pollrec pr[POLL_LIST_LEN]
int priv_check(struct thread *td, int priv)
struct proc * pfind(pid_t pid)
struct sx __exclusive_cache_line allproc_lock
int p_cansched(struct thread *td, struct proc *p)
int sysctl_handle_string(SYSCTL_HANDLER_ARGS)
struct thread * tdfind(lwpid_t tid, pid_t pid)
struct iommu_domain ** domain
void sched_affinity(struct thread *td)
static bool kasan_enabled __read_mostly
int sprintf(char *buf, const char *cfmt,...)
int snprintf(char *str, size_t size, const char *format,...)
int sscanf(const char *ibuf, const char *fmt,...)
int alloc_unr(struct unrhdr *uh)
struct unrhdr * new_unrhdr(int low, int high, struct mtx *mutex)
void free_unr(struct unrhdr *uh, u_int item)