49#include <sys/domainset.h>
51#include <sys/kernel.h>
52#include <sys/malloc.h>
56#include <sys/rwlock.h>
58#include <sys/sysctl.h>
60#include <sys/vmmeter.h>
74 "Too many physsegs.");
89struct vm_phys_fictitious_seg;
91 struct vm_phys_fictitious_seg *);
93RB_HEAD(fict_tree, vm_phys_fictitious_seg) vm_phys_fictitious_tree =
94 RB_INITIALIZER(&vm_phys_fictitious_tree);
96struct vm_phys_fictitious_seg {
97 RB_ENTRY(vm_phys_fictitious_seg) node;
111 vm_phys_free_queues[MAXMEMDOM][VM_NFREELIST][VM_NFREEPOOL]
141#ifdef VM_FREELIST_DMA32
142#define VM_DMA32_BOUNDARY ((vm_paddr_t)1 << 32)
149#if defined(VM_LOWMEM_BOUNDARY) && defined(VM_DMA32_BOUNDARY)
150CTASSERT(VM_LOWMEM_BOUNDARY < VM_DMA32_BOUNDARY);
154SYSCTL_OID(_vm, OID_AUTO, phys_free,
155 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0,
160SYSCTL_OID(_vm, OID_AUTO, phys_segs,
161 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0,
166static int sysctl_vm_phys_locality(SYSCTL_HANDLER_ARGS);
167SYSCTL_OID(_vm, OID_AUTO, phys_locality,
168 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0,
169 sysctl_vm_phys_locality,
"A",
170 "Phys Locality Info");
173SYSCTL_INT(_vm, OID_AUTO, ndomains, CTLFLAG_RD,
174 &
vm_ndomains, 0,
"Number of physical memory domains available.");
177 u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment,
178 vm_paddr_t boundary);
182 int order,
int tail);
188vm_phys_fictitious_in_range(
struct vm_phys_fictitious_seg *p,
189 struct vm_phys_fictitious_seg *range)
192 KASSERT(range->start != 0 && range->end != 0,
193 (
"Invalid range passed on search for vm_fictitious page"));
194 if (p->start >= range->end)
196 if (p->start < range->start)
204 struct vm_phys_fictitious_seg *p2)
209 return (vm_phys_fictitious_in_range(p1, p2));
211 KASSERT(p2->end != 0,
212 (
"Invalid range passed as second parameter to vm fictitious comparison"));
215 if (p1->end <= p2->start)
217 if (p1->start >= p2->end)
220 panic(
"Trying to add overlapping vm fictitious ranges:\n"
221 "[%#jx:%#jx] and [%#jx:%#jx]", (uintmax_t)p1->start,
222 (uintmax_t)p1->end, (uintmax_t)p2->start, (uintmax_t)p2->end);
235 DOMAINSET_ZERO(&mask);
243 if (prefer != -1 && DOMAINSET_ISSET(prefer, &mask))
245 if (DOMAINSET_EMPTY(&mask))
246 panic(
"vm_phys_domain_match: Impossible constraint");
247 return (DOMAINSET_FFS(&mask) - 1);
262 int dom, error, flind, oind, pind;
264 error = sysctl_wire_old_buffer(req, 0);
267 sbuf_new_for_sysctl(&sbuf, NULL, 128 *
vm_ndomains, req);
269 sbuf_printf(&sbuf,
"\nDOMAIN %d:\n", dom);
270 for (flind = 0; flind < vm_nfreelists; flind++) {
271 sbuf_printf(&sbuf,
"\nFREE LIST %d:\n"
272 "\n ORDER (SIZE) | NUMBER"
274 for (pind = 0; pind < VM_NFREEPOOL; pind++)
275 sbuf_printf(&sbuf,
" | POOL %d", pind);
276 sbuf_printf(&sbuf,
"\n-- ");
277 for (pind = 0; pind < VM_NFREEPOOL; pind++)
278 sbuf_printf(&sbuf,
"-- -- ");
279 sbuf_printf(&sbuf,
"--\n");
280 for (oind = VM_NFREEORDER - 1; oind >= 0; oind--) {
281 sbuf_printf(&sbuf,
" %2d (%6dK)", oind,
282 1 << (PAGE_SHIFT - 10 + oind));
283 for (pind = 0; pind < VM_NFREEPOOL; pind++) {
284 fl = vm_phys_free_queues[dom][flind][pind];
285 sbuf_printf(&sbuf,
" | %6d",
288 sbuf_printf(&sbuf,
"\n");
292 error = sbuf_finish(&sbuf);
307 error = sysctl_wire_old_buffer(req, 0);
310 sbuf_new_for_sysctl(&sbuf, NULL, 128, req);
312 sbuf_printf(&sbuf,
"\nSEGMENT %d:\n\n", segind);
314 sbuf_printf(&sbuf,
"start: %#jx\n",
315 (uintmax_t)seg->
start);
316 sbuf_printf(&sbuf,
"end: %#jx\n",
317 (uintmax_t)seg->
end);
318 sbuf_printf(&sbuf,
"domain: %d\n", seg->
domain);
319 sbuf_printf(&sbuf,
"free list: %p\n", seg->
free_queues);
321 error = sbuf_finish(&sbuf);
334 if (mem_locality == NULL)
349sysctl_vm_phys_locality(SYSCTL_HANDLER_ARGS)
354 error = sysctl_wire_old_buffer(req, 0);
357 sbuf_new_for_sysctl(&sbuf, NULL, 128, req);
359 sbuf_printf(&sbuf,
"\n");
362 sbuf_printf(&sbuf,
"%d: ", i);
366 sbuf_printf(&sbuf,
"\n");
368 error = sbuf_finish(&sbuf);
380 TAILQ_INSERT_TAIL(&fl[order].pl, m, listq);
382 TAILQ_INSERT_HEAD(&fl[order].pl, m, listq);
390 TAILQ_REMOVE(&fl[order].pl, m, listq);
392 m->order = VM_NFREEORDER;
404 (
"vm_phys_create_seg: increase VM_PHYSSEG_MAX"));
406 (
"vm_phys_create_seg: invalid domain provided"));
430 panic(
"Reached end of affinity info");
434 panic(
"No affinity info for start %jx",
458 KASSERT((
start & PAGE_MASK) == 0,
459 (
"vm_phys_define_seg: start is not page aligned"));
460 KASSERT((
end & PAGE_MASK) == 0,
461 (
"vm_phys_define_seg: end is not page aligned"));
468#ifdef VM_FREELIST_LOWMEM
469 if (paddr < VM_LOWMEM_BOUNDARY && end > VM_LOWMEM_BOUNDARY) {
471 paddr = VM_LOWMEM_BOUNDARY;
474#ifdef VM_FREELIST_DMA32
475 if (paddr < VM_DMA32_BOUNDARY && end > VM_DMA32_BOUNDARY) {
477 paddr = VM_DMA32_BOUNDARY;
492 struct vm_phys_seg *end_seg, *prev_seg, *seg, *tmp_seg;
494 int dom, flind, freelist, oind, pind, segind;
506#ifdef VM_FREELIST_LOWMEM
507 if (seg->
end <= VM_LOWMEM_BOUNDARY)
508 vm_freelist_to_flind[VM_FREELIST_LOWMEM] = 1;
511#ifdef VM_FREELIST_DMA32
513#ifdef VM_DMA32_NPAGES_THRESHOLD
519 npages > VM_DMA32_NPAGES_THRESHOLD &&
521 seg->
end <= VM_DMA32_BOUNDARY)
522 vm_freelist_to_flind[VM_FREELIST_DMA32] = 1;
526 npages += atop(seg->
end - seg->
start);
527 vm_freelist_to_flind[VM_FREELIST_DEFAULT] = 1;
531 for (freelist = 1; freelist < VM_NFREELIST; freelist++) {
532 vm_freelist_to_flind[freelist] +=
533 vm_freelist_to_flind[freelist - 1];
535 vm_nfreelists = vm_freelist_to_flind[VM_NFREELIST - 1];
536 KASSERT(vm_nfreelists > 0, (
"vm_phys_init: no free lists"));
538 for (freelist = 0; freelist < VM_NFREELIST; freelist++)
539 vm_freelist_to_flind[freelist]--;
545#ifdef VM_PHYSSEG_SPARSE
550#ifdef VM_PHYSSEG_SPARSE
552 npages += atop(seg->
end - seg->
start);
556#ifdef VM_FREELIST_LOWMEM
557 if (seg->
end <= VM_LOWMEM_BOUNDARY) {
558 flind = vm_freelist_to_flind[VM_FREELIST_LOWMEM];
560 (
"vm_phys_init: LOWMEM flind < 0"));
563#ifdef VM_FREELIST_DMA32
564 if (seg->
end <= VM_DMA32_BOUNDARY) {
565 flind = vm_freelist_to_flind[VM_FREELIST_DMA32];
567 (
"vm_phys_init: DMA32 flind < 0"));
571 flind = vm_freelist_to_flind[VM_FREELIST_DEFAULT];
573 (
"vm_phys_init: DEFAULT flind < 0"));
585 while (seg < end_seg) {
590 (
"vm_phys_init: free queues cannot span domains"));
593 for (tmp_seg = seg; tmp_seg < end_seg; tmp_seg++)
594 *tmp_seg = *(tmp_seg + 1);
605 for (flind = 0; flind < vm_nfreelists; flind++) {
606 for (pind = 0; pind < VM_NFREEPOOL; pind++) {
607 fl = vm_phys_free_queues[dom][flind][pind];
608 for (oind = 0; oind < VM_NFREEORDER; oind++)
609 TAILQ_INIT(&fl[oind].pl);
634 TUNABLE_INT_FETCH(
"vm.numa.disabled", &d);
641 mem_locality = locality;
668 while (oind > order) {
670 m_buddy = &m[1 << oind];
671 KASSERT(m_buddy->order == VM_NFREEORDER,
672 (
"vm_phys_split_pages: page %p has unexpected order %d",
673 m_buddy, m_buddy->order));
696 KASSERT(npages > 0, (
"vm_phys_enq_range: npages is 0"));
698 ((PAGE_SIZE << (fls(npages) - 1)) - 1)) == 0,
699 (
"vm_phys_enq_range: page %p and npages %u are misaligned",
702 KASSERT(m->order == VM_NFREEORDER,
703 (
"vm_phys_enq_range: page %p has unexpected order %d",
705 order = ffs(npages) - 1;
706 KASSERT(order < VM_NFREEORDER,
707 (
"vm_phys_enq_range: order %d is out of range", order));
712 }
while (npages > 0);
723 for (m_tmp = m; m_tmp < &m[1 << order]; m_tmp++)
744 int avail, end, flind, freelist, i, need, oind, pind;
747 (
"vm_phys_alloc_npages: domain %d is out of range", domain));
748 KASSERT(pool < VM_NFREEPOOL,
749 (
"vm_phys_alloc_npages: pool %d is out of range", pool));
750 KASSERT(npages <= 1 << (VM_NFREEORDER - 1),
751 (
"vm_phys_alloc_npages: npages %d is out of range", npages));
754 for (freelist = 0; freelist < VM_NFREELIST; freelist++) {
755 flind = vm_freelist_to_flind[freelist];
758 fl = vm_phys_free_queues[domain][flind][pool];
759 for (oind = 0; oind < VM_NFREEORDER; oind++) {
760 while ((m = TAILQ_FIRST(&fl[oind].
pl)) != NULL) {
763 need = imin(npages - i, avail);
764 for (end = i + need; i < end;)
774 }
else if (i == npages)
778 for (oind = VM_NFREEORDER - 1; oind >= 0; oind--) {
779 for (pind = 0; pind < VM_NFREEPOOL; pind++) {
780 alt = vm_phys_free_queues[domain][flind][pind];
781 while ((m = TAILQ_FIRST(&alt[oind].
pl)) !=
786 need = imin(npages - i, avail);
787 for (end = i + need; i < end;)
798 }
else if (i == npages)
819 for (freelist = 0; freelist < VM_NFREELIST; freelist++) {
839 int oind, pind, flind;
842 (
"vm_phys_alloc_freelist_pages: domain %d is out of range",
844 KASSERT(freelist < VM_NFREELIST,
845 (
"vm_phys_alloc_freelist_pages: freelist %d is out of range",
847 KASSERT(pool < VM_NFREEPOOL,
848 (
"vm_phys_alloc_freelist_pages: pool %d is out of range", pool));
849 KASSERT(order < VM_NFREEORDER,
850 (
"vm_phys_alloc_freelist_pages: order %d is out of range", order));
852 flind = vm_freelist_to_flind[freelist];
858 fl = &vm_phys_free_queues[domain][flind][pool][0];
859 for (oind = order; oind < VM_NFREEORDER; oind++) {
860 m = TAILQ_FIRST(&fl[oind].
pl);
875 for (oind = VM_NFREEORDER - 1; oind >= order; oind--) {
876 for (pind = 0; pind < VM_NFREEPOOL; pind++) {
877 alt = &vm_phys_free_queues[domain][flind][pind][0];
878 m = TAILQ_FIRST(&alt[oind].
pl);
902 if (pa >= seg->
start && pa < seg->
end)
911 struct vm_phys_fictitious_seg tmp, *seg;
919 seg = RB_FIND(fict_tree, &vm_phys_fictitious_tree, &tmp);
924 m = &seg->first_page[atop(pa - seg->start)];
925 KASSERT((m->flags &
PG_FICTITIOUS) != 0, (
"%p not fictitious", m));
932 long page_count, vm_memattr_t memattr)
936 bzero(range, page_count *
sizeof(*range));
937 for (i = 0; i < page_count; i++) {
939 range[i].oflags &= ~VPO_UNMANAGED;
946 vm_memattr_t memattr)
948 struct vm_phys_fictitious_seg *seg;
951#ifdef VM_PHYSSEG_DENSE
957 (
"Start of segment isn't less than end (start: %jx end: %jx)",
958 (uintmax_t)start, (uintmax_t)end));
960 page_count = (end - start) / PAGE_SIZE;
962#ifdef VM_PHYSSEG_DENSE
979 page_count -= dpage_count;
980 start += ptoa(dpage_count);
998 end -= ptoa(dpage_count);
999 page_count -= dpage_count;
1010 fp = malloc(page_count *
sizeof(
struct vm_page), M_FICT_PAGES,
1012#ifdef VM_PHYSSEG_DENSE
1017 seg = malloc(
sizeof(*seg), M_FICT_PAGES, M_WAITOK | M_ZERO);
1020 seg->first_page = fp;
1023 RB_INSERT(fict_tree, &vm_phys_fictitious_tree, seg);
1032 struct vm_phys_fictitious_seg *seg, tmp;
1033#ifdef VM_PHYSSEG_DENSE
1037 KASSERT(start < end,
1038 (
"Start of segment isn't less than end (start: %jx end: %jx)",
1039 (uintmax_t)start, (uintmax_t)end));
1041#ifdef VM_PHYSSEG_DENSE
1070 "Unregistering not registered fictitious range [%#jx:%#jx]",
1071 (uintmax_t)start, (uintmax_t)end);
1078 seg = RB_FIND(fict_tree, &vm_phys_fictitious_tree, &tmp);
1079 if (seg->start != start || seg->end != end) {
1082 "Unregistering not registered fictitious range [%#jx:%#jx]",
1083 (uintmax_t)start, (uintmax_t)end);
1085 RB_REMOVE(fict_tree, &vm_phys_fictitious_tree, seg);
1087 free(seg->first_page, M_FICT_PAGES);
1088 free(seg, M_FICT_PAGES);
1104 KASSERT(m->order == VM_NFREEORDER,
1105 (
"vm_phys_free_pages: page %p has unexpected order %d",
1107 KASSERT(m->pool < VM_NFREEPOOL,
1108 (
"vm_phys_free_pages: page %p has unexpected pool %d",
1110 KASSERT(order < VM_NFREEORDER,
1111 (
"vm_phys_free_pages: order %d is out of range", order));
1114 if (order < VM_NFREEORDER - 1) {
1117 pa ^= ((vm_paddr_t)1 << (PAGE_SHIFT + order));
1118 if (pa < seg->
start || pa >= seg->
end)
1121 if (m_buddy->order != order)
1125 if (m_buddy->pool != m->pool)
1128 pa &= ~(((vm_paddr_t)1 << (PAGE_SHIFT + order)) - 1);
1130 }
while (order < VM_NFREEORDER - 1);
1151 VM_NFREEORDER - 1));
1177 while ((order =
max_order(m)) < VM_NFREEORDER - 1 &&
1178 m + (1 << order) <= m_end) {
1180 (
"%s: page range [%p,%p) spans multiple segments",
1181 __func__, m_end - npages, m));
1186 while (m + (1 << order) <= m_end) {
1188 (
"%s: page range [%p,%p) spans multiple segments",
1189 __func__, m_end - npages, m));
1196 (
"%s: page range [%p,%p) spans multiple segments",
1197 __func__, m_end - npages, m));
1198 order = flsl(m_end - m) - 1;
1212 int order_start, order_end;
1213 vm_page_t m_start, m_end;
1219 if (order_start < VM_NFREEORDER - 1)
1220 m_start += 1 << order_start;
1223 if (order_end < VM_NFREEORDER - 1)
1224 m_end -= 1 << order_end;
1229 if (m_start < m_end)
1231 if (order_start < VM_NFREEORDER - 1)
1233 if (order_end < VM_NFREEORDER - 1)
1250 u_long alignment, vm_paddr_t boundary,
int options)
1253 vm_page_t m_end, m_run, m_start;
1257 KASSERT(npages > 0, (
"npages is 0"));
1258 KASSERT(powerof2(alignment), (
"alignment is not a power of 2"));
1259 KASSERT(powerof2(boundary), (
"boundary is not a power of 2"));
1266 if (seg->
start >= high)
1268 if (low >= seg->
end)
1270 if (low <= seg->
start)
1274 if (high < seg->
end)
1282 alignment, boundary, options);
1301 vm_paddr_t pa, pa_half;
1302 vm_page_t m_set, m_tmp;
1312 for (m_set = m, order = 0; m_set->order == VM_NFREEORDER &&
1313 order < VM_NFREEORDER - 1; ) {
1315 pa = m->phys_addr & (~(vm_paddr_t)0 << (PAGE_SHIFT + order));
1316 if (pa >= seg->
start)
1321 if (m_set->order < order)
1323 if (m_set->order == VM_NFREEORDER)
1325 KASSERT(m_set->order < VM_NFREEORDER,
1326 (
"vm_phys_unfree_page: page %p has unexpected order %d",
1327 m_set, m_set->order));
1336 order = m_set->order;
1340 pa_half = m_set->phys_addr ^ (1 << (PAGE_SHIFT + order));
1341 if (m->phys_addr < pa_half)
1349 KASSERT(m_set == m, (
"vm_phys_unfree_page: fatal inconsistency"));
1365 u_long alignment, vm_paddr_t boundary)
1367 vm_paddr_t pa_end, pa_start;
1372 KASSERT(npages > 0, (
"npages is 0"));
1373 KASSERT(powerof2(alignment), (
"alignment is not a power of 2"));
1374 KASSERT(powerof2(boundary), (
"boundary is not a power of 2"));
1383 if (low >= seg->
end)
1385 if (low <= seg->
start)
1386 pa_start = seg->
start;
1389 if (high < seg->
end)
1393 if (pa_end - pa_start < ptoa(npages))
1396 alignment, boundary);
1409 vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary)
1412 vm_paddr_t pa, pa_end, size;
1415 int oind, order, pind;
1417 KASSERT(npages > 0, (
"npages is 0"));
1418 KASSERT(powerof2(alignment), (
"alignment is not a power of 2"));
1419 KASSERT(powerof2(boundary), (
"boundary is not a power of 2"));
1422 order = flsl(npages - 1);
1424 size = npages << PAGE_SHIFT;
1425 for (oind = min(order, VM_NFREEORDER - 1); oind < VM_NFREEORDER;
1427 for (pind = 0; pind < VM_NFREEPOOL; pind++) {
1429 TAILQ_FOREACH(m_ret, &fl[oind].
pl, listq) {
1438 if (pa < low || pa_end > high ||
1446 if (order < VM_NFREEORDER)
1455 pa < seg->start || seg->
end < pa_end)
1465 (PAGE_SHIFT + VM_NFREEORDER - 1);
1468 }
while (VM_NFREEORDER - 1 == seg->
first_page[
1469 atop(pa - seg->
start)].order);
1475 for (m = m_ret; m < &m_ret[npages]; m = &m[1 << oind]) {
1478 if (m->pool != VM_FREEPOOL_DEFAULT)
1482 npages_end = roundup2(npages, 1 << oind);
1483 if (npages < npages_end) {
1502 panic(
"Improperly terminated phys_avail %d entries", i);
1514 panic(
"Unaligned phys_avail[%d]: %#jx", i,
1517 panic(
"Unaligned phys_avail[%d + 1]: %#jx", i,
1520 panic(
"phys_avail[%d] start %#jx < end %#jx", i,
1529vm_phys_avail_find(vm_paddr_t pa)
1546 vm_paddr_t sz, largesz;
1580 panic(
"vm_phys_avail_split: invalid address");
1619 panic(
"%s: called after initialization", __func__);
1621 panic(
"%s: ran out of early segments", __func__);
1635 int i, mem_index, biggestone;
1636 vm_paddr_t pa, mem_start, mem_end, size, biggestsize, align;
1639 (
"%s: invalid domain index %d", __func__,
domain));
1658 if (size > biggestsize) {
1674 for (i = 0;
phys_avail[i + 1] != 0; i += 2) {
1676 if (
phys_avail[i+1] - alloc_size < mem_start ||
1680 if (size > biggestsize) {
1685 alloc_size = round_page(alloc_size);
1690 if (alloc_size == PAGE_SIZE) {
1700 align =
phys_avail[biggestone + 1] & (alloc_size - 1);
1701 if (alloc_size + align > biggestsize)
1702 panic(
"cannot find a large enough size\n");
1721 for (i = 0;
phys_avail[i + 1] != 0; i += 2) {
1755DB_SHOW_COMMAND(freepages, db_show_freepages)
1758 int flind, oind, pind, dom;
1761 db_printf(
"DOMAIN: %d\n", dom);
1762 for (flind = 0; flind < vm_nfreelists; flind++) {
1763 db_printf(
"FREE LIST %d:\n"
1764 "\n ORDER (SIZE) | NUMBER"
1766 for (pind = 0; pind < VM_NFREEPOOL; pind++)
1767 db_printf(
" | POOL %d", pind);
1769 for (pind = 0; pind < VM_NFREEPOOL; pind++)
1770 db_printf(
"-- -- ");
1772 for (oind = VM_NFREEORDER - 1; oind >= 0; oind--) {
1773 db_printf(
" %2.2d (%6.6dK)", oind,
1774 1 << (PAGE_SHIFT - 10 + oind));
1775 for (pind = 0; pind < VM_NFREEPOOL; pind++) {
1776 fl = vm_phys_free_queues[dom][flind][pind];
1777 db_printf(
" | %6.6d", fl[oind].
lcnt);
#define VM_NFREEORDER_MAX
SYSCTL_INT(_vm_memguard, OID_AUTO, options, CTLFLAG_RWTUN, &memguard_options, 0, "MemGuard options:\n" "\t0x001 - add guard pages around each allocation\n" "\t0x002 - always use MemGuard for allocations over a page\n" "\t0x004 - guard uma(9) zones with UMA_ZONE_NOFREE flag")
struct vm_freelist(* free_queues)[VM_NFREEPOOL][VM_NFREEORDER_MAX]
vm_paddr_t dump_avail[PHYS_AVAIL_COUNT]
static bool vm_addr_ok(vm_paddr_t pa, vm_paddr_t size, u_long alignment, vm_paddr_t boundary)
CTASSERT((ZERO_REGION_SIZE &PAGE_MASK)==0)
vm_page_t PHYS_TO_VM_PAGE(vm_paddr_t pa)
void vm_page_initfake(vm_page_t m, vm_paddr_t paddr, vm_memattr_t memattr)
vm_page_t vm_page_scan_contig(u_long npages, vm_page_t m_start, vm_page_t m_end, u_long alignment, vm_paddr_t boundary, int options)
#define VM_PAGE_TO_PHYS(entry)
#define vm_domain_free_assert_locked(n)
static struct vm_domain * vm_pagequeue_domain(vm_page_t m)
#define PHYS_AVAIL_ENTRIES
static struct vm_freelist __aligned(CACHE_LINE_SIZE)
void vm_phys_early_startup(void)
void vm_phys_fictitious_unreg_range(vm_paddr_t start, vm_paddr_t end)
MALLOC_DEFINE(M_FICT_PAGES, "vm_fictitious", "Fictitious VM pages")
int vm_phys_fictitious_reg_range(vm_paddr_t start, vm_paddr_t end, vm_memattr_t memattr)
static void vm_phys_enq_range(vm_page_t m, u_int npages, struct vm_freelist *fl, int tail)
static void vm_phys_avail_check(int i)
static void vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end)
bool vm_phys_is_dumpable(vm_paddr_t pa)
static struct vm_phys_seg vm_phys_early_segs[8]
vm_page_t vm_phys_fictitious_to_vm_page(vm_paddr_t pa)
static int max_order(vm_page_t m)
vm_page_t vm_phys_alloc_contig(int domain, u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary)
static __inline void vm_phys_split_pages(vm_page_t m, int oind, struct vm_freelist *fl, int order, int tail)
vm_page_t vm_phys_scan_contig(int domain, u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary, int options)
int __read_mostly vm_ndomains
int vm_phys_alloc_npages(int domain, int pool, int npages, vm_page_t ma[])
static void _vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end, int domain)
vm_paddr_t vm_phys_avail_size(int i)
static struct rwlock_padalign vm_phys_fictitious_reg_lock
static vm_page_t vm_phys_alloc_seg_contig(struct vm_phys_seg *seg, u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary)
static int vm_phys_avail_count(void)
void vm_phys_enqueue_contig(vm_page_t m, u_long npages)
void vm_phys_add_seg(vm_paddr_t start, vm_paddr_t end)
struct vm_phys_seg __read_mostly vm_phys_segs[VM_PHYSSEG_MAX]
RB_HEAD(fict_tree, vm_phys_fictitious_seg)
static int vm_phys_avail_split(vm_paddr_t pa, int i)
int vm_phys_mem_affinity(int f, int t)
vm_paddr_t vm_phys_early_alloc(int domain, size_t alloc_size)
void vm_phys_register_domains(int ndomains, struct mem_affinity *affinity, int *locality)
static int vm_phys_fictitious_cmp(struct vm_phys_fictitious_seg *, struct vm_phys_fictitious_seg *)
void vm_phys_free_pages(vm_page_t m, int order)
vm_page_t vm_phys_alloc_freelist_pages(int domain, int freelist, int pool, int order)
static int sysctl_vm_phys_segs(SYSCTL_HANDLER_ARGS)
int __read_mostly vm_phys_nsegs
static void vm_phys_set_pool(int pool, vm_page_t m, int order)
boolean_t vm_phys_unfree_page(vm_page_t m)
static int vm_phys_early_nsegs
vm_page_t vm_phys_alloc_pages(int domain, int pool, int order)
static void vm_freelist_rem(struct vm_freelist *fl, vm_page_t m, int order)
static void vm_freelist_add(struct vm_freelist *fl, vm_page_t m, int order, int tail)
int vm_phys_avail_largest(void)
static void vm_phys_fictitious_init_range(vm_page_t range, vm_paddr_t start, long page_count, vm_memattr_t memattr)
domainset_t __read_mostly all_domains
vm_page_t vm_phys_paddr_to_vm_page(vm_paddr_t pa)
void vm_phys_free_contig(vm_page_t m, u_long npages)
_Static_assert(sizeof(long) *NBBY >=VM_PHYSSEG_MAX, "Too many physsegs.")
void vm_phys_early_add_seg(vm_paddr_t start, vm_paddr_t end)
RB_GENERATE_STATIC(fict_tree, vm_phys_fictitious_seg, node, vm_phys_fictitious_cmp)
static int sysctl_vm_phys_free(SYSCTL_HANDLER_ARGS)
int vm_phys_domain_match(int prefer, vm_paddr_t low, vm_paddr_t high)