40#include <sys/kernel.h>
42#include <sys/malloc.h>
53#include <sys/linker.h>
54#include <sys/sysctl.h>
56#include <machine/elf.h>
60#include <security/mac/mac_framework.h>
63#include <vm/vm_param.h>
65#include <vm/vm_object.h>
66#include <vm/vm_kern.h>
67#include <vm/vm_extern.h>
72#include <sys/link_elf.h>
79 struct linker_file
lf;
141 const char *, linker_file_t *);
154 c_linker_sym_t *,
long *);
159 void ***,
void ***,
int *);
161 int (*)(
const char *,
void *),
void *);
163 linker_function_nameval_callback_t,
void *);
167static int elf_lookup(linker_file_t, Elf_Size,
int, Elf_Addr *);
189#if ELF_TARG_CLASS == ELFCLASS32
200 "Allow local symbols to participate in global module symbol resolution");
203 const void *
data,
int type, elf_lookup_fn lookup);
213static struct elf_set_head set_vnet_list;
226 TAILQ_FOREACH(iter, list, es_link) {
229 (
"linker sets intersection: to insert: 0x%jx-0x%jx; inserted: 0x%jx-0x%jx",
230 (uintmax_t)
set->es_start, (uintmax_t)
set->es_stop,
234 TAILQ_INSERT_BEFORE(iter,
set, es_link);
240 TAILQ_INSERT_TAIL(list,
set, es_link);
248 TAILQ_FOREACH(
set, list, es_link) {
253 *base =
set->es_base;
266 TAILQ_FOREACH(
set, list, es_link) {
270 TAILQ_REMOVE(list,
set, es_link);
275 KASSERT(0, (
"deleting unknown linker set (start = 0x%jx)",
280static void r_debug_state(
struct r_debug *,
struct link_map *);
285struct r_debug r_debug;
287#define GDB_STATE(s) do { \
288 r_debug.r_state = s; r_debug_state(NULL, NULL); \
295r_debug_state(
struct r_debug *dummy_one __unused,
296 struct link_map *dummy_two __unused)
301link_elf_add_gdb(
struct link_map *l)
303 struct link_map *prev;
307 if (r_debug.r_map == NULL) {
313 for (prev = r_debug.r_map;
314 prev->l_next != NULL;
323link_elf_delete_gdb(
struct link_map *l)
325 if (l->l_prev == NULL) {
327 if ((r_debug.r_map = l->l_next) != NULL)
328 l->l_next->l_prev = NULL;
331 if ((l->l_prev->l_next = l->l_next) != NULL)
332 l->l_next->l_prev = l->l_prev;
346 printf(
"kldload: %s\n", s);
357 if (
addr == NULL || size == 0)
359 cnt = size /
sizeof(*ctor);
361 for (i = 0; i < cnt; i++) {
381 error = elf_cpu_load_file(lf);
388 newfilename =
malloc(strlen(lf->filename) + 1, M_LINKER, M_WAITOK);
389 strcpy(newfilename, lf->filename);
390 ef->gdb.l_name = newfilename;
392 link_elf_add_gdb(&ef->gdb);
393 GDB_STATE(RT_CONSISTENT);
401#ifdef RELOCATABLE_KERNEL
417extern vm_offset_t __startkernel, __endkernel;
423 SYSCTL_NULL_ULONG_PTR, KERNBASE,
"Kernel base address");
431 Elf_Addr *ctors_addrp;
432 Elf_Size *ctors_sizep;
433 caddr_t modptr, baseptr, sizeptr;
449 panic(
"%s: Can't create linker structures for kernel",
454#ifdef RELOCATABLE_KERNEL
456 ef->
address = (caddr_t) (__startkernel - KERNBASE);
467#ifdef RELOCATABLE_KERNEL
476 if (modptr != NULL) {
485 MODINFO_METADATA | MODINFOMD_CTORS_ADDR);
487 MODINFO_METADATA | MODINFOMD_CTORS_SIZE);
488 if (ctors_addrp != NULL && ctors_sizep != NULL) {
497 r_debug.r_map = NULL;
498 r_debug.r_brk = r_debug_state;
499 r_debug.r_state = RT_CONSISTENT;
506 TAILQ_INIT(&set_vnet_list);
516 caddr_t ssym, esym, base;
525 MODINFO_METADATA | MODINFOMD_SSYM);
528 ssym = *(caddr_t *)pointer;
530 MODINFO_METADATA | MODINFOMD_ESYM);
533 esym = *(caddr_t *)pointer;
537 symcnt = *(
long *)base;
538 base +=
sizeof(long);
540 base += roundup(symcnt,
sizeof(
long));
542 if (base > esym || base < ssym) {
543 printf(
"Symbols are corrupt!\n");
547 strcnt = *(
long *)base;
548 base +=
sizeof(long);
550 base += roundup(strcnt,
sizeof(
long));
552 if (base > esym || base < ssym) {
553 printf(
"Symbols are corrupt!\n");
558 ef->
ddbsymcnt = symcnt /
sizeof(Elf_Sym);
569 int plttype = DT_REL;
571 for (dp = ef->
dynamic; dp->d_tag != DT_NULL; dp++) {
576 const Elf_Hashelt *hashtab = (
const Elf_Hashelt *)
577 (ef->
address + dp->d_un.d_ptr);
588 ef->
strsz = dp->d_un.d_val;
594 if (dp->d_un.d_val !=
sizeof(Elf_Sym))
598 ef->
got = (Elf_Addr *) (ef->
address + dp->d_un.d_ptr);
601 ef->
rel = (
const Elf_Rel *) (ef->
address + dp->d_un.d_ptr);
607 if (dp->d_un.d_val !=
sizeof(Elf_Rel))
611 ef->
pltrel = (
const Elf_Rel *) (ef->
address + dp->d_un.d_ptr);
617 ef->
rela = (
const Elf_Rela *) (ef->
address + dp->d_un.d_ptr);
623 if (dp->d_un.d_val !=
sizeof(Elf_Rela))
627 plttype = dp->d_un.d_val;
628 if (plttype != DT_REL && plttype != DT_RELA)
633 dp->d_un.d_ptr = (Elf_Addr)&r_debug;
639 if (plttype == DT_RELA) {
654#define LS_PADDING 0x90909090
677 uprintf(
"Kernel module '%s' must be recompiled with "
678 "linker script\n", ef->
lf.pathname);
683 pad = *(uint32_t *)((uintptr_t)ef->
pcpu_stop -
sizeof(pad));
685 uprintf(
"Kernel module '%s' must be recompiled with "
686 "linker script, invalid padding %#04x (%#04x)\n",
701 printf(
"%s: pcpu module space is out of space; "
702 "cannot allocate %d for %s\n",
703 __func__, size, ef->
lf.pathname);
726 (
void ***)&ef->vnet_stop, NULL);
730 size = (uintptr_t)ef->vnet_stop - (uintptr_t)ef->vnet_start;
737 uprintf(
"Kernel module '%s' must be recompiled with "
738 "linker script\n", ef->
lf.pathname);
743 pad = *(uint32_t *)((uintptr_t)ef->vnet_stop -
sizeof(pad));
745 uprintf(
"Kernel module '%s' must be recompiled with "
746 "linker script, invalid padding %#04x (%#04x)\n",
759 ef->vnet_base = (Elf_Addr)(uintptr_t)vnet_data_alloc(size);
760 if (ef->vnet_base == 0) {
761 printf(
"%s: vnet module space is out of space; "
762 "cannot allocate %d for %s\n",
763 __func__, size, ef->
lf.pathname);
766 memcpy((
void *)ef->vnet_base, (
void *)ef->vnet_start, size);
767 vnet_data_copy((
void *)ef->vnet_base, size);
768 elf_set_add(&set_vnet_list, ef->vnet_start, ef->vnet_stop,
783#if defined(__aarch64__) || defined(__amd64__)
785 Elf_Phdr *phdr, *phlimit;
791 phdr = (Elf_Phdr *)(ef->
address + hdr->e_phoff);
792 phlimit = phdr + hdr->e_phnum;
793 for (; phdr < phlimit; phdr++) {
794 if (phdr->p_type != PT_LOAD)
797 nprot = prot | VM_PROT_READ;
798 if ((phdr->p_flags & PF_W) != 0)
799 nprot |= VM_PROT_WRITE;
800 if ((phdr->p_flags & PF_X) != 0)
801 nprot |= VM_PROT_EXECUTE;
802 error = pmap_change_prot((vm_offset_t)ef->
address +
803 phdr->p_vaddr, round_page(phdr->p_memsz), nprot);
820link_elf_locate_exidx(linker_file_t lf, Elf_Shdr *shdr,
int nhdr)
824 for (i = 0; i < nhdr; i++) {
825 if (shdr[i].sh_type == SHT_ARM_EXIDX) {
826 lf->exidx_addr = shdr[i].sh_addr + lf->address;
827 lf->exidx_size = shdr[i].sh_size;
840link_elf_locate_exidx_preload(
struct linker_file *lf, caddr_t modptr)
847 MODINFO_METADATA | MODINFOMD_SHDR);
848 if (modinfo != NULL) {
849 shdr = (Elf_Shdr *)modinfo;
850 nhdr = modinfo[-1] /
sizeof(Elf_Shdr);
851 link_elf_locate_exidx(lf, shdr, nhdr);
861 Elf_Addr *ctors_addrp;
862 Elf_Size *ctors_sizep;
863 caddr_t modptr, baseptr, sizeptr, dynptr;
879 MODINFO_METADATA | MODINFOMD_DYNAMIC);
882 strcmp(
type,
"elf module") != 0))
884 if (baseptr == NULL || sizeptr == NULL || dynptr == NULL)
894 ef->
address = *(caddr_t *)baseptr;
898 dp = (vm_offset_t)ef->
address + *(vm_offset_t *)dynptr;
901 lf->size = *(
size_t *)sizeptr;
904 MODINFO_METADATA | MODINFOMD_CTORS_ADDR);
906 MODINFO_METADATA | MODINFOMD_CTORS_SIZE);
907 if (ctors_addrp != NULL && ctors_sizep != NULL) {
908 lf->ctors_addr = ef->
address + *ctors_addrp;
909 lf->ctors_size = *ctors_sizep;
913 link_elf_locate_exidx_preload(lf, modptr);
921 error = parse_vnet(ef);
956 struct thread* td = curthread;
958 caddr_t firstpage, segbase;
968 Elf_Addr base_vlimit;
986 NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE,
filename);
991 NDFREE(&nd, NDF_ONLY_PNBUF);
992 if (nd.ni_vp->v_type != VREG) {
998 error = mac_kld_check_load(curthread->td_ucred, nd.ni_vp);
1008 firstpage =
malloc(PAGE_SIZE, M_LINKER, M_WAITOK);
1009 hdr = (Elf_Ehdr *)firstpage;
1010 error =
vn_rdwr(UIO_READ, nd.ni_vp, firstpage, PAGE_SIZE, 0,
1011 UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED,
1013 nbytes = PAGE_SIZE - resid;
1017 if (!IS_ELF(*hdr)) {
1022 if (hdr->e_ident[EI_CLASS] != ELF_TARG_CLASS ||
1023 hdr->e_ident[EI_DATA] != ELF_TARG_DATA) {
1028 if (hdr->e_ident[EI_VERSION] != EV_CURRENT ||
1029 hdr->e_version != EV_CURRENT) {
1034 if (hdr->e_type != ET_EXEC && hdr->e_type != ET_DYN) {
1038 if (hdr->e_machine != ELF_TARG_MACH) {
1050 if (!((hdr->e_phentsize ==
sizeof(Elf_Phdr)) &&
1051 (hdr->e_phoff + hdr->e_phnum*
sizeof(Elf_Phdr) <= PAGE_SIZE) &&
1052 (hdr->e_phoff + hdr->e_phnum*
sizeof(Elf_Phdr) <=
nbytes)))
1061 phdr = (Elf_Phdr *) (firstpage + hdr->e_phoff);
1062 phlimit = phdr + hdr->e_phnum;
1065 while (phdr < phlimit) {
1066 switch (phdr->p_type) {
1091 if (phdyn == NULL) {
1107 base_vaddr = trunc_page(segs[0]->p_vaddr);
1108 base_vlimit = round_page(segs[nsegs - 1]->p_vaddr +
1109 segs[nsegs - 1]->p_memsz);
1110 mapsize = base_vlimit - base_vaddr;
1119#ifdef SPARSE_MAPPING
1120 ef->
object = vm_pager_allocate(OBJT_PHYS, NULL, mapsize, VM_PROT_ALL,
1121 0, thread0.td_ucred);
1122 if (ef->
object == NULL) {
1127 mapbase = (caddr_t)KERNBASE;
1129 mapbase = (caddr_t)vm_map_min(kernel_map);
1134 error = vm_map_find(kernel_map, ef->
object, 0,
1135 (vm_offset_t *)&mapbase, mapsize, 0, VMFS_OPTIMAL_SPACE,
1136 VM_PROT_ALL, VM_PROT_ALL, 0);
1138 vm_object_deallocate(ef->
object);
1143 mapbase =
malloc_exec(mapsize, M_LINKER, M_WAITOK);
1150 for (i = 0; i < nsegs; i++) {
1151 segbase = mapbase + segs[i]->p_vaddr - base_vaddr;
1153#ifdef SPARSE_MAPPING
1158 if (((vm_offset_t)segbase & PAGE_MASK) != 0) {
1163 error = vm_map_wire(kernel_map,
1164 (vm_offset_t)segbase,
1165 (vm_offset_t)segbase + round_page(segs[i]->p_memsz),
1166 VM_MAP_WIRE_SYSTEM | VM_MAP_WIRE_NOHOLES);
1167 if (error != KERN_SUCCESS) {
1173 error =
vn_rdwr(UIO_READ, nd.ni_vp,
1174 segbase, segs[i]->p_filesz, segs[i]->p_offset,
1175 UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED,
1179 bzero(segbase + segs[i]->p_filesz,
1180 segs[i]->p_memsz - segs[i]->p_filesz);
1186 kmupetext((uintfptr_t)(mapbase + segs[0]->p_vaddr - base_vaddr +
1191 ef->
dynamic = (Elf_Dyn *) (mapbase + phdyn->p_vaddr - base_vaddr);
1203 error = parse_vnet(ef);
1209 VOP_UNLOCK(nd.ni_vp);
1211 vn_lock(nd.ni_vp, LK_EXCLUSIVE | LK_RETRY);
1218#ifdef SPARSE_MAPPING
1223 for (i = 0; i < nsegs; i++) {
1226 if (segs[i]->p_type != PT_LOAD)
1229 prot = VM_PROT_READ;
1230 if ((segs[i]->p_flags & PF_W) != 0)
1231 prot |= VM_PROT_WRITE;
1232 if ((segs[i]->p_flags & PF_X) != 0)
1233 prot |= VM_PROT_EXECUTE;
1234 segbase = mapbase + segs[i]->p_vaddr - base_vaddr;
1235 error = vm_map_protect(kernel_map,
1236 (vm_offset_t)segbase,
1237 (vm_offset_t)segbase + round_page(segs[i]->p_memsz),
1238 prot, 0, VM_MAP_PROTECT_SET_PROT);
1239 if (error != KERN_SUCCESS) {
1250 nbytes = hdr->e_shnum * hdr->e_shentsize;
1251 if (
nbytes == 0 || hdr->e_shoff == 0)
1254 error =
vn_rdwr(UIO_READ, nd.ni_vp,
1255 (caddr_t)shdr,
nbytes, hdr->e_shoff,
1256 UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED,
1262 shstrindex = hdr->e_shstrndx;
1263 if (shstrindex != 0 && shdr[shstrindex].sh_type == SHT_STRTAB &&
1264 shdr[shstrindex].sh_size != 0) {
1265 nbytes = shdr[shstrindex].sh_size;
1267 error =
vn_rdwr(UIO_READ, nd.ni_vp, (caddr_t)shstrs,
nbytes,
1268 shdr[shstrindex].sh_offset, UIO_SYSSPACE, IO_NODELOCKED,
1269 td->td_ucred, NOCRED, &resid, td);
1276 for (i = 0; i < hdr->e_shnum; i++) {
1277 if (shdr[i].sh_type == SHT_SYMTAB) {
1279 symstrindex = shdr[i].sh_link;
1280 }
else if (shstrs != NULL && shdr[i].sh_name != 0 &&
1281 strcmp(shstrs + shdr[i].sh_name,
".ctors") == 0) {
1283 lf->ctors_addr = mapbase + shdr[i].sh_addr - base_vaddr;
1284 lf->ctors_size = shdr[i].sh_size;
1287 if (symtabindex < 0 || symstrindex < 0)
1290 symcnt = shdr[symtabindex].sh_size;
1292 strcnt = shdr[symstrindex].sh_size;
1295 error =
vn_rdwr(UIO_READ, nd.ni_vp,
1296 ef->
symbase, symcnt, shdr[symtabindex].sh_offset,
1297 UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED,
1301 error =
vn_rdwr(UIO_READ, nd.ni_vp,
1302 ef->
strbase, strcnt, shdr[symstrindex].sh_offset,
1303 UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED,
1308 ef->
ddbsymcnt = symcnt /
sizeof(Elf_Sym);
1316 link_elf_locate_exidx(lf, shdr, hdr->e_shnum);
1326 VOP_UNLOCK(nd.ni_vp);
1327 vn_close(nd.ni_vp, FREAD, td->td_ucred, td);
1328 if (error != 0 && lf != NULL)
1330 free(shdr, M_LINKER);
1331 free(firstpage, M_LINKER);
1332 free(shstrs, M_LINKER);
1343 (
"elf_relocaddr: unexpected linker file %p", lf));
1346 if (x >= ef->
pcpu_start && x < ef->pcpu_stop)
1349 if (x >= ef->vnet_start && x < ef->vnet_stop)
1350 return ((x - ef->vnet_start) + ef->vnet_base);
1366 if (ef->vnet_base != 0) {
1367 vnet_data_free((
void *)ef->vnet_base,
1368 ef->vnet_stop - ef->vnet_start);
1373 if (ef->gdb.l_ld != NULL) {
1374 GDB_STATE(RT_DELETE);
1375 free((
void *)(uintptr_t)ef->gdb.l_name, M_LINKER);
1376 link_elf_delete_gdb(&ef->gdb);
1377 GDB_STATE(RT_CONSISTENT);
1382 elf_cpu_unload_file(file);
1389#ifdef SPARSE_MAPPING
1390 if (ef->
object != NULL) {
1391 vm_map_remove(kernel_map, (vm_offset_t) ef->
address,
1393 + (ef->
object->size << PAGE_SHIFT));
1409 if (file->pathname != NULL)
1418 if (ELF_R_SYM(r_info)) {
1419 ref = ef->
symtab + ELF_R_SYM(r_info);
1420 return (ef->
strtab + ref->st_name);
1430 if (ELF_R_SYM(r_info)) {
1431 ref = ef->
symtab + ELF_R_SYM(r_info);
1432 return (ELF_ST_TYPE(ref->st_info));
1434 return (STT_NOTYPE);
1442 const Elf_Rela *rela;
1443 const char *symname;
1445#define APPLY_RELOCS(iter, tbl, tblsize, type) do { \
1446 for ((iter) = (tbl); (iter) != NULL && \
1447 (iter) < (tbl) + (tblsize) / sizeof(*(iter)); (iter)++) { \
1448 if ((symbol_type(ef, (iter)->r_info) == \
1450 elf_is_ifunc_reloc((iter)->r_info)) != ifuncs) \
1452 if (reloc(&ef->lf, (Elf_Addr)ef->address, \
1453 (iter), (type), lookup)) { \
1454 symname = symbol_name(ef, (iter)->r_info); \
1455 printf("link_elf: symbol %s undefined\n", \
1490 const unsigned char *p = (
const unsigned char *)
name;
1491 unsigned long h = 0;
1494 while (*p !=
'\0') {
1495 h = (h << 4) + *p++;
1496 if ((g = h & 0xf0000000) != 0)
1508 unsigned long symnum;
1509 const Elf_Sym*
symp;
1515 printf(
"link_elf_lookup_symbol: missing symbol hash table\n");
1523 while (symnum != STN_UNDEF) {
1525 printf(
"%s: corrupt symbol table\n", __func__);
1530 if (
symp->st_name == 0) {
1531 printf(
"%s: corrupt symbol table\n", __func__);
1537 if (strcmp(
name, strp) == 0) {
1538 if (
symp->st_shndx != SHN_UNDEF ||
1539 (
symp->st_value != 0 &&
1540 (ELF_ST_TYPE(
symp->st_info) == STT_FUNC ||
1541 ELF_ST_TYPE(
symp->st_info) == STT_GNU_IFUNC))) {
1543 ELF_ST_BIND(
symp->st_info) != STB_LOCAL) {
1551 symnum = ef->
chains[symnum];
1567 c_linker_sym_t *
sym)
1570 const Elf_Sym*
symp;
1579 if (strcmp(
name, strp) == 0) {
1580 if (
symp->st_shndx != SHN_UNDEF ||
1581 (
symp->st_value != 0 &&
1582 (ELF_ST_TYPE(
symp->st_info) == STT_FUNC ||
1583 ELF_ST_TYPE(
symp->st_info) == STT_GNU_IFUNC))) {
1596 linker_symval_t *symval,
bool see_local)
1603 es = (
const Elf_Sym *)
sym;
1605 if (!see_local && ELF_ST_BIND(es->st_info) == STB_LOCAL)
1607 symval->name = ef->
strtab + es->st_name;
1608 val = (caddr_t)ef->
address + es->st_value;
1609 if (ELF_ST_TYPE(es->st_info) == STT_GNU_IFUNC)
1610 val = ((caddr_t (*)(void))val)();
1611 symval->value = val;
1612 symval->size = es->st_size;
1620 linker_symval_t *symval)
1629 linker_symval_t *symval)
1632 const Elf_Sym *es = (
const Elf_Sym *)
sym;
1641 symval->name = ef->
ddbstrtab + es->st_name;
1642 val = (caddr_t)ef->
address + es->st_value;
1643 if (ELF_ST_TYPE(es->st_info) == STT_GNU_IFUNC)
1644 val = ((caddr_t (*)(void))val)();
1645 symval->value = val;
1646 symval->size = es->st_size;
1657 u_long off = (uintptr_t)(
void *)
value;
1661 const Elf_Sym *best = NULL;
1664 for (i = 0, es = ef->
ddbsymtab; i < ef->ddbsymcnt; i++, es++) {
1665 if (es->st_name == 0)
1667 st_value = es->st_value + (uintptr_t) (
void *) ef->
address;
1668 if (off >= st_value) {
1669 if (off - st_value < diff) {
1670 diff = off - st_value;
1674 }
else if (off - st_value == diff) {
1683 *
sym = (c_linker_sym_t) best;
1693 void ***startp,
void ***stopp,
int *countp)
1696 linker_symval_t symval;
1699 int len, error = 0,
count;
1701 len = strlen(
name) +
sizeof(
"__start_set_");
1702 setsym =
malloc(len, M_LINKER, M_WAITOK);
1710 if (symval.value == 0) {
1714 start = (
void **)symval.value;
1722 if (symval.value == 0) {
1726 stop = (
void **)symval.value;
1740 free(setsym, M_LINKER);
1749 const Elf_Sym *
symp;
1754 if (
symp->st_value != 0 &&
1755 (ELF_ST_TYPE(
symp->st_info) == STT_FUNC ||
1756 ELF_ST_TYPE(
symp->st_info) == STT_GNU_IFUNC)) {
1769 linker_symval_t symval;
1771 const Elf_Sym *
symp;
1776 if (
symp->st_value != 0 &&
1777 (ELF_ST_TYPE(
symp->st_info) == STT_FUNC ||
1778 ELF_ST_TYPE(
symp->st_info) == STT_GNU_IFUNC)) {
1780 (c_linker_sym_t)
symp, &symval);
1797 return (ef->
symtab + symidx);
1839 if (ELF_ST_BIND(
sym->st_info) == STB_LOCAL) {
1841 if (
sym->st_shndx == SHN_UNDEF ||
sym->st_value == 0) {
1865 if (
addr == 0 && ELF_ST_BIND(
sym->st_info) != STB_WEAK) {
1883 const Elf_Rel *rellim;
1885 const Elf_Rela *relalim;
1886 const Elf_Rela *rela;
1890 if ((rel = ef->
rel) != NULL) {
1891 rellim = (
const Elf_Rel *)((
const char *)ef->
rel + ef->
relsize);
1892 while (rel < rellim) {
1893 elf_reloc_local(lf, (Elf_Addr)ef->
address, rel,
1900 if ((rela = ef->
rela) != NULL) {
1901 relalim = (
const Elf_Rela *)
1903 while (rela < relalim) {
1904 elf_reloc_local(lf, (Elf_Addr)ef->
address, rela,
1937#if defined(__i386__) || defined(__amd64__) || defined(__aarch64__) || defined(__powerpc__)
1944elf_lookup_ifunc(linker_file_t lf, Elf_Size symidx,
int deps __unused,
1948 const Elf_Sym *
symp;
1953 if (ELF_ST_TYPE(
symp->st_info) == STT_GNU_IFUNC) {
1955 *
res = ((Elf_Addr (*)(void))val)();
1962link_elf_ireloc(caddr_t kmdp)
1969 bzero_early(ef,
sizeof(*ef));
1972 ef->dynamic = (Elf_Dyn *)&
_DYNAMIC;
1974#ifdef RELOCATABLE_KERNEL
1975 ef->address = (caddr_t) (__startkernel - KERNBASE);
1985#if defined(__aarch64__) || defined(__amd64__)
1987link_elf_late_ireloc(
void)
1992 (
"link_elf_late_ireloc: No kernel linker file found"));
device_property_type_t type
static int link_elf_ctf_get(linker_file_t lf, linker_ctf_t *lc)
int linker_ctf_get(linker_file_t file, linker_ctf_t *lc)
static int linker_debug_symbol_values(c_linker_sym_t sym, linker_symval_t *symval)
linker_file_t linker_kernel_file
int linker_file_unload(linker_file_t file, int flags)
caddr_t linker_file_lookup_symbol(linker_file_t file, const char *name, int deps)
static int linker_load_file(const char *filename, linker_file_t *result)
int linker_load_dependencies(linker_file_t lf)
linker_file_t linker_make_file(const char *pathname, linker_class_t lc)
int linker_add_class(linker_class_t lc)
void *() malloc(size_t size, struct malloc_type *mtp, int flags)
void * malloc_exec(size_t size, struct malloc_type *mtp, int flags)
void free(void *addr, struct malloc_type *mtp)
struct mtx __exclusive_cache_line Giant
void panic(const char *fmt,...)
static int link_elf_lookup_debug_symbol(linker_file_t, const char *, c_linker_sym_t *)
static int link_elf_preload_parse_symbols(elf_file_t)
static int link_elf_symbol_values1(linker_file_t lf, c_linker_sym_t sym, linker_symval_t *symval, bool see_local)
static void link_elf_unload_preload(linker_file_t)
static void link_elf_init(void *arg)
static int link_elf_link_preload_finish(linker_file_t)
static unsigned long elf_hash(const char *name)
static kobj_method_t link_elf_methods[]
const Elf_Sym * elf_get_sym(linker_file_t lf, Elf_Size symidx)
TAILQ_HEAD(elf_set_head, elf_set)
static bool link_elf_leak_locals
static int elf_lookup(linker_file_t, Elf_Size, int, Elf_Addr *)
static long link_elf_symtab_get(linker_file_t, const Elf_Sym **)
static int link_elf_debug_symbol_values(linker_file_t, c_linker_sym_t, linker_symval_t *)
static int relocate_file1(elf_file_t ef, elf_lookup_fn lookup, elf_reloc_fn reloc, bool ifuncs)
static int link_elf_symbol_values(linker_file_t, c_linker_sym_t, linker_symval_t *)
static void link_elf_reloc_local(linker_file_t)
static int link_elf_each_function_nameval(linker_file_t, linker_function_nameval_callback_t, void *)
static int link_elf_each_function_name(linker_file_t, int(*)(const char *, void *), void *)
SYSCTL_BOOL(_debug, OID_AUTO, link_elf_leak_locals, CTLFLAG_RWTUN, &link_elf_leak_locals, 0, "Allow local symbols to participate in global module symbol resolution")
static int parse_dpcpu(elf_file_t ef)
const char * elf_get_symname(linker_file_t lf, Elf_Size symidx)
static int link_elf_lookup_symbol(linker_file_t, const char *, c_linker_sym_t *)
int(* elf_reloc_fn)(linker_file_t lf, Elf_Addr relocbase, const void *data, int type, elf_lookup_fn lookup)
static void elf_set_delete(struct elf_set_head *list, Elf_Addr start)
static long link_elf_strtab_get(linker_file_t, caddr_t *)
static void link_elf_unload_file(linker_file_t)
static struct linker_class link_elf_class
static int link_elf_link_common_finish(linker_file_t)
static int preload_protect(elf_file_t ef, vm_prot_t prot)
static void link_elf_invoke_ctors(caddr_t addr, size_t size)
SYSINIT(link_elf, SI_SUB_KLD, SI_ORDER_THIRD, link_elf_init, NULL)
static int parse_dynamic(elf_file_t)
static void elf_set_add(struct elf_set_head *list, Elf_Addr start, Elf_Addr stop, Elf_Addr base)
#define APPLY_RELOCS(iter, tbl, tblsize, type)
static int symbol_type(elf_file_t ef, Elf_Size r_info)
static const char * symbol_name(elf_file_t ef, Elf_Size r_info)
static int link_elf_lookup_symbol1(linker_file_t lf, const char *name, c_linker_sym_t *sym, bool see_local)
Elf_Addr elf_relocaddr(linker_file_t lf, Elf_Addr x)
SYSCTL_ULONG(_kern, OID_AUTO, base_address, CTLFLAG_RD, SYSCTL_NULL_ULONG_PTR, KERNBASE, "Kernel base address")
static unsigned long kern_relbase
struct elf_file * elf_file_t
static int elf_set_find(struct elf_set_head *list, Elf_Addr addr, Elf_Addr *start, Elf_Addr *base)
static int link_elf_link_preload(linker_class_t cls, const char *, linker_file_t *)
static int link_elf_load_file(linker_class_t, const char *, linker_file_t *)
static int link_elf_search_symbol(linker_file_t, caddr_t, c_linker_sym_t *, long *)
static int relocate_file(elf_file_t)
static int link_elf_lookup_set(linker_file_t, const char *, void ***, void ***, int *)
static struct elf_set_head set_pcpu_list
static void link_elf_error(const char *filename, const char *s)
linker_function_name_callback_t callback
const Elf_Sym * ddbsymtab
const Elf_Hashelt * buckets
const Elf_Hashelt * chains
caddr_t preload_search_info(caddr_t mod, int inf)
caddr_t preload_search_by_name(const char *name)
void preload_delete_name(const char *name)
caddr_t preload_search_by_type(const char *type)
void dpcpu_copy(void *s, int size)
void * dpcpu_alloc(int size)
void dpcpu_free(void *s, int size)
int printf(const char *fmt,...)
int snprintf(char *str, size_t size, const char *format,...)
int uprintf(const char *fmt,...)
void() NDFREE(struct nameidata *ndp, const u_int flags)
int vn_rdwr(enum uio_rw rw, struct vnode *vp, void *base, int len, off_t offset, enum uio_seg segflg, int ioflg, struct ucred *active_cred, struct ucred *file_cred, ssize_t *aresid, struct thread *td)
int vn_open(struct nameidata *ndp, int *flagp, int cmode, struct file *fp)
int vn_close(struct vnode *vp, int flags, struct ucred *file_cred, struct thread *td)