36__KERNEL_RCSID(0,
"$NetBSD: subr_asan.c,v 1.26 2020/09/10 14:10:46 maxv Exp $");
42#include <sys/kernel.h>
44#include <sys/sysctl.h>
46#include <machine/asan.h>
49#define KASAN_SHADOW_MASK (KASAN_SHADOW_SCALE - 1)
50#define KASAN_ALLOCA_SCALE_SIZE 32
53#if defined(__clang__) && (__clang_major__ - 0 >= 6)
54#define ASAN_ABI_VERSION 8
55#elif __GNUC_PREREQ__(7, 1) && !defined(__clang__)
56#define ASAN_ABI_VERSION 8
57#elif __GNUC_PREREQ__(6, 1) && !defined(__clang__)
58#define ASAN_ABI_VERSION 6
60#error "Unsupported compiler version"
63#define __RET_ADDR (unsigned long)__builtin_return_address(0)
80#if ASAN_ABI_VERSION >= 7
81 uintptr_t odr_indicator;
85FEATURE(kasan,
"Kernel address sanitizer");
87static SYSCTL_NODE(_debug, OID_AUTO, kasan, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
93 "Panic if an invalid access is detected");
102 size_t sz, npages, i;
103 vm_offset_t sva, eva;
105 KASSERT(
addr % KASAN_SHADOW_SCALE == 0,
106 (
"%s: invalid address %#lx", __func__,
addr));
108 sz = roundup(size, KASAN_SHADOW_SCALE) / KASAN_SHADOW_SCALE;
110 sva = kasan_md_addr_to_shad(
addr);
111 eva = kasan_md_addr_to_shad(
addr) + sz;
113 sva = rounddown(sva, PAGE_SIZE);
114 eva = roundup(eva, PAGE_SIZE);
116 npages = (eva - sva) / PAGE_SIZE;
118 KASSERT(sva >= KASAN_MIN_ADDRESS && eva < KASAN_MAX_ADDRESS,
119 (
"%s: invalid address range %#lx-%#lx", __func__, sva, eva));
121 for (i = 0; i < npages; i++)
122 pmap_san_enter(sva + ptoa(i));
131 TUNABLE_INT_FETCH(
"debug.kasan.disabled", &disabled);
139 kasan_enabled =
true;
142static inline const char *
146 case KASAN_GENERIC_REDZONE:
147 return "GenericRedZone";
148 case KASAN_MALLOC_REDZONE:
149 return "MallocRedZone";
150 case KASAN_KMEM_REDZONE:
151 return "KmemRedZone";
152 case KASAN_UMA_FREED:
153 return "UMAUseAfterFree";
154 case KASAN_KSTACK_FREED:
155 return "KernelStack";
156 case KASAN_EXEC_ARGS_FREED:
159 return "RedZonePartial";
160 case KASAN_STACK_LEFT:
162 case KASAN_STACK_MID:
163 return "StackMiddle";
164 case KASAN_STACK_RIGHT:
166 case KASAN_USE_AFTER_RET:
167 return "UseAfterRet";
168 case KASAN_USE_AFTER_SCOPE:
169 return "UseAfterScope";
175#define REPORT(f, ...) do { \
176 if (panic_on_violation) { \
177 kasan_enabled = false; \
178 panic(f, __VA_ARGS__); \
183 printf(f "\n", __VA_ARGS__); \
184 stack_print_ddb(&st); \
192 REPORT(
"ASan: Invalid access, %zu-byte %s at %#lx, %s(%x)",
197static __always_inline
void
200 int8_t *
byte = (int8_t *)kasan_md_addr_to_shad(
addr);
206static __always_inline
void
211 for (i = 0; i < size; i++) {
216static __always_inline
void
221 if (__predict_false(size == 0))
223 if (__predict_false(kasan_md_unsupported((vm_offset_t)
addr)))
226 KASSERT((vm_offset_t)
addr % KASAN_SHADOW_SCALE == 0,
227 (
"%s: invalid address %p", __func__,
addr));
228 KASSERT(size % KASAN_SHADOW_SCALE == 0,
229 (
"%s: invalid size %zu", __func__, size));
231 shad = (
void *)kasan_md_addr_to_shad((uintptr_t)
addr);
232 size = size >> KASAN_SHADOW_SCALE_SHIFT;
234 __builtin_memset(shad, code, size);
253 if ((vm_offset_t)
addr >= DMAP_MIN_ADDRESS &&
254 (vm_offset_t)
addr < DMAP_MAX_ADDRESS)
257 KASSERT((vm_offset_t)
addr >= VM_MIN_KERNEL_ADDRESS &&
258 (vm_offset_t)
addr < VM_MAX_KERNEL_ADDRESS,
259 (
"%s: invalid address %p", __func__,
addr));
260 KASSERT((vm_offset_t)
addr % KASAN_SHADOW_SCALE == 0,
261 (
"%s: invalid address %p", __func__,
addr));
262 redz = redzsize - roundup(size, KASAN_SHADOW_SCALE);
263 KASSERT(redz % KASAN_SHADOW_SCALE == 0,
264 (
"%s: invalid size %zu", __func__, redz));
265 shad = (int8_t *)kasan_md_addr_to_shad((uintptr_t)
addr);
268 n = size / KASAN_SHADOW_SCALE;
269 for (i = 0; i < n; i++) {
279 n = redz / KASAN_SHADOW_SCALE;
280 for (i = 0; i < n; i++) {
287#define ADDR_CROSSES_SCALE_BOUNDARY(addr, size) \
288 (addr >> KASAN_SHADOW_SCALE_SHIFT) != \
289 ((addr + size - 1) >> KASAN_SHADOW_SCALE_SHIFT)
291static __always_inline
bool
294 int8_t *
byte = (int8_t *)kasan_md_addr_to_shad(
addr);
297 if (__predict_true(*
byte == 0 || last <= *
byte)) {
304static __always_inline
bool
314 byte = (int8_t *)kasan_md_addr_to_shad(
addr);
317 if (__predict_true(*
byte == 0 || last <= *
byte)) {
324static __always_inline
bool
334 byte = (int8_t *)kasan_md_addr_to_shad(
addr);
337 if (__predict_true(*
byte == 0 || last <= *
byte)) {
344static __always_inline
bool
354 byte = (int8_t *)kasan_md_addr_to_shad(
addr);
357 if (__predict_true(*
byte == 0 || last <= *
byte)) {
364static __always_inline
bool
369 for (i = 0; i < size; i++) {
377static __always_inline
void
379 unsigned long retaddr)
384 if (__predict_false(!kasan_enabled))
386 if (__predict_false(size == 0))
388 if (__predict_false(kasan_md_unsupported(
addr)))
390 if (__predict_false(
panicstr != NULL))
393 if (__builtin_constant_p(size)) {
415 if (__predict_false(!valid)) {
427 return (__builtin_memcpy(dst,
src, len));
435 return (__builtin_memcmp(b1, b2, len));
442 return (__builtin_memset(b, c, len));
450 return (__builtin_memmove(dst,
src, len));
499 return (*(
const unsigned char *)s1 - *(
const unsigned char *)s2);
506 return (copyin(uaddr, kaddr, len));
513 return (copyinstr(uaddr, kaddr, len, done));
520 return (copyout(kaddr, uaddr, len));
528 return (fubyte(base));
534 return (fuword16(base));
541 return (fueword(base, val));
548 return (fueword32(base, val));
555 return (fueword64(base, val));
561 return (subyte(base,
byte));
567 return (suword(base, word));
573 return (suword16(base, word));
579 return (suword32(base, word));
585 return (suword64(base, word));
594 return (casueword32(base, oldval, oldvalp, newval));
603 return (casueword(base, oldval, oldvalp, newval));
608#include <machine/atomic.h>
609#include <sys/atomic_san.h>
611#define _ASAN_ATOMIC_FUNC_ADD(name, type) \
612 void kasan_atomic_add_##name(volatile type *ptr, type val) \
614 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
616 atomic_add_##name(ptr, val); \
619#define ASAN_ATOMIC_FUNC_ADD(name, type) \
620 _ASAN_ATOMIC_FUNC_ADD(name, type) \
621 _ASAN_ATOMIC_FUNC_ADD(acq_##name, type) \
622 _ASAN_ATOMIC_FUNC_ADD(rel_##name, type)
624#define _ASAN_ATOMIC_FUNC_SUBTRACT(name, type) \
625 void kasan_atomic_subtract_##name(volatile type *ptr, type val) \
627 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
629 atomic_subtract_##name(ptr, val); \
632#define ASAN_ATOMIC_FUNC_SUBTRACT(name, type) \
633 _ASAN_ATOMIC_FUNC_SUBTRACT(name, type) \
634 _ASAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type) \
635 _ASAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type)
637#define _ASAN_ATOMIC_FUNC_SET(name, type) \
638 void kasan_atomic_set_##name(volatile type *ptr, type val) \
640 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
642 atomic_set_##name(ptr, val); \
645#define ASAN_ATOMIC_FUNC_SET(name, type) \
646 _ASAN_ATOMIC_FUNC_SET(name, type) \
647 _ASAN_ATOMIC_FUNC_SET(acq_##name, type) \
648 _ASAN_ATOMIC_FUNC_SET(rel_##name, type)
650#define _ASAN_ATOMIC_FUNC_CLEAR(name, type) \
651 void kasan_atomic_clear_##name(volatile type *ptr, type val) \
653 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
655 atomic_clear_##name(ptr, val); \
658#define ASAN_ATOMIC_FUNC_CLEAR(name, type) \
659 _ASAN_ATOMIC_FUNC_CLEAR(name, type) \
660 _ASAN_ATOMIC_FUNC_CLEAR(acq_##name, type) \
661 _ASAN_ATOMIC_FUNC_CLEAR(rel_##name, type)
663#define ASAN_ATOMIC_FUNC_FETCHADD(name, type) \
664 type kasan_atomic_fetchadd_##name(volatile type *ptr, type val) \
666 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
668 return (atomic_fetchadd_##name(ptr, val)); \
671#define ASAN_ATOMIC_FUNC_READANDCLEAR(name, type) \
672 type kasan_atomic_readandclear_##name(volatile type *ptr) \
674 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
676 return (atomic_readandclear_##name(ptr)); \
679#define ASAN_ATOMIC_FUNC_TESTANDCLEAR(name, type) \
680 int kasan_atomic_testandclear_##name(volatile type *ptr, u_int v) \
682 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
684 return (atomic_testandclear_##name(ptr, v)); \
687#define ASAN_ATOMIC_FUNC_TESTANDSET(name, type) \
688 int kasan_atomic_testandset_##name(volatile type *ptr, u_int v) \
690 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
692 return (atomic_testandset_##name(ptr, v)); \
695#define ASAN_ATOMIC_FUNC_SWAP(name, type) \
696 type kasan_atomic_swap_##name(volatile type *ptr, type val) \
698 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
700 return (atomic_swap_##name(ptr, val)); \
703#define _ASAN_ATOMIC_FUNC_CMPSET(name, type) \
704 int kasan_atomic_cmpset_##name(volatile type *ptr, type oval, \
707 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
709 return (atomic_cmpset_##name(ptr, oval, nval)); \
712#define ASAN_ATOMIC_FUNC_CMPSET(name, type) \
713 _ASAN_ATOMIC_FUNC_CMPSET(name, type) \
714 _ASAN_ATOMIC_FUNC_CMPSET(acq_##name, type) \
715 _ASAN_ATOMIC_FUNC_CMPSET(rel_##name, type)
717#define _ASAN_ATOMIC_FUNC_FCMPSET(name, type) \
718 int kasan_atomic_fcmpset_##name(volatile type *ptr, type *oval, \
721 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
723 return (atomic_fcmpset_##name(ptr, oval, nval)); \
726#define ASAN_ATOMIC_FUNC_FCMPSET(name, type) \
727 _ASAN_ATOMIC_FUNC_FCMPSET(name, type) \
728 _ASAN_ATOMIC_FUNC_FCMPSET(acq_##name, type) \
729 _ASAN_ATOMIC_FUNC_FCMPSET(rel_##name, type)
731#define ASAN_ATOMIC_FUNC_THREAD_FENCE(name) \
732 void kasan_atomic_thread_fence_##name(void) \
734 atomic_thread_fence_##name(); \
737#define _ASAN_ATOMIC_FUNC_LOAD(name, type) \
738 type kasan_atomic_load_##name(volatile type *ptr) \
740 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
742 return (atomic_load_##name(ptr)); \
745#define ASAN_ATOMIC_FUNC_LOAD(name, type) \
746 _ASAN_ATOMIC_FUNC_LOAD(name, type) \
747 _ASAN_ATOMIC_FUNC_LOAD(acq_##name, type)
749#define _ASAN_ATOMIC_FUNC_STORE(name, type) \
750 void kasan_atomic_store_##name(volatile type *ptr, type val) \
752 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
754 atomic_store_##name(ptr, val); \
757#define ASAN_ATOMIC_FUNC_STORE(name, type) \
758 _ASAN_ATOMIC_FUNC_STORE(name, type) \
759 _ASAN_ATOMIC_FUNC_STORE(rel_##name, type)
869#include <machine/bus.h>
870#include <sys/bus_san.h>
874 int flags, bus_space_handle_t *handlep)
876 return (bus_space_map(tag, hnd, size,
flags, handlep));
883 bus_space_unmap(tag, hnd, size);
888 bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep)
890 return (bus_space_subregion(tag, hnd, offset, size, handlep));
897 bus_space_free(tag, hnd, size);
902 bus_size_t offset, bus_size_t size,
int flags)
904 bus_space_barrier(tag, hnd, offset, size,
flags);
907#define ASAN_BUS_READ_FUNC(func, width, type) \
908 type kasan_bus_space_read##func##_##width(bus_space_tag_t tag, \
909 bus_space_handle_t hnd, bus_size_t offset) \
911 return (bus_space_read##func##_##width(tag, hnd, \
915#define ASAN_BUS_READ_PTR_FUNC(func, width, type) \
916 void kasan_bus_space_read_##func##_##width(bus_space_tag_t tag, \
917 bus_space_handle_t hnd, bus_size_t size, type *buf, \
920 kasan_shadow_check((uintptr_t)buf, sizeof(type) * count,\
921 false, __RET_ADDR); \
922 bus_space_read_##func##_##width(tag, hnd, size, buf, \
949#define ASAN_BUS_WRITE_FUNC(func, width, type) \
950 void kasan_bus_space_write##func##_##width(bus_space_tag_t tag, \
951 bus_space_handle_t hnd, bus_size_t offset, type value) \
953 bus_space_write##func##_##width(tag, hnd, offset, value);\
956#define ASAN_BUS_WRITE_PTR_FUNC(func, width, type) \
957 void kasan_bus_space_write_##func##_##width(bus_space_tag_t tag,\
958 bus_space_handle_t hnd, bus_size_t size, const type *buf, \
961 kasan_shadow_check((uintptr_t)buf, sizeof(type) * count,\
963 bus_space_write_##func##_##width(tag, hnd, size, buf, \
990#define ASAN_BUS_SET_FUNC(func, width, type) \
991 void kasan_bus_space_set_##func##_##width(bus_space_tag_t tag, \
992 bus_space_handle_t hnd, bus_size_t offset, type value, \
995 bus_space_set_##func##_##width(tag, hnd, offset, value, \
1024 for (i = 0; i < n; i++) {
1026 globals[i].size_with_redzone, KASAN_GENERIC_REDZONE);
1035 for (i = 0; i < n; i++) {
1036 kasan_mark(globals[i].beg, globals[i].size_with_redzone,
1037 globals[i].size_with_redzone, 0);
1041#define ASAN_LOAD_STORE(size) \
1042 void __asan_load##size(unsigned long); \
1043 void __asan_load##size(unsigned long addr) \
1045 kasan_shadow_check(addr, size, false, __RET_ADDR);\
1047 void __asan_load##size##_noabort(unsigned long); \
1048 void __asan_load##size##_noabort(unsigned long addr) \
1050 kasan_shadow_check(addr, size, false, __RET_ADDR);\
1052 void __asan_store##size(unsigned long); \
1053 void __asan_store##size(unsigned long addr) \
1055 kasan_shadow_check(addr, size, true, __RET_ADDR);\
1057 void __asan_store##size##_noabort(unsigned long); \
1058 void __asan_store##size##_noabort(unsigned long addr) \
1060 kasan_shadow_check(addr, size, true, __RET_ADDR);\
1105#define ASAN_SET_SHADOW(byte) \
1106 void __asan_set_shadow_##byte(void *, size_t); \
1107 void __asan_set_shadow_##byte(void *addr, size_t size) \
1109 __builtin_memset((void *)addr, 0x##byte, size); \
1125 size = roundup(size, KASAN_SHADOW_SCALE);
1144 (
"%s: invalid address %p", __func__,
addr));
1160 if (__predict_false(!stkbegin))
1162 if (__predict_false((uintptr_t)stkbegin > (uintptr_t)stkend))
1164 size = (uintptr_t)stkend - (uintptr_t)stkbegin;
struct intr_irqsrc ** src
struct __asan_global_source_location * location
unsigned long has_dynamic_init
void * kasan_memcpy(void *dst, const void *src, size_t len)
static __always_inline void kasan_shadow_1byte_markvalid(unsigned long addr)
static const char * kasan_code_name(uint8_t code)
void kasan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd, bus_size_t size)
void kasan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd, bus_size_t offset, bus_size_t size, int flags)
static __always_inline bool kasan_shadow_4byte_isvalid(unsigned long addr, uint8_t *code)
#define ASAN_BUS_SET_FUNC(func, width, type)
int kasan_fueword64(volatile const void *base, int64_t *val)
static int panic_on_violation
#define ASAN_ATOMIC_FUNC_TESTANDSET(name, type)
char * kasan_strcpy(char *dst, const char *src)
void __asan_loadN(unsigned long, size_t)
int kasan_suword(volatile void *base, long word)
void __asan_poison_stack_memory(const void *, size_t)
int kasan_suword32(volatile void *base, int32_t word)
#define ASAN_ATOMIC_FUNC_FCMPSET(name, type)
void __asan_poison_memory_region(const void *addr, size_t size)
#define ASAN_LOAD_STORE(size)
void __asan_storeN_noabort(unsigned long, size_t)
static __always_inline bool kasan_shadow_8byte_isvalid(unsigned long addr, uint8_t *code)
#define ASAN_BUS_READ_PTR_FUNC(func, width, type)
#define ASAN_BUS_READ_FUNC(func, width, type)
#define KASAN_ALLOCA_SCALE_SIZE
#define ASAN_ATOMIC_FUNC_TESTANDCLEAR(name, type)
int kasan_strcmp(const char *s1, const char *s2)
static __always_inline bool kasan_shadow_2byte_isvalid(unsigned long addr, uint8_t *code)
int kasan_casueword32(volatile uint32_t *base, uint32_t oldval, uint32_t *oldvalp, uint32_t newval)
static __always_inline bool kasan_shadow_Nbyte_isvalid(unsigned long addr, size_t size, uint8_t *code)
#define ASAN_ATOMIC_FUNC_ADD(name, type)
#define ASAN_ATOMIC_FUNC_THREAD_FENCE(name)
int kasan_memcmp(const void *b1, const void *b2, size_t len)
void kasan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd, bus_size_t size)
static __always_inline void kasan_shadow_Nbyte_markvalid(const void *addr, size_t size)
FEATURE(kasan, "Kernel address sanitizer")
int kasan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done)
void __asan_alloca_poison(const void *, size_t)
#define ADDR_CROSSES_SCALE_BOUNDARY(addr, size)
static void kasan_report(unsigned long addr, size_t size, bool write, unsigned long pc, uint8_t code)
#define ASAN_ATOMIC_FUNC_SUBTRACT(name, type)
#define ASAN_ATOMIC_FUNC_STORE(name, type)
static bool kasan_enabled __read_mostly
int kasan_fueword32(volatile const void *base, int32_t *val)
int kasan_copyin(const void *uaddr, void *kaddr, size_t len)
#define ASAN_ATOMIC_FUNC_READANDCLEAR(name, type)
void __asan_allocas_unpoison(const void *, const void *)
int kasan_subyte(volatile void *base, int byte)
int kasan_suword16(volatile void *base, int word)
void * kasan_memset(void *b, int c, size_t len)
void __asan_loadN_noabort(unsigned long, size_t)
void __asan_storeN(unsigned long, size_t)
void kasan_mark(const void *addr, size_t size, size_t redzsize, uint8_t code)
int kasan_fueword(volatile const void *base, long *val)
static __always_inline bool kasan_shadow_1byte_isvalid(unsigned long addr, uint8_t *code)
static SYSCTL_NODE(_debug, OID_AUTO, kasan, CTLFLAG_RD|CTLFLAG_MPSAFE, 0, "KASAN options")
int kasan_casueword(volatile u_long *base, u_long oldval, u_long *oldvalp, u_long newval)
int kasan_fubyte(volatile const void *base)
void __asan_unpoison_stack_memory(const void *, size_t)
#define ASAN_ATOMIC_FUNC_CLEAR(name, type)
int kasan_fuword16(volatile const void *base)
SYSCTL_INT(_debug_kasan, OID_AUTO, panic_on_violation, CTLFLAG_RDTUN, &panic_on_violation, 0, "Panic if an invalid access is detected")
void __asan_register_globals(struct __asan_global *, size_t)
void __asan_unpoison_memory_region(const void *addr, size_t size)
int kasan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd, bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep)
int kasan_copyout(const void *kaddr, void *uaddr, size_t len)
#define ASAN_ATOMIC_FUNC_CMPSET(name, type)
static __always_inline void kasan_shadow_check(unsigned long addr, size_t size, bool write, unsigned long retaddr)
void __asan_handle_no_return(void)
void kasan_shadow_map(vm_offset_t addr, size_t size)
void __asan_unregister_globals(struct __asan_global *, size_t)
#define ASAN_ATOMIC_FUNC_LOAD(name, type)
static __always_inline void kasan_shadow_Nbyte_fill(const void *addr, size_t size, uint8_t code)
int kasan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size, int flags, bus_space_handle_t *handlep)
#define ASAN_SET_SHADOW(byte)
#define ASAN_BUS_WRITE_FUNC(func, width, type)
#define ASAN_BUS_WRITE_PTR_FUNC(func, width, type)
#define ASAN_ATOMIC_FUNC_FETCHADD(name, type)
#define KASAN_SHADOW_MASK
void kasan_atomic_interrupt_fence(void)
#define ASAN_ATOMIC_FUNC_SET(name, type)
size_t kasan_strlen(const char *str)
#define ASAN_ATOMIC_FUNC_SWAP(name, type)
void * kasan_memmove(void *dst, const void *src, size_t len)
int kasan_suword64(volatile void *base, int64_t word)