41#include <sys/kernel.h>
51#include <ddb/db_sym.h>
76#define __RET_ADDR (uintptr_t)__builtin_return_address(0)
78#define KCSAN_NACCESSES 1024
84#include <machine/csan.h>
92 printf(
"Enabling KCSCAN, expect reduced performance.\n");
108 const char *newsym, *oldsym;
113 sym = db_search_symbol((vm_offset_t)new->pc, DB_STGY_PROC, &offset);
114 db_symbol_values(
sym, &newsym, NULL);
116 sym = db_search_symbol((vm_offset_t)old->
pc, DB_STGY_PROC, &offset);
117 db_symbol_values(
sym, &oldsym, NULL);
122 REPORT(
"CSan: Racy Access "
123 "[Cpu%u %s%s Addr=%p Size=%u PC=%p<%s>] "
124 "[Cpu%u %s%s Addr=%p Size=%u PC=%p<%s>]\n",
126 (new->atomic ?
"Atomic " :
""), (new->write ?
"Write" :
"Read"),
127 (
void *)new->addr, new->size, (
void *)new->pc, newsym,
129 (old->
atomic ?
"Atomic " :
""), (old->
write ?
"Write" :
"Read"),
130 (
void *)old->
addr, old->
size, (
void *)old->
pc, oldsym);
137 if (new->write && !new->atomic)
152 if (__predict_false(!kcsan_enabled))
154 if (__predict_false(kcsan_md_unsupported((vm_offset_t)
addr)))
156 if (KERNEL_PANICKED())
166 __builtin_memcpy(&old, &
kcsan_cpus[i].cell,
sizeof(old));
168 if (old.
addr + old.
size <=
new.addr)
170 if (
new.
addr +
new.size <= old.
addr)
172 if (__predict_true(!old.
write && !
new.write))
181 if (__predict_false(!kcsan_md_is_avail()))
184 kcsan_md_disable_intrs(&intr);
187 if (__predict_false(!cpu->
inited))
190 if (__predict_true(cpu->
cnt != 0))
193 __builtin_memcpy(&cpu->
cell, &
new,
sizeof(
new));
195 __builtin_memset(&cpu->
cell, 0,
sizeof(
new));
198 kcsan_md_enable_intrs(&intr);
201#define CSAN_READ(size) \
202 void __tsan_read##size(uintptr_t); \
203 void __tsan_read##size(uintptr_t addr) \
205 kcsan_access(addr, size, false, false, __RET_ADDR); \
207 void __tsan_unaligned_read##size(uintptr_t); \
208 void __tsan_unaligned_read##size(uintptr_t addr) \
210 kcsan_access(addr, size, false, false, __RET_ADDR); \
219#define CSAN_WRITE(size) \
220 void __tsan_write##size(uintptr_t); \
221 void __tsan_write##size(uintptr_t addr) \
223 kcsan_access(addr, size, true, false, __RET_ADDR); \
225 void __tsan_unaligned_write##size(uintptr_t); \
226 void __tsan_unaligned_write##size(uintptr_t addr) \
228 kcsan_access(addr, size, true, false, __RET_ADDR); \
278 return __builtin_memcpy(dst,
src, len);
286 return __builtin_memcmp(b1, b2, len);
293 return __builtin_memset(b, c, len);
301 return __builtin_memmove(dst,
src, len);
334 return (*(
const unsigned char *)s1 - *(
const unsigned char *)s2);
357 return copyin(uaddr, kaddr, len);
364 return copyinstr(uaddr, kaddr, len, done);
371 return copyout(kaddr, uaddr, len);
376#include <machine/atomic.h>
377#include <sys/atomic_san.h>
379#define _CSAN_ATOMIC_FUNC_ADD(name, type) \
380 void kcsan_atomic_add_##name(volatile type *ptr, type val) \
382 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
384 atomic_add_##name(ptr, val); \
387#define CSAN_ATOMIC_FUNC_ADD(name, type) \
388 _CSAN_ATOMIC_FUNC_ADD(name, type) \
389 _CSAN_ATOMIC_FUNC_ADD(acq_##name, type) \
390 _CSAN_ATOMIC_FUNC_ADD(rel_##name, type)
392#define _CSAN_ATOMIC_FUNC_CLEAR(name, type) \
393 void kcsan_atomic_clear_##name(volatile type *ptr, type val) \
395 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
397 atomic_clear_##name(ptr, val); \
400#define CSAN_ATOMIC_FUNC_CLEAR(name, type) \
401 _CSAN_ATOMIC_FUNC_CLEAR(name, type) \
402 _CSAN_ATOMIC_FUNC_CLEAR(acq_##name, type) \
403 _CSAN_ATOMIC_FUNC_CLEAR(rel_##name, type)
405#define _CSAN_ATOMIC_FUNC_CMPSET(name, type) \
406 int kcsan_atomic_cmpset_##name(volatile type *ptr, type val1, \
409 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
411 return (atomic_cmpset_##name(ptr, val1, val2)); \
414#define CSAN_ATOMIC_FUNC_CMPSET(name, type) \
415 _CSAN_ATOMIC_FUNC_CMPSET(name, type) \
416 _CSAN_ATOMIC_FUNC_CMPSET(acq_##name, type) \
417 _CSAN_ATOMIC_FUNC_CMPSET(rel_##name, type)
419#define _CSAN_ATOMIC_FUNC_FCMPSET(name, type) \
420 int kcsan_atomic_fcmpset_##name(volatile type *ptr, type *val1, \
423 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
425 return (atomic_fcmpset_##name(ptr, val1, val2)); \
428#define CSAN_ATOMIC_FUNC_FCMPSET(name, type) \
429 _CSAN_ATOMIC_FUNC_FCMPSET(name, type) \
430 _CSAN_ATOMIC_FUNC_FCMPSET(acq_##name, type) \
431 _CSAN_ATOMIC_FUNC_FCMPSET(rel_##name, type)
433#define CSAN_ATOMIC_FUNC_FETCHADD(name, type) \
434 type kcsan_atomic_fetchadd_##name(volatile type *ptr, type val) \
436 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
438 return (atomic_fetchadd_##name(ptr, val)); \
441#define _CSAN_ATOMIC_FUNC_LOAD(name, type) \
442 type kcsan_atomic_load_##name(volatile type *ptr) \
444 kcsan_access((uintptr_t)ptr, sizeof(type), false, true, \
446 return (atomic_load_##name(ptr)); \
449#define CSAN_ATOMIC_FUNC_LOAD(name, type) \
450 _CSAN_ATOMIC_FUNC_LOAD(name, type) \
451 _CSAN_ATOMIC_FUNC_LOAD(acq_##name, type) \
453#define CSAN_ATOMIC_FUNC_READANDCLEAR(name, type) \
454 type kcsan_atomic_readandclear_##name(volatile type *ptr) \
456 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
458 return (atomic_readandclear_##name(ptr)); \
461#define _CSAN_ATOMIC_FUNC_SET(name, type) \
462 void kcsan_atomic_set_##name(volatile type *ptr, type val) \
464 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
466 atomic_set_##name(ptr, val); \
469#define CSAN_ATOMIC_FUNC_SET(name, type) \
470 _CSAN_ATOMIC_FUNC_SET(name, type) \
471 _CSAN_ATOMIC_FUNC_SET(acq_##name, type) \
472 _CSAN_ATOMIC_FUNC_SET(rel_##name, type)
474#define _CSAN_ATOMIC_FUNC_SUBTRACT(name, type) \
475 void kcsan_atomic_subtract_##name(volatile type *ptr, type val) \
477 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
479 atomic_subtract_##name(ptr, val); \
482#define CSAN_ATOMIC_FUNC_SUBTRACT(name, type) \
483 _CSAN_ATOMIC_FUNC_SUBTRACT(name, type) \
484 _CSAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type) \
485 _CSAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type)
487#define _CSAN_ATOMIC_FUNC_STORE(name, type) \
488 void kcsan_atomic_store_##name(volatile type *ptr, type val) \
490 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
492 atomic_store_##name(ptr, val); \
495#define CSAN_ATOMIC_FUNC_STORE(name, type) \
496 _CSAN_ATOMIC_FUNC_STORE(name, type) \
497 _CSAN_ATOMIC_FUNC_STORE(rel_##name, type)
499#define CSAN_ATOMIC_FUNC_SWAP(name, type) \
500 type kcsan_atomic_swap_##name(volatile type *ptr, type val) \
502 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
504 return(atomic_swap_##name(ptr, val)); \
507#define CSAN_ATOMIC_FUNC_TESTANDCLEAR(name, type) \
508 int kcsan_atomic_testandclear_##name(volatile type *ptr, u_int val) \
510 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
512 return(atomic_testandclear_##name(ptr, val)); \
515#define CSAN_ATOMIC_FUNC_TESTANDSET(name, type) \
516 int kcsan_atomic_testandset_##name(volatile type *ptr, u_int val) \
518 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
520 return (atomic_testandset_##name(ptr, val)); \
648#if !defined(__amd64__)
662#define CSAN_ATOMIC_FUNC_THREAD_FENCE(name) \
663 void kcsan_atomic_thread_fence_##name(void) \
665 atomic_thread_fence_##name(); \
676 atomic_interrupt_fence();
682#include <machine/bus.h>
683#include <sys/bus_san.h>
687 int flags, bus_space_handle_t *handlep)
690 return (bus_space_map(tag, hnd, size,
flags, handlep));
698 bus_space_unmap(tag, hnd, size);
703 bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep)
706 return (bus_space_subregion(tag, hnd, offset, size, handlep));
709#if !defined(__amd64__)
712 bus_addr_t reg_end, bus_size_t size, bus_size_t alignment,
713 bus_size_t boundary,
int flags, bus_addr_t *addrp,
714 bus_space_handle_t *handlep)
717 return (bus_space_alloc(tag, reg_start, reg_end, size, alignment,
718 boundary,
flags, addrp, handlep));
727 bus_space_free(tag, hnd, size);
732 bus_size_t offset, bus_size_t size,
int flags)
735 bus_space_barrier(tag, hnd, offset, size,
flags);
738#define CSAN_BUS_READ_FUNC(func, width, type) \
739 type kcsan_bus_space_read##func##_##width(bus_space_tag_t tag, \
740 bus_space_handle_t hnd, bus_size_t offset) \
742 return (bus_space_read##func##_##width(tag, hnd, \
746#define CSAN_BUS_READ_PTR_FUNC(func, width, type) \
747 void kcsan_bus_space_read_##func##_##width(bus_space_tag_t tag, \
748 bus_space_handle_t hnd, bus_size_t size, type *buf, \
751 kcsan_access((uintptr_t)buf, sizeof(type) * count, \
752 false, false, __RET_ADDR); \
753 bus_space_read_##func##_##width(tag, hnd, size, buf, \
779#if defined(__aarch64__)
787#define CSAN_BUS_WRITE_FUNC(func, width, type) \
788 void kcsan_bus_space_write##func##_##width(bus_space_tag_t tag, \
789 bus_space_handle_t hnd, bus_size_t offset, type value) \
791 bus_space_write##func##_##width(tag, hnd, offset, value); \
794#define CSAN_BUS_WRITE_PTR_FUNC(func, width, type) \
795 void kcsan_bus_space_write_##func##_##width(bus_space_tag_t tag, \
796 bus_space_handle_t hnd, bus_size_t size, const type *buf, \
799 kcsan_access((uintptr_t)buf, sizeof(type) * count, \
800 true, false, __RET_ADDR); \
801 bus_space_write_##func##_##width(tag, hnd, size, buf, \
827#if defined(__aarch64__)
835#define CSAN_BUS_SET_FUNC(func, width, type) \
836 void kcsan_bus_space_set_##func##_##width(bus_space_tag_t tag, \
837 bus_space_handle_t hnd, bus_size_t offset, type value, \
840 bus_space_set_##func##_##width(tag, hnd, offset, value, \
846#if !defined(__aarch64__)
853#if !defined(__aarch64__)
860#if !defined(__aarch64__)
865#if !defined(__amd64__)
868#if !defined(__aarch64__)
874#define CSAN_BUS_PEEK_FUNC(width, type) \
875 int kcsan_bus_space_peek_##width(bus_space_tag_t tag, \
876 bus_space_handle_t hnd, bus_size_t offset, type *value) \
878 kcsan_access((uintptr_t)value, sizeof(type), true, false, \
880 return (bus_space_peek_##width(tag, hnd, offset, value)); \
886#if !defined(__i386__)
890#define CSAN_BUS_POKE_FUNC(width, type) \
891 int kcsan_bus_space_poke_##width(bus_space_tag_t tag, \
892 bus_space_handle_t hnd, bus_size_t offset, type value) \
894 return (bus_space_poke_##width(tag, hnd, offset, value)); \
900#if !defined(__i386__)
struct intr_irqsrc ** src
int kcsan_copyout(const void *kaddr, void *uaddr, size_t len)
void * kcsan_memset(void *b, int c, size_t len)
#define CSAN_ATOMIC_FUNC_CMPSET(name, type)
int kcsan_bus_space_alloc(bus_space_tag_t tag, bus_addr_t reg_start, bus_addr_t reg_end, bus_size_t size, bus_size_t alignment, bus_size_t boundary, int flags, bus_addr_t *addrp, bus_space_handle_t *handlep)
#define CSAN_BUS_SET_FUNC(func, width, type)
char * kcsan_strcpy(char *dst, const char *src)
static bool kcsan_access_is_atomic(csan_cell_t *new, csan_cell_t *old)
#define CSAN_ATOMIC_FUNC_SUBTRACT(name, type)
int kcsan_copyin(const void *uaddr, void *kaddr, size_t len)
static void kcsan_report(csan_cell_t *new, u_int newcpu, csan_cell_t *old, u_int oldcpu)
#define CSAN_ATOMIC_FUNC_SET(name, type)
static void kcsan_enable(void *dummy __unused)
void kcsan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd, bus_size_t size)
#define CSAN_BUS_WRITE_FUNC(func, width, type)
#define CSAN_ATOMIC_FUNC_TESTANDSET(name, type)
void * kcsan_memmove(void *dst, const void *src, size_t len)
#define CSAN_BUS_PEEK_FUNC(width, type)
static bool kcsan_enabled __read_mostly
#define _CSAN_ATOMIC_FUNC_STORE(name, type)
#define CSAN_ATOMIC_FUNC_CLEAR(name, type)
int kcsan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd, bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep)
#define CSAN_ATOMIC_FUNC_TESTANDCLEAR(name, type)
void kcsan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd, bus_size_t offset, bus_size_t size, int flags)
void __tsan_func_exit(void)
static csan_cpu_t kcsan_cpus[MAXCPU]
void kcsan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd, bus_size_t size)
void __tsan_func_entry(void *)
static void kcsan_access(uintptr_t addr, size_t size, bool write, bool atomic, uintptr_t pc)
#define CSAN_BUS_READ_FUNC(func, width, type)
#define CSAN_ATOMIC_FUNC_SWAP(name, type)
size_t kcsan_strlen(const char *str)
#define CSAN_ATOMIC_FUNC_FCMPSET(name, type)
int kcsan_strcmp(const char *s1, const char *s2)
void __tsan_write_range(uintptr_t, size_t)
#define CSAN_ATOMIC_FUNC_STORE(name, type)
#define CSAN_BUS_WRITE_PTR_FUNC(func, width, type)
#define CSAN_ATOMIC_FUNC_ADD(name, type)
#define CSAN_ATOMIC_FUNC_LOAD(name, type)
void kcsan_cpu_init(u_int cpu)
void __tsan_read_range(uintptr_t, size_t)
int kcsan_memcmp(const void *b1, const void *b2, size_t len)
int kcsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done)
void * kcsan_memcpy(void *dst, const void *src, size_t len)
SYSINIT(kcsan_enable, SI_SUB_SMP, SI_ORDER_SECOND, kcsan_enable, NULL)
#define CSAN_ATOMIC_FUNC_READANDCLEAR(name, type)
#define CSAN_ATOMIC_FUNC_FETCHADD(name, type)
int kcsan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size, int flags, bus_space_handle_t *handlep)
#define CSAN_BUS_READ_PTR_FUNC(func, width, type)
#define CSAN_BUS_POKE_FUNC(width, type)
#define CSAN_ATOMIC_FUNC_THREAD_FENCE(name)
void kcsan_atomic_interrupt_fence(void)
int printf(const char *fmt,...)