39#if defined(KASAN) || defined(KCSAN)
49#include <sys/eventhandler.h>
51#include <sys/kernel.h>
52#include <sys/limits.h>
54#include <sys/malloc.h>
58#include <sys/rwlock.h>
59#include <sys/sysctl.h>
63#include <vm/vm_extern.h>
64#include <vm/vm_object.h>
65#include <vm/vm_page.h>
66#include <vm/vm_pager.h>
67#include <vm/vm_param.h>
71#define KCOV_ELEMENT_SIZE sizeof(uint64_t)
146static void kcov_init(
const void *unused);
149 .d_version = D_VERSION,
163 "Maximum number of entries in the kcov buffer");
168static struct kcov_info * __nosanitizeaddress __nosanitizememory
181 if (td->td_intr_nesting_level > 0 || td->td_intr_frame != NULL)
187 info = td->td_kcov_info;
195static void __nosanitizeaddress __nosanitizememory
200 uint64_t *
buf, index;
210 if (info->
mode != KCOV_MODE_TRACE_PC)
213 KASSERT(info->
kvaddr != 0, (
"%s: NULL buf while running", __func__));
222 buf[index + 1] = ret;
226static bool __nosanitizeaddress __nosanitizememory
231 uint64_t *
buf, index;
241 if (info->
mode != KCOV_MODE_TRACE_CMP)
244 KASSERT(info->
kvaddr != 0, (
"%s: NULL buf while running", __func__));
252 if (index * 4 + 4 + 1 > info->
entries)
257 buf[index * 4 + 2] = arg1;
258 buf[index * 4 + 3] = arg2;
259 buf[index * 4 + 4] = ret;
276 struct thread *thread;
288 atomic_thread_fence_seq_cst();
306kcov_open(
struct cdev *dev,
int oflags,
int devtype,
struct thread *td)
311 info =
malloc(
sizeof(
struct kcov_info), M_KCOV_INFO, M_ZERO | M_WAITOK);
328 if ((error = devfs_get_cdevpriv((
void **)&info)) != 0)
331 KASSERT(info != NULL, (
"kcov_close with no kcov_info structure"));
342 struct vm_object **
object,
int nprot)
347 if ((nprot & (PROT_EXEC | PROT_READ | PROT_WRITE)) !=
348 (PROT_READ | PROT_WRITE))
351 if ((error = devfs_get_cdevpriv((
void **)&info)) != 0)
357 vm_object_reference(info->
bufobj);
369 KASSERT(info->
kvaddr == 0, (
"kcov_alloc: Already have a buffer"));
371 (
"kcov_alloc: Not in open state (%x)", info->
state));
378 pages = info->
bufsize / PAGE_SIZE;
383 info->
bufobj = vm_pager_allocate(OBJT_PHYS, 0, info->
bufsize,
384 PROT_READ | PROT_WRITE, 0, curthread->td_ucred);
386 VM_OBJECT_WLOCK(info->
bufobj);
387 for (n = 0; n < pages; n++) {
388 m = vm_page_grab(info->
bufobj, n,
389 VM_ALLOC_ZERO | VM_ALLOC_WIRED);
392 pmap_qenter(info->
kvaddr + n * PAGE_SIZE, &m, 1);
394 VM_OBJECT_WUNLOCK(info->
bufobj);
411 if (info->
bufobj != NULL) {
412 VM_OBJECT_WLOCK(info->
bufobj);
413 m = vm_page_lookup(info->
bufobj, 0);
414 for (i = 0; i < info->
bufsize / PAGE_SIZE; i++) {
415 vm_page_unwire_noq(m);
418 VM_OBJECT_WUNLOCK(info->
bufobj);
419 vm_object_deallocate(info->
bufobj);
421 free(info, M_KCOV_INFO);
431 if ((error = devfs_get_cdevpriv((
void **)&info)) != 0)
434 if (cmd == KIOSETBUFSIZE) {
455 if (td->td_kcov_info != NULL) {
460 if (
mode != KCOV_MODE_TRACE_PC &&
mode != KCOV_MODE_TRACE_CMP) {
467 (
"%s: Open too many times", __func__));
474 KASSERT(info->
thread == NULL,
475 (
"Enabling kcov when already enabled"));
483 td->td_kcov_info = info;
488 info != td->td_kcov_info) {
492 KASSERT(
active_count > 0, (
"%s: Open count is zero", __func__));
499 td->td_kcov_info = NULL;
505 atomic_thread_fence_rel();
523 info = td->td_kcov_info;
528 KASSERT(
active_count > 0, (
"%s: Open count is zero", __func__));
534 td->td_kcov_info = NULL;
541 atomic_thread_fence_seq_cst();
563 struct make_dev_args args;
566 mtx_init(&
kcov_lock,
"kcov lock", NULL, MTX_SPIN);
568 make_dev_args_init(&args);
570 args.mda_uid = UID_ROOT;
571 args.mda_gid = GID_WHEEL;
572 args.mda_mode = 0600;
574 printf(
"%s",
"Failed to create kcov device");
579 EVENTHANDLER_PRI_ANY);
device_property_type_t type
int make_dev_s(struct make_dev_args *args, struct cdev **dres, const char *fmt,...)
static d_open_t kcov_open
static d_ioctl_t kcov_ioctl
static u_int kcov_max_entries
static struct kcov_info *__nosanitizeaddress __nosanitizememory get_kinfo(struct thread *td)
SYSINIT(kcovdev, SI_SUB_LAST, SI_ORDER_ANY, kcov_init, NULL)
static void __nosanitizeaddress __nosanitizememory trace_pc(uintptr_t ret)
static d_mmap_single_t kcov_mmap_single
SYSCTL_NODE(_kern, OID_AUTO, kcov, CTLFLAG_RW|CTLFLAG_MPSAFE, 0, "Kernel coverage")
static struct mtx kcov_lock
static void kcov_mmap_cleanup(void *arg)
static bool __nosanitizeaddress __nosanitizememory trace_cmp(uint64_t type, uint64_t arg1, uint64_t arg2, uint64_t ret)
SYSCTL_UINT(_kern_kcov, OID_AUTO, max_entries, CTLFLAG_RW, &kcov_max_entries, 0, "Maximum number of entries in the kcov buffer")
MALLOC_DEFINE(M_KCOV_INFO, "kcovinfo", "KCOV info type")
static void kcov_free(struct kcov_info *info)
static void kcov_init(const void *unused)
#define KCOV_ELEMENT_SIZE
static int kcov_alloc(struct kcov_info *info, size_t entries)
static void kcov_thread_dtor(void *arg __unused, struct thread *td)
static d_close_t kcov_close
static struct cdevsw kcov_cdevsw
void *() malloc(size_t size, struct malloc_type *mtp, int flags)
void free(void *addr, struct malloc_type *mtp)
static void thread_dtor(void *mem, int size, void *arg)
int atomic_cmpset_64(volatile uint64_t *p, uint64_t old, uint64_t new)
void cov_register_pc(cov_trace_pc_t trace_pc)
void cov_unregister_cmp(void)
void cov_unregister_pc(void)
void cov_register_cmp(cov_trace_cmp_t trace_cmp)
int printf(const char *fmt,...)