39#include <sys/malloc.h>
40#include <sys/kernel.h>
43#include <sys/rwlock.h>
44#include <sys/selinfo.h>
48#include <sys/ioccom.h>
51#include <sys/module.h>
53#include <sys/bitset.h>
56#include <sys/syslog.h>
57#include <sys/taskqueue.h>
60#include <vm/vm_param.h>
61#include <vm/vm_extern.h>
62#include <vm/vm_kern.h>
63#include <vm/vm_page.h>
65#include <vm/vm_object.h>
66#include <vm/vm_pager.h>
68#include <machine/md_var.h>
70#include <xen/xen-os.h>
71#include <xen/hypervisor.h>
73#include <xen/xen_intr.h>
74#include <xen/gnttab.h>
75#include <xen/gntdev.h>
79#define MAX_OFFSET_COUNT ((0xffffffffffffffffull >> PAGE_SHIFT) + 1)
86 .d_version = D_VERSION,
104 uint64_t file_offset;
111 return (f1->file_offset - f2->file_offset);
130 uint64_t *file_offset)
137 RB_FOREACH_SAFE(offset, file_offset_head, &priv_user->
file_offset,
139 if (offset->count >= count) {
140 offset->count -= count;
141 *file_offset = offset->file_offset + offset->count *
143 if (offset->count == 0) {
144 RB_REMOVE(file_offset_head,
146 free(offset, M_GNTDEV);
159 uint64_t file_offset)
163 offset = malloc(
sizeof(*offset), M_GNTDEV, M_WAITOK | M_ZERO);
164 offset->file_offset = file_offset;
165 offset->count = count;
168 RB_INSERT(file_offset_head, &priv_user->
file_offset, offset);
169 offset_nxt = RB_NEXT(file_offset_head, &priv_user->
file_offset, offset);
170 offset_prv = RB_PREV(file_offset_head, &priv_user->
file_offset, offset);
171 if (offset_nxt != NULL &&
172 offset_nxt->file_offset == offset->file_offset + offset->count *
174 offset->count += offset_nxt->count;
175 RB_REMOVE(file_offset_head, &priv_user->
file_offset,
177 free(offset_nxt, M_GNTDEV);
179 if (offset_prv != NULL &&
180 offset->file_offset == offset_prv->file_offset + offset_prv->count *
182 offset_prv->count += offset->count;
183 RB_REMOVE(file_offset_head, &priv_user->
file_offset, offset);
184 free(offset, M_GNTDEV);
190 vm_prot_t prot, vm_ooffset_t foff,
struct ucred *cred, u_short *color);
193 int prot, vm_page_t *mres);
213 "gntdev to_kill_grefs mutex", MTX_DEF);
215 "gntdev to_kill_gmaps mutex", MTX_DEF);
259 struct gref_list_head tmp_grefs;
260 struct gntdev_gref *gref, *gref_tmp, *gref_previous;
262 STAILQ_INIT(&tmp_grefs);
267 gref_previous = NULL;
268 STAILQ_FOREACH_SAFE(gref, &tmp_grefs,
gref_next.list, gref_tmp) {
269 if (gref->
page && gref->
page->object == NULL) {
273 if (gref->
gref_id != GRANT_REF_INVALID) {
281 vm_page_unwire_noq(gref->
page);
282 vm_page_free(gref->
page);
285 if (gref->
page == NULL) {
286 if (gref_previous == NULL)
287 STAILQ_REMOVE_HEAD(&tmp_grefs,
gref_next.list);
289 STAILQ_REMOVE_AFTER(&tmp_grefs, gref_previous,
292 free(gref->
notify, M_GNTDEV);
293 free(gref, M_GNTDEV);
296 gref_previous = gref;
299 if (!STAILQ_EMPTY(&tmp_grefs)) {
312 uint64_t index, uint32_t count)
314 struct gntdev_gref find_gref, *gref, *gref_start = NULL;
319 gref_start = RB_FIND(gref_tree_head, &priv_user->
gref_tree, &find_gref);
320 for (gref = gref_start; gref != NULL && count > 0; gref =
321 RB_NEXT(gref_tree_head, &priv_user->
gref_tree, gref)) {
346 uint64_t file_offset;
350 readonly = !(arg->flags & GNTDEV_ALLOC_FLAG_WRITABLE);
352 error = devfs_get_cdevpriv((
void**) &priv_user);
365 grefs = malloc(
sizeof(*grefs) * arg->count, M_GNTDEV, M_WAITOK);
367 for (i = 0; i < arg->count; i++) {
369 grefs[i].
gref_id = GRANT_REF_INVALID;
371 grefs[i].
page = vm_page_alloc_noobj(VM_ALLOC_WIRED |
373 if (grefs[i].page == NULL) {
374 log(LOG_ERR,
"Page allocation failed.");
378 grefs[i].
page->valid = VM_PAGE_BITS_ALL;
381 (VM_PAGE_TO_PHYS(grefs[i].page) >> PAGE_SHIFT),
382 readonly, &grefs[i].gref_id);
384 log(LOG_ERR,
"Grant Table Hypercall failed.");
397 for (i = 0; i < arg->count; i++)
399 &grefs[i], gref_next.list);
409 for (i = 0; i < arg->count; i++)
410 suword32(&arg->gref_ids[i], grefs[i].
gref_id);
414 for (i = 0; i < arg->count; i++)
415 RB_INSERT(gref_tree_head, &priv_user->
gref_tree, &grefs[i]);
435 error = devfs_get_cdevpriv((
void**) &priv_user);
441 log(LOG_ERR,
"Can't find requested grant-refs.");
449 for (; gref != NULL && count > 0; gref = gref_tmp) {
450 gref_tmp = RB_NEXT(gref_tree_head, &priv_user->
gref_tree, gref);
451 RB_REMOVE(gref_tree_head, &priv_user->
gref_tree, gref);
502 struct gmap_list_head tmp_gmaps;
503 struct gntdev_gmap *gmap, *gmap_tmp, *gmap_previous;
505 STAILQ_INIT(&tmp_gmaps);
510 gmap_previous = NULL;
511 STAILQ_FOREACH_SAFE(gmap, &tmp_gmaps,
gmap_next.list, gmap_tmp) {
512 if (gmap->
map == NULL) {
513 if (gmap_previous == NULL)
514 STAILQ_REMOVE_HEAD(&tmp_gmaps,
gmap_next.list);
516 STAILQ_REMOVE_AFTER(&tmp_gmaps, gmap_previous,
520 free(gmap->
notify, M_GNTDEV);
522 free(gmap, M_GNTDEV);
525 gmap_previous = gmap;
528 if (!STAILQ_EMPTY(&tmp_gmaps)) {
541 uint64_t index, uint32_t
count)
548 gmap = RB_FIND(gmap_tree_head, &priv_user->
gmap_tree, &find_gmap);
568 struct gnttab_unmap_grant_ref *unmap_ops;
570 unmap_ops = malloc(
sizeof(
struct gnttab_unmap_grant_ref) * gmap->
count,
575 for (i = 0; i < gmap->
count; i++) {
578 unmap_ops[count].host_addr =
580 unmap_ops[count].dev_bus_addr = 0;
586 if (count > 0 && gmap->
notify) {
588 uint64_t page_offset;
596 VM_OBJECT_WLOCK(gmap->
map->
mem);
598 for (i = 0; i < gmap->
count; i++) {
599 m = vm_page_lookup(gmap->
map->
mem, i);
602 if (vm_page_busy_acquire(m, VM_ALLOC_WAITFAIL) == 0)
604 cdev_pager_free_page(gmap->
map->
mem, m);
606 VM_OBJECT_WUNLOCK(gmap->
map->
mem);
609 error = HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref,
612 for (i = 0; i < gmap->
count; i++) {
621 (
"Unable to release memory resource: %d", error));
623 free(gmap->
map, M_GNTDEV);
627 free(unmap_ops, M_GNTDEV);
646 error = devfs_get_cdevpriv((
void**) &priv_user);
650 gmap = malloc(
sizeof(*gmap), M_GNTDEV, M_WAITOK | M_ZERO);
651 gmap->
count = arg->count;
653 malloc(
sizeof(
struct gnttab_map_grant_ref) * arg->count,
654 M_GNTDEV, M_WAITOK | M_ZERO);
656 for (i = 0; i < arg->count; i++) {
657 struct ioctl_gntdev_grant_ref ref;
659 error = copyin(&arg->refs[i], &ref,
sizeof(ref));
662 free(gmap, M_GNTDEV);
674 free(gmap, M_GNTDEV);
679 RB_INSERT(gmap_tree_head, &priv_user->
gmap_tree, gmap);
701 error = devfs_get_cdevpriv((
void**) &priv_user);
707 log(LOG_ERR,
"Can't find requested grant-map.");
713 RB_REMOVE(gmap_tree_head, &priv_user->
gmap_tree, gmap);
719 vm_object_deallocate(gmap->
map->
mem);
743 vm_map_entry_t entry;
751 map = &td->td_proc->p_vmspace->vm_map;
752 error = vm_map_lookup(&
map, arg->vaddr, VM_PROT_NONE, &entry,
753 &mem, &pindex, &prot, &wired);
754 if (error != KERN_SUCCESS)
757 if ((mem->type != OBJT_MGTDEVICE) ||
765 (entry->end - entry->start) != (gmap->
count * PAGE_SIZE)) {
775 vm_map_lookup_done(
map, entry);
783 vm_ooffset_t foff,
struct ucred *cred, u_short *color)
801 vm_pindex_t pidx, ridx;
803 vm_ooffset_t relative_offset;
805 if (gmap->
map == NULL)
806 return (VM_PAGER_FAIL);
810 pidx = OFF_TO_IDX(offset);
811 ridx = OFF_TO_IDX(relative_offset);
812 if (ridx >= gmap->
count ||
814 return (VM_PAGER_FAIL);
818 return (VM_PAGER_FAIL);
820 KASSERT((page->flags & PG_FICTITIOUS) != 0,
821 (
"not fictitious %p", page));
822 KASSERT(vm_page_wired(page), (
"page %p is not wired", page));
823 KASSERT(!vm_page_busied(page), (
"page %p is busy", page));
825 vm_page_busy_acquire(page, 0);
828 vm_page_replace(page,
object, pidx, *mres);
830 vm_page_insert(page,
object, pidx);
832 return (VM_PAGER_OK);
840 if (
notify->action & UNMAP_NOTIFY_CLEAR_BYTE) {
844 offset =
notify->index & PAGE_MASK;
845 mem = (uint8_t *)pmap_quick_enter_page(page);
847 pmap_quick_remove_page((vm_offset_t)mem);
849 if (
notify->action & UNMAP_NOTIFY_SEND_EVENT) {
850 xen_intr_signal(
notify->notify_evtchn_handle);
851 xen_intr_unbind(&
notify->notify_evtchn_handle);
862 struct ioctl_gntdev_unmap_notify *source)
864 xen_intr_handle_t handlep = NULL;
870 if (source->action & UNMAP_NOTIFY_SEND_EVENT)
871 if (xen_intr_get_evtchn_from_port(source->event_channel_port,
875 if (destination->
action & UNMAP_NOTIFY_SEND_EVENT)
878 destination->
action = source->action;
880 destination->
index = source->index;
901 error = devfs_get_cdevpriv((
void**) &priv_user);
905 if (arg->action & ~(UNMAP_NOTIFY_CLEAR_BYTE|UNMAP_NOTIFY_SEND_EVENT))
908 index = arg->index & ~PAGE_MASK;
912 gref->
notify = malloc(
sizeof(*arg), M_GNTDEV,
919 RB_FOREACH(gmap, gmap_tree_head, &priv_user->
gmap_tree) {
923 gmap->
notify = malloc(
sizeof(*arg), M_GNTDEV,
957 RB_FOREACH_SAFE(gref, gref_tree_head, &priv_user->
gref_tree, gref_tmp) {
958 RB_REMOVE(gref_tree_head, &priv_user->
gref_tree, gref);
965 RB_FOREACH_SAFE(gmap, gmap_tree_head, &priv_user->
gmap_tree, gmap_tmp) {
966 RB_REMOVE(gmap_tree_head, &priv_user->
gmap_tree, gmap);
970 vm_object_deallocate(gmap->
map->
mem);
974 RB_FOREACH_SAFE(offset, file_offset_head, &priv_user->
file_offset,
976 RB_REMOVE(file_offset_head, &priv_user->
file_offset, offset);
977 free(offset, M_GNTDEV);
985 free(priv_user, M_GNTDEV);
989gntdev_open(
struct cdev *dev,
int flag,
int otyp,
struct thread *td)
995 priv_user = malloc(
sizeof(*priv_user), M_GNTDEV, M_WAITOK | M_ZERO);
999 offset = malloc(
sizeof(*offset), M_GNTDEV, M_WAITOK | M_ZERO);
1000 offset->file_offset = 0;
1002 RB_INSERT(file_offset_head, &priv_user->
file_offset, offset);
1004 "per user data mutex", NULL, MTX_DEF);
1015 int fflag,
struct thread *td)
1020 case IOCTL_GNTDEV_SET_UNMAP_NOTIFY:
1022 (
struct ioctl_gntdev_unmap_notify*) data);
1024 case IOCTL_GNTDEV_ALLOC_GREF:
1026 (
struct ioctl_gntdev_alloc_gref*) data);
1028 case IOCTL_GNTDEV_DEALLOC_GREF:
1030 (
struct ioctl_gntdev_dealloc_gref*) data);
1032 case IOCTL_GNTDEV_MAP_GRANT_REF:
1034 (
struct ioctl_gntdev_map_grant_ref*) data);
1036 case IOCTL_GNTDEV_UNMAP_GRANT_REF:
1038 (
struct ioctl_gntdev_unmap_grant_ref*) data);
1040 case IOCTL_GNTDEV_GET_OFFSET_FOR_VADDR:
1042 (
struct ioctl_gntdev_get_offset_for_vaddr*) data, td);
1059 uint32_t count, vm_size_t size,
struct vm_object **
object)
1061 vm_object_t mem_obj;
1064 mem_obj = vm_pager_allocate(OBJT_PHYS, NULL, size, VM_PROT_ALL, 0,
1065 curthread->td_ucred);
1066 if (mem_obj == NULL)
1070 VM_OBJECT_WLOCK(mem_obj);
1071 for (gref = gref_start; gref != NULL && count > 0; gref =
1072 RB_NEXT(gref_tree_head, &priv_user->
gref_tree, gref)) {
1073 if (gref->
page->object)
1076 vm_page_insert(gref->
page, mem_obj,
1081 VM_OBJECT_WUNLOCK(mem_obj);
1085 vm_object_deallocate(mem_obj);
1100 vm_ooffset_t *offset, vm_size_t size,
struct vm_object **
object,
int nprot)
1110 if (gmap_start->
map) {
1111 vm_object_reference(gmap_start->
map->
mem);
1112 *
object = gmap_start->
map->
mem;
1116 gmap_start->
map = malloc(
sizeof(*(gmap_start->
map)), M_GNTDEV,
1124 free(gmap_start->
map, M_GNTDEV);
1125 gmap_start->
map = NULL;
1132 gmap_start->
map->
mem = cdev_pager_allocate(gmap_start, OBJT_MGTDEVICE,
1134 if (gmap_start->
map->
mem == NULL) {
1137 free(gmap_start->
map, M_GNTDEV);
1138 gmap_start->
map = NULL;
1142 for (i = 0; i < gmap_start->
count; i++) {
1146 if ((nprot & PROT_WRITE) == 0)
1150 error = HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref,
1158 vm_object_deallocate(gmap_start->
map->
mem);
1164 for (i = 0; i < gmap_start->
count; i++) {
1166 while (delay < 256 &&
1168 HYPERVISOR_grant_table_op( GNTTABOP_map_grant_ref,
1170 pause((
"gntmap"), delay * SBT_1MS);
1183 vm_object_deallocate(gmap_start->
map->
mem);
1196 vm_object_reference(gmap_start->
map->
mem);
1198 *
object = gmap_start->
map->
mem;
1205 struct vm_object **
object,
int nprot)
1213 error = devfs_get_cdevpriv((
void**) &priv_user);
1217 count = OFF_TO_IDX(size);
1221 error =
mmap_gref(priv_user, gref_start, count, size,
object);
1227 error =
mmap_gmap(priv_user, gmap_start, offset, size,
object,
1240 KASSERT((xen_domain()),
1241 (
"Trying to attach gntdev device on non Xen domain"));
1243 if (BUS_ADD_CHILD(parent, 0,
"gntdev", 0) == NULL)
1244 panic(
"unable to attach gntdev user-space device");
1252 device_set_desc(dev,
"Xen grant-table user-space device");
1253 return (BUS_PROBE_NOWILDCARD);
1260 make_dev_credf(MAKEDEV_ETERNAL, &
gntdev_devsw, 0, NULL, UID_ROOT,
1261 GID_WHEEL, 0600,
"xen/gntdev");
STAILQ_HEAD(gref_list_head, gntdev_gref)
RB_GENERATE_STATIC(file_offset_head, file_offset_struct, next, offset_cmp)
static void gmap_list_dtor(struct cleanup_data_struct *cleanup_data)
static void put_file_offset(struct per_user_data *priv_user, uint32_t count, uint64_t file_offset)
static int mmap_gmap(struct per_user_data *priv_user, struct gntdev_gmap *gmap_start, vm_ooffset_t *offset, vm_size_t size, struct vm_object **object, int nprot)
MALLOC_DEFINE(M_GNTDEV, "gntdev", "Xen grant-table user-space device")
static int mmap_gref(struct per_user_data *priv_user, struct gntdev_gref *gref_start, uint32_t count, vm_size_t size, struct vm_object **object)
static void gntdev_gmap_pg_dtor(void *handle)
DRIVER_MODULE(gntdev, xenpv, gntdev_driver, gntdev_devclass, 0, 0)
static struct gntdev_gref * gntdev_find_grefs(struct per_user_data *priv_user, uint64_t index, uint32_t count)
static int gntdev_get_offset_for_vaddr(struct ioctl_gntdev_get_offset_for_vaddr *arg, struct thread *td)
static struct task cleanup_task
static int notify_unmap_cleanup(struct gntdev_gmap *gmap)
RB_HEAD(gref_tree_head, gntdev_gref)
devclass_t gntdev_devclass
static int offset_cmp(struct file_offset_struct *f1, struct file_offset_struct *f2)
static int gref_cmp(struct gntdev_gref *g1, struct gntdev_gref *g2)
static int gntdev_gmap_pg_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, vm_ooffset_t foff, struct ucred *cred, u_short *color)
static driver_t gntdev_driver
MODULE_DEPEND(gntdev, xenpv, 1, 1, 1)
static device_method_t gntdev_methods[]
static int gntdev_set_unmap_notify(struct ioctl_gntdev_unmap_notify *arg)
static device_t gntdev_dev
static int gntdev_dealloc_gref(struct ioctl_gntdev_dealloc_gref *arg)
static void notify(struct notify_data *notify, vm_page_t page)
static int gntdev_gmap_pg_fault(vm_object_t object, vm_ooffset_t offset, int prot, vm_page_t *mres)
static void cleanup_function(void *arg, __unused int pending)
static struct cleanup_data_struct cleanup_data
static d_mmap_single_t gntdev_mmap_single
static struct cdev_pager_ops gntdev_gmap_pg_ops
static void gref_list_dtor(struct cleanup_data_struct *cleanup_data)
static int gntdev_attach(device_t dev)
static struct gntdev_gmap * gntdev_find_gmap(struct per_user_data *priv_user, uint64_t index, uint32_t count)
static int gntdev_alloc_gref(struct ioctl_gntdev_alloc_gref *arg)
static int get_file_offset(struct per_user_data *priv_user, uint32_t count, uint64_t *file_offset)
static int copy_notify_helper(struct notify_data *destination, struct ioctl_gntdev_unmap_notify *source)
static d_open_t gntdev_open
static void gntdev_identify(driver_t *driver, device_t parent)
static int gntdev_unmap_grant_ref(struct ioctl_gntdev_unmap_grant_ref *arg)
static d_ioctl_t gntdev_ioctl
static int gntdev_map_grant_ref(struct ioctl_gntdev_map_grant_ref *arg)
MTX_SYSINIT(to_kill_grefs_mtx, &cleanup_data.to_kill_grefs_mtx, "gntdev to_kill_grefs mutex", MTX_DEF)
static int gmap_cmp(struct gntdev_gmap *g1, struct gntdev_gmap *g2)
static int gntdev_probe(device_t dev)
static struct cdevsw gntdev_devsw
static void per_user_data_dtor(void *arg)
void gnttab_free_grant_reference(grant_ref_t ref)
int gnttab_end_foreign_access_ref(grant_ref_t ref)
int gnttab_query_foreign_access(grant_ref_t ref)
int gnttab_grant_foreign_access(domid_t domid, unsigned long frame, int readonly, grant_ref_t *result)
struct gmap_list_head to_kill_gmaps
struct gref_list_head to_kill_grefs
struct mtx to_kill_grefs_mtx
struct mtx to_kill_gmaps_mtx
vm_paddr_t phys_base_addr
struct resource * pseudo_phys_res
struct gntdev_gmap_map * map
struct notify_data * notify
union gntdev_gmap::gmap_next_union gmap_next
struct gnttab_map_grant_ref * grant_map_ops
union gntdev_gref::gref_next_union gref_next
struct notify_data * notify
uint32_t event_channel_port
xen_intr_handle_t notify_evtchn_handle
struct mtx user_data_lock
struct file_offset_head file_offset
struct gref_tree_head gref_tree
struct gmap_tree_head gmap_tree
struct resource * xenmem_alloc(device_t dev, int *res_id, size_t size)
int xenmem_free(device_t dev, int res_id, struct resource *res)