34#include <sys/kernel.h>
38#include <sys/ioccom.h>
40#include <sys/linker.h>
42#include <sys/malloc.h>
43#include <sys/module.h>
48#include <sys/sysctl.h>
50#include <machine/bus.h>
51#include <machine/stdarg.h>
54#include <sys/iov_schema.h>
70 .d_version = D_VERSION,
86#define IOV_READ(d, r, w) \
87 pci_read_config((d)->cfg.dev, (d)->cfg.iov->iov_pos + r, w)
89#define IOV_WRITE(d, r, v, w) \
90 pci_write_config((d)->cfg.dev, (d)->cfg.iov->iov_pos + r, v, w)
95 nvlist_t **driver_schema);
97 nvlist_t **driver_schema);
104 struct nvlist *
vf_schema,
const char *fmt, ...)
106 char buf[NAME_MAX + 1];
110 vsnprintf(buf,
sizeof(buf), fmt, ap);
120 struct pci_devinfo *dinfo;
127 dinfo = device_get_ivars(
dev);
139 "Unsupported version of SR-IOV (%d) detected\n",
145 iov = malloc(
sizeof(*dinfo->cfg.iov), M_SRIOV, M_WAITOK | M_ZERO);
148 if (dinfo->cfg.iov != NULL) {
156 if (schema == NULL) {
167 UID_ROOT, GID_WHEEL, 0600,
"iov/%s",
name);
174 dinfo->cfg.iov = iov;
181 nvlist_destroy(schema);
192 struct pci_devinfo *dinfo;
197 dinfo = device_get_ivars(
dev);
198 iov = dinfo->cfg.iov;
216 dinfo->cfg.iov = NULL;
233 nvlist_t *schema, *pf_driver, *vf_driver;
248 if (nvlist_error(schema) != 0)
254 nvlist_destroy(schema);
255 nvlist_destroy(pf_driver);
256 nvlist_destroy(vf_driver);
267 nvlist_set_error(schema, ENOMEM);
279 nvlist_move_nvlist(
pf_schema, DRIVER_CONFIG_NAME, *driver_schema);
281 nvlist_move_nvlist(schema, PF_CONFIG_NAME,
pf_schema);
282 *driver_schema = NULL;
292 nvlist_set_error(schema, ENOMEM);
304 nvlist_move_nvlist(
vf_schema, DRIVER_CONFIG_NAME, *driver_schema);
306 nvlist_move_nvlist(schema, VF_SCHEMA_NAME,
vf_schema);
307 *driver_schema = NULL;
342 struct resource *res;
345 rman_res_t
start, end;
349 iov = dinfo->cfg.iov;
350 dev = dinfo->cfg.dev;
351 bus = device_get_parent(
dev);
353 bar_size = 1 << bar_shift;
365 start = rman_get_start(res);
366 end = rman_get_end(res);
367 return (rman_manage_region(&iov->
rman,
start, end));
379 if (bar->
res != NULL) {
380 bar_start = rman_get_start(bar->
res) +
381 dinfo->cfg.vf.index * bar->
bar_size;
398 packed_config = NULL;
405 packed_config = malloc(arg->len, M_SRIOV, M_WAITOK);
407 error = copyin(arg->config, packed_config, arg->len);
411 config = nvlist_unpack(packed_config, arg->len, NV_FLAG_IGNORE_CASE);
421 error = nvlist_error(
config);
430 free(packed_config, M_SRIOV);
444 int i, error, devcount, lowest_func, lowest_pos, iov_pos, dev_func;
448 if (!PCIB_ARI_ENABLED(device_get_parent(
bus)))
451 error = device_get_children(
bus, &devlist, &devcount);
457 for (i = 0; i < devcount; i++) {
458 if (pci_find_extcap(devlist[i],
PCIZ_SRIOV, &iov_pos) == 0) {
459 dev_func = pci_get_function(devlist[i]);
460 if (lowest == NULL || dev_func < lowest_func) {
462 lowest_func = dev_func;
463 lowest_pos = iov_pos;
467 free(devlist, M_TEMP);
473 KASSERT(lowest != NULL,
474 (
"Could not find child of %s with SR-IOV capability",
475 device_get_nameunit(
bus)));
477 iov_ctl = pci_read_config(lowest, lowest_pos +
PCIR_SRIOV_CTL, 2);
479 pci_write_config(lowest, lowest_pos +
PCIR_SRIOV_CTL, iov_ctl, 2);
482 device_printf(lowest,
"failed to enable ARI\n");
491 uint32_t page_cap, page_size;
500 page_size = (1 << 0);
505 if (!(page_size & page_cap))
515 const nvlist_t *device, *driver_config;
517 device = nvlist_get_nvlist(
config, PF_CONFIG_NAME);
518 driver_config = nvlist_get_nvlist(device, DRIVER_CONFIG_NAME);
519 return (PCI_IOV_INIT(
dev,
num_vfs, driver_config));
527 iov->
rman.rm_start = 0;
528 iov->
rman.rm_end = ~0;
529 iov->
rman.rm_type = RMAN_ARRAY;
531 device_get_nameunit(
pf));
534 error = rman_init(&iov->
rman);
546 rman_res_t
start, end;
547 struct resource *res;
548 struct resource_list *rl;
549 struct resource_list_entry *rle;
551 rl = &dinfo->resources;
552 iov = dinfo->cfg.iov;
554 rle = resource_list_find(rl, SYS_RES_MEMORY,
557 rle = resource_list_find(rl, SYS_RES_IOPORT,
567 start = rman_get_start(res);
568 end = rman_get_end(res);
570 return (rman_manage_region(&iov->
rman,
start, end));
579 int i, last_64, error;
581 iov = dinfo->cfg.iov;
582 dev = dinfo->cfg.dev;
606 &bar_value, &testval, &last_64);
623 uint16_t first_rid, uint16_t rid_stride)
625 char device_name[VF_MAX_NAME];
626 const nvlist_t *device, *driver_config, *iov_config;
629 struct pci_devinfo *vfinfo;
631 uint16_t
vid,
did, next_rid;
633 iov = dinfo->cfg.iov;
634 dev = dinfo->cfg.dev;
635 bus = device_get_parent(
dev);
636 next_rid = first_rid;
637 vid = pci_get_vendor(
dev);
640 for (i = 0; i < iov->
iov_num_vfs; i++, next_rid += rid_stride) {
641 snprintf(device_name,
sizeof(device_name), VF_PREFIX
"%d", i);
642 device = nvlist_get_nvlist(
config, device_name);
643 iov_config = nvlist_get_nvlist(device, IOV_CONFIG_NAME);
644 driver_config = nvlist_get_nvlist(device, DRIVER_CONFIG_NAME);
646 vf = PCI_CREATE_IOV_CHILD(
bus,
dev, next_rid,
vid,
did);
655 if (nvlist_get_bool(iov_config,
"passthrough"))
656 device_set_devclass_fixed(vf,
"ppt");
658 vfinfo = device_get_ivars(vf);
660 vfinfo->cfg.iov = iov;
661 vfinfo->cfg.vf.index = i;
665 error = PCI_IOV_ADD_VF(
dev, i, driver_config);
667 device_printf(
dev,
"Failed to add VF %d\n", i);
668 device_delete_child(
bus, vf);
672 bus_generic_attach(
bus);
679 struct pci_devinfo *dinfo;
683 uint16_t rid_off, rid_stride;
684 uint16_t first_rid, last_rid;
690 dinfo = cdev->si_drv1;
691 iov = dinfo->cfg.iov;
692 dev = dinfo->cfg.dev;
693 bus = device_get_parent(
dev);
732 first_rid = pci_get_rid(
dev) + rid_off;
733 last_rid = first_rid + (
num_vfs - 1) * rid_stride;
760 pause(
"iov", roundup(hz, 10));
784 rman_fini(&iov->
rman);
800 iov = dinfo->cfg.iov;
812 iov = dinfo->cfg.iov;
822 struct pci_devinfo *vfinfo;
824 vfinfo = device_get_ivars(
child);
829 return (
pf == vfinfo->cfg.iov);
835 device_t
bus,
dev, vf, *devlist;
837 int i, error, devcount;
840 mtx_assert(&Giant, MA_OWNED);
842 iov = dinfo->cfg.iov;
843 dev = dinfo->cfg.dev;
844 bus = device_get_parent(
dev);
849 error = device_get_children(
bus, &devlist, &devcount);
854 for (i = 0; i < devcount; i++) {
860 error = device_detach(vf);
863 "Could not disable SR-IOV: failed to detach VF %s\n",
864 device_get_nameunit(vf));
869 for (i = 0; i < devcount; i++) {
873 device_delete_child(
bus, vf);
896 rman_fini(&iov->
rman);
902 free(devlist, M_TEMP);
910 struct pci_devinfo *dinfo;
915 dinfo = cdev->si_drv1;
916 iov = dinfo->cfg.iov;
937 struct pci_devinfo *dinfo;
939 size_t output_len, size;
945 dinfo = cdev->si_drv1;
946 packed = nvlist_pack(dinfo->cfg.iov->iov_schema, &size);
949 if (packed == NULL) {
954 output_len = output->len;
956 if (size <= output_len) {
957 error = copyout(packed, output->schema, size);
969 output->error = EMSGSIZE;
974 free(packed, M_NVLIST);
991 (
struct pci_iov_schema *)
data));
999 rman_res_t
start, rman_res_t end, rman_res_t
count, u_int flags)
1001 struct pci_devinfo *dinfo;
1004 struct resource *res;
1005 struct resource_list_entry *rle;
1006 rman_res_t bar_start, bar_end;
1010 dinfo = device_get_ivars(
child);
1011 iov = dinfo->cfg.iov;
1017 bar_length = 1 << map->
pm_size;
1019 bar_end = bar_start + bar_length - 1;
1022 if (bar_start >= end || bar_end <= bar_start ||
count != 1)
1026 if (bar_start <
start)
1030 bar_length = bar_end - bar_start + 1;
1032 res = rman_reserve_resource(&iov->
rman, bar_start, bar_end,
1033 bar_length, flags,
child);
1037 rle = resource_list_add(&dinfo->resources, SYS_RES_MEMORY, *
rid,
1038 bar_start, bar_end, 1);
1040 rman_release_resource(res);
1044 rman_set_rid(res, *
rid);
1046 if (flags & RF_ACTIVE) {
1047 error = bus_activate_resource(
child, SYS_RES_MEMORY, *
rid, res);
1049 resource_list_delete(&dinfo->resources, SYS_RES_MEMORY,
1051 rman_release_resource(res);
1064 struct pci_devinfo *dinfo;
1065 struct resource_list_entry *rle;
1068 dinfo = device_get_ivars(
child);
1070 if (rman_get_flags(r) & RF_ACTIVE) {
1071 error = bus_deactivate_resource(
child, SYS_RES_MEMORY,
rid, r);
1076 rle = resource_list_find(&dinfo->resources, SYS_RES_MEMORY,
rid);
1079 resource_list_delete(&dinfo->resources, SYS_RES_MEMORY,
1083 return (rman_release_resource(r));
int pci_ea_is_enabled(device_t dev, int rid)
struct pci_map * pci_find_bar(device_t dev, int reg)
int pci_release_resource(device_t dev, device_t child, int type, int rid, struct resource *r)
void pci_delete_resource(device_t dev, device_t child, int type, int rid)
int pci_mapsize(uint64_t testval)
struct resource * pci_alloc_multi_resource(device_t dev, device_t child, int type, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_long num, u_int flags)
void pci_add_resources_ea(device_t bus, device_t dev, int alloc_iov)
struct pci_map * pci_add_bar(device_t dev, int reg, pci_addr_t value, pci_addr_t size)
void pci_read_bar(device_t dev, int reg, pci_addr_t *mapp, pci_addr_t *testvalp, int *bar64)
struct nvlist * vf_schema
struct nvlist * pf_schema
static d_ioctl_t pci_iov_ioctl
int pci_iov_attach_method(device_t bus, device_t dev, nvlist_t *pf_schema, nvlist_t *vf_schema, const char *name)
static void pci_iov_build_pf_schema(nvlist_t *schema, nvlist_t **driver_schema)
static void pci_iov_enumerate_vfs(struct pci_devinfo *dinfo, const nvlist_t *config, uint16_t first_rid, uint16_t rid_stride)
int pci_iov_attach_name(device_t dev, struct nvlist *pf_schema, struct nvlist *vf_schema, const char *fmt,...)
static MALLOC_DEFINE(M_SRIOV, "sr_iov", "PCI SR-IOV allocations")
static nvlist_t * pci_iov_get_pf_subsystem_schema(void)
static nvlist_t * pci_iov_get_vf_subsystem_schema(void)
SYSCTL_ULONG(_hw_pci, OID_AUTO, iov_max_config, CTLFLAG_RWTUN, &pci_iov_max_config, 0, "Maximum allowed size of SR-IOV configuration.")
static int pci_iov_delete_iov_children(struct pci_devinfo *dinfo)
static int pci_iov_delete(struct cdev *cdev)
int pci_iov_detach_method(device_t bus, device_t dev)
static int pci_iov_init_rman(device_t pf, struct pcicfg_iov *iov)
struct resource * pci_vf_alloc_mem_resource(device_t dev, device_t child, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_int flags)
void pci_iov_cfg_save(device_t dev, struct pci_devinfo *dinfo)
static int pci_iov_parse_config(struct pcicfg_iov *iov, struct pci_iov_arg *arg, nvlist_t **ret)
static void pci_iov_add_bars(struct pcicfg_iov *iov, struct pci_devinfo *dinfo)
static int pci_iov_alloc_bar_ea(struct pci_devinfo *dinfo, int bar)
static int pci_iov_config_page_size(struct pci_devinfo *dinfo)
#define IOV_WRITE(d, r, v, w)
static void pci_iov_build_vf_schema(nvlist_t *schema, nvlist_t **driver_schema)
int pci_vf_release_mem_resource(device_t dev, device_t child, int rid, struct resource *r)
static int pci_iov_set_ari(device_t bus)
static int pci_iov_config(struct cdev *cdev, struct pci_iov_arg *arg)
static int pci_iov_setup_bars(struct pci_devinfo *dinfo)
static int pci_iov_is_child_vf(struct pcicfg_iov *pf, device_t child)
static int pci_iov_get_schema_ioctl(struct cdev *cdev, struct pci_iov_schema *output)
static int pci_iov_alloc_bar(struct pci_devinfo *dinfo, int bar, pci_addr_t bar_shift)
static u_long pci_iov_max_config
static struct cdevsw iov_cdevsw
static int pci_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t *config)
void pci_iov_cfg_restore(device_t dev, struct pci_devinfo *dinfo)
#define IOV_READ(d, r, w)
static nvlist_t * pci_iov_build_schema(nvlist_t **pf_schema, nvlist_t **vf_schema)
const struct nvlist * config
void pci_iov_schema_add_bool(nvlist_t *schema, const char *name, uint32_t flags, int defaultVal)
uint16_t pci_iov_config_get_num_vfs(const nvlist_t *config)
void pci_iov_schema_add_uint16(nvlist_t *schema, const char *name, uint32_t flags, uint16_t defaultVal)
int pci_iov_validate_schema(const nvlist_t *schema)
nvlist_t * pci_iov_schema_alloc_node(void)
void pci_iov_schema_add_string(nvlist_t *schema, const char *name, uint32_t flags, const char *defaultVal)
int pci_iov_schema_validate_config(const nvlist_t *schema, nvlist_t *config)
#define PCIR_SRIOV_VF_STRIDE
#define PCIM_SRIOV_ARI_EN
#define PCI_SRIOV_BASE_PAGE_SHIFT
#define PCI_EXTCAP_VER(ecap)
#define PCIR_SRIOV_VF_DID
#define PCIR_SRIOV_PAGE_CAP
#define PCIR_SRIOV_VF_OFF
#define PCIR_SRIOV_BAR(x)
#define PCIR_SRIOV_TOTAL_VFS
#define PCIR_SRIOV_PAGE_SIZE
#define PCIR_SRIOV_NUM_VFS
#define PCIM_SRIOV_VF_MSE
struct pci_iov_bar iov_bar[PCIR_MAX_BAR_0+1]