diff --git a/etc/mtree/BSD.include.dist b/etc/mtree/BSD.include.dist index 42980d0e5f0e..f08c13ae558a 100644 --- a/etc/mtree/BSD.include.dist +++ b/etc/mtree/BSD.include.dist @@ -90,6 +90,8 @@ cam ata .. + nvme + .. scsi .. .. diff --git a/include/Makefile b/include/Makefile index 30034481395a..e4cdf72ddd4d 100644 --- a/include/Makefile +++ b/include/Makefile @@ -42,7 +42,7 @@ LHDRS= aio.h errno.h fcntl.h linker_set.h poll.h stdatomic.h stdint.h \ LDIRS= bsm cam geom net net80211 netgraph netinet netinet6 \ netipsec netnatm netsmb nfs nfsclient nfsserver sys vm -LSUBDIRS= cam/ata cam/scsi \ +LSUBDIRS= cam/ata cam/nvme cam/scsi \ dev/acpica dev/agp dev/an dev/bktr dev/ciss dev/filemon dev/firewire \ dev/hwpmc \ dev/ic dev/iicbus dev/io dev/lmc dev/mfi dev/nvme \ diff --git a/sys/cam/cam_ccb.h b/sys/cam/cam_ccb.h index 6286c43f50eb..1745644d85f7 100644 --- a/sys/cam/cam_ccb.h +++ b/sys/cam/cam_ccb.h @@ -359,6 +359,8 @@ struct ccb_getdev { u_int8_t serial_num[252]; u_int8_t inq_flags; u_int8_t serial_num_len; + const struct nvme_controller_data *nvme_cdata; + const struct nvme_namespace_data *nvme_data; }; /* Device Statistics CCB */ @@ -619,6 +621,11 @@ struct ccb_pathinq_settings_fc { struct ccb_pathinq_settings_sas { u_int32_t bitrate; /* Mbps */ }; + +struct ccb_pathinq_settings_nvme { + uint16_t nsid; /* Namespace ID for this path */ +}; + #define PATHINQ_SETTINGS_SIZE 128 struct ccb_pathinq { @@ -649,6 +656,7 @@ struct ccb_pathinq { struct ccb_pathinq_settings_spi spi; struct ccb_pathinq_settings_fc fc; struct ccb_pathinq_settings_sas sas; + struct ccb_pathinq_settings_nvme nvme; char ccb_pathinq_settings_opaque[PATHINQ_SETTINGS_SIZE]; } xport_specific; u_int maxio; /* Max supported I/O size, in bytes. */ @@ -975,6 +983,18 @@ struct ccb_trans_settings_sata { #define CTS_SATA_CAPS_D_APST 0x00020000 }; +struct ccb_trans_settings_nvme +{ + u_int valid; /* Which fields to honor */ +#define CTS_NVME_VALID_SPEC 0x01 +#define CTS_NVME_VALID_CAPS 0x02 + u_int spec_major; /* Major version of spec supported */ + u_int spec_minor; /* Minor verison of spec supported */ + u_int spec_tiny; /* Tiny version of spec supported */ + u_int max_xfer; /* Max transfer size (0 -> unlimited */ + u_int caps; +}; + /* Get/Set transfer rate/width/disconnection/tag queueing settings */ struct ccb_trans_settings { struct ccb_hdr ccb_h; @@ -987,6 +1007,7 @@ struct ccb_trans_settings { u_int valid; /* Which fields to honor */ struct ccb_trans_settings_ata ata; struct ccb_trans_settings_scsi scsi; + struct ccb_trans_settings_nvme nvme; } proto_specific; union { u_int valid; /* Which fields to honor */ @@ -995,6 +1016,7 @@ struct ccb_trans_settings { struct ccb_trans_settings_sas sas; struct ccb_trans_settings_pata ata; struct ccb_trans_settings_sata sata; + struct ccb_trans_settings_nvme nvme; } xport_specific; }; diff --git a/sys/cam/nvme/nvme_xpt.c b/sys/cam/nvme/nvme_xpt.c index 9139e29f7bff..8410fc37f145 100644 --- a/sys/cam/nvme/nvme_xpt.c +++ b/sys/cam/nvme/nvme_xpt.c @@ -137,7 +137,7 @@ static cam_status nvme_probe_register(struct cam_periph *periph, static void nvme_probe_schedule(struct cam_periph *nvme_probe_periph); static void nvme_probe_start(struct cam_periph *periph, union ccb *start_ccb); static void nvme_probe_cleanup(struct cam_periph *periph); -static void nvme_find_quirk(struct cam_ed *device); +//static void nvme_find_quirk(struct cam_ed *device); static void nvme_scan_lun(struct cam_periph *periph, struct cam_path *path, cam_flags flags, union ccb *ccb); @@ -312,6 +312,7 @@ nvme_probe_cleanup(struct cam_periph *periph) free(periph->softc, M_CAMXPT); } +#if 0 /* XXX should be used, don't delete */ static void nvme_find_quirk(struct cam_ed *device) @@ -334,6 +335,7 @@ nvme_find_quirk(struct cam_ed *device) device->maxtags = quirk->maxtags; } } +#endif static void nvme_scan_lun(struct cam_periph *periph, struct cam_path *path, diff --git a/sys/conf/files b/sys/conf/files index 49dca44063de..7982fd716527 100644 --- a/sys/conf/files +++ b/sys/conf/files @@ -87,7 +87,7 @@ cam/ata/ata_all.c optional scbus cam/ata/ata_xpt.c optional scbus cam/ata/ata_pmp.c optional scbus cam/nvme/nvme_all.c optional scbus nvme -cam/nvme/nvme_da.c optional scbus nvme da +cam/nvme/nvme_da.c optional scbus nvme da !nvd cam/nvme/nvme_xpt.c optional scbus nvme cam/scsi/scsi_xpt.c optional scbus cam/scsi/scsi_all.c optional scbus diff --git a/sys/dev/nvme/nvme.h b/sys/dev/nvme/nvme.h index 7c65fbb461a3..970902295721 100644 --- a/sys/dev/nvme/nvme.h +++ b/sys/dev/nvme/nvme.h @@ -47,7 +47,8 @@ */ #define NVME_GLOBAL_NAMESPACE_TAG ((uint32_t)0xFFFFFFFF) -#define NVME_MAX_XFER_SIZE MAXPHYS +/* Cap nvme to 1MB transfers driver explodes with larger sizes */ +#define NVME_MAX_XFER_SIZE (MAXPHYS < (1<<20) ? MAXPHYS : (1<<20)) union cap_lo_register { uint32_t raw; @@ -903,6 +904,52 @@ uint32_t nvme_ns_get_stripesize(struct nvme_namespace *ns); int nvme_ns_bio_process(struct nvme_namespace *ns, struct bio *bp, nvme_cb_fn_t cb_fn); +/* Command building helper functions -- shared with CAM */ +static inline +void nvme_ns_flush_cmd(struct nvme_command *cmd, uint16_t nsid) +{ + + cmd->opc = NVME_OPC_FLUSH; + cmd->nsid = nsid; +} + +static inline +void nvme_ns_rw_cmd(struct nvme_command *cmd, uint32_t rwcmd, uint16_t nsid, + uint64_t lba, uint32_t count) +{ + cmd->opc = rwcmd; + cmd->nsid = nsid; + *(uint64_t *)&cmd->cdw10 = lba; + cmd->cdw12 = count-1; + cmd->cdw13 = 0; + cmd->cdw14 = 0; + cmd->cdw15 = 0; +} + +static inline +void nvme_ns_write_cmd(struct nvme_command *cmd, uint16_t nsid, + uint64_t lba, uint32_t count) +{ + nvme_ns_rw_cmd(cmd, NVME_OPC_WRITE, nsid, lba, count); +} + +static inline +void nvme_ns_read_cmd(struct nvme_command *cmd, uint16_t nsid, + uint64_t lba, uint32_t count) +{ + nvme_ns_rw_cmd(cmd, NVME_OPC_READ, nsid, lba, count); +} + +static inline +void nvme_ns_trim_cmd(struct nvme_command *cmd, uint16_t nsid, + uint32_t num_ranges) +{ + cmd->opc = NVME_OPC_DATASET_MANAGEMENT; + cmd->nsid = nsid; + cmd->cdw10 = num_ranges - 1; + cmd->cdw11 = NVME_DSM_ATTR_DEALLOCATE; +} + #endif /* _KERNEL */ #endif /* __NVME_H__ */ diff --git a/sys/dev/nvme/nvme_ctrlr.c b/sys/dev/nvme/nvme_ctrlr.c index 00277f08263e..fa889eb21a79 100644 --- a/sys/dev/nvme/nvme_ctrlr.c +++ b/sys/dev/nvme/nvme_ctrlr.c @@ -27,6 +27,8 @@ #include __FBSDID("$FreeBSD$"); +#include "opt_cam.h" + #include #include #include @@ -801,7 +803,7 @@ nvme_ctrlr_reset_task(void *arg, int pending) atomic_cmpset_32(&ctrlr->is_resetting, 1, 0); } -static void +void nvme_ctrlr_intx_handler(void *arg) { struct nvme_controller *ctrlr = arg; diff --git a/sys/dev/nvme/nvme_ns_cmd.c b/sys/dev/nvme/nvme_ns_cmd.c index b5c45dcbedf5..303e03c27039 100644 --- a/sys/dev/nvme/nvme_ns_cmd.c +++ b/sys/dev/nvme/nvme_ns_cmd.c @@ -34,20 +34,14 @@ nvme_ns_cmd_read(struct nvme_namespace *ns, void *payload, uint64_t lba, uint32_t lba_count, nvme_cb_fn_t cb_fn, void *cb_arg) { struct nvme_request *req; - struct nvme_command *cmd; req = nvme_allocate_request_vaddr(payload, lba_count*nvme_ns_get_sector_size(ns), cb_fn, cb_arg); if (req == NULL) return (ENOMEM); - cmd = &req->cmd; - cmd->opc = NVME_OPC_READ; - cmd->nsid = ns->id; - /* TODO: create a read command data structure */ - *(uint64_t *)&cmd->cdw10 = lba; - cmd->cdw12 = lba_count-1; + nvme_ns_read_cmd(&req->cmd, ns->id, lba, lba_count); nvme_ctrlr_submit_io_request(ns->ctrlr, req); @@ -59,7 +53,6 @@ nvme_ns_cmd_read_bio(struct nvme_namespace *ns, struct bio *bp, nvme_cb_fn_t cb_fn, void *cb_arg) { struct nvme_request *req; - struct nvme_command *cmd; uint64_t lba; uint64_t lba_count; @@ -67,16 +60,10 @@ nvme_ns_cmd_read_bio(struct nvme_namespace *ns, struct bio *bp, if (req == NULL) return (ENOMEM); - cmd = &req->cmd; - cmd->opc = NVME_OPC_READ; - cmd->nsid = ns->id; lba = bp->bio_offset / nvme_ns_get_sector_size(ns); lba_count = bp->bio_bcount / nvme_ns_get_sector_size(ns); - - /* TODO: create a read command data structure */ - *(uint64_t *)&cmd->cdw10 = lba; - cmd->cdw12 = lba_count-1; + nvme_ns_read_cmd(&req->cmd, ns->id, lba, lba_count); nvme_ctrlr_submit_io_request(ns->ctrlr, req); @@ -88,7 +75,6 @@ nvme_ns_cmd_write(struct nvme_namespace *ns, void *payload, uint64_t lba, uint32_t lba_count, nvme_cb_fn_t cb_fn, void *cb_arg) { struct nvme_request *req; - struct nvme_command *cmd; req = nvme_allocate_request_vaddr(payload, lba_count*nvme_ns_get_sector_size(ns), cb_fn, cb_arg); @@ -96,13 +82,7 @@ nvme_ns_cmd_write(struct nvme_namespace *ns, void *payload, uint64_t lba, if (req == NULL) return (ENOMEM); - cmd = &req->cmd; - cmd->opc = NVME_OPC_WRITE; - cmd->nsid = ns->id; - - /* TODO: create a write command data structure */ - *(uint64_t *)&cmd->cdw10 = lba; - cmd->cdw12 = lba_count-1; + nvme_ns_write_cmd(&req->cmd, ns->id, lba, lba_count); nvme_ctrlr_submit_io_request(ns->ctrlr, req); @@ -114,7 +94,6 @@ nvme_ns_cmd_write_bio(struct nvme_namespace *ns, struct bio *bp, nvme_cb_fn_t cb_fn, void *cb_arg) { struct nvme_request *req; - struct nvme_command *cmd; uint64_t lba; uint64_t lba_count; @@ -122,16 +101,9 @@ nvme_ns_cmd_write_bio(struct nvme_namespace *ns, struct bio *bp, if (req == NULL) return (ENOMEM); - cmd = &req->cmd; - cmd->opc = NVME_OPC_WRITE; - cmd->nsid = ns->id; - lba = bp->bio_offset / nvme_ns_get_sector_size(ns); lba_count = bp->bio_bcount / nvme_ns_get_sector_size(ns); - - /* TODO: create a write command data structure */ - *(uint64_t *)&cmd->cdw10 = lba; - cmd->cdw12 = lba_count-1; + nvme_ns_write_cmd(&req->cmd, ns->id, lba, lba_count); nvme_ctrlr_submit_io_request(ns->ctrlr, req); @@ -168,17 +140,13 @@ int nvme_ns_cmd_flush(struct nvme_namespace *ns, nvme_cb_fn_t cb_fn, void *cb_arg) { struct nvme_request *req; - struct nvme_command *cmd; req = nvme_allocate_request_null(cb_fn, cb_arg); if (req == NULL) return (ENOMEM); - cmd = &req->cmd; - cmd->opc = NVME_OPC_FLUSH; - cmd->nsid = ns->id; - + nvme_ns_flush_cmd(&req->cmd, ns->id); nvme_ctrlr_submit_io_request(ns->ctrlr, req); return (0); diff --git a/sys/dev/nvme/nvme_private.h b/sys/dev/nvme/nvme_private.h index 33307117b5be..aa02b45227d6 100644 --- a/sys/dev/nvme/nvme_private.h +++ b/sys/dev/nvme/nvme_private.h @@ -245,6 +245,10 @@ struct nvme_controller { struct mtx lock; + struct cam_sim *sim; + struct cam_path *path; + int cam_ref; + uint32_t ready_timeout_in_ms; bus_space_tag_t bus_tag; @@ -528,4 +532,6 @@ void nvme_notify_async_consumers(struct nvme_controller *ctrlr, void nvme_notify_fail_consumers(struct nvme_controller *ctrlr); void nvme_notify_new_controller(struct nvme_controller *ctrlr); +void nvme_ctrlr_intx_handler(void *arg); + #endif /* __NVME_PRIVATE_H__ */