diff options
| author | Linus Torvalds | 2019-03-09 16:53:47 -0800 |
|---|---|---|
| committer | Linus Torvalds | 2019-03-09 16:53:47 -0800 |
| commit | 92fff53b7191cae566be9ca6752069426c7f8241 (patch) | |
| tree | 019396be4719ad3969d0395cfa0a90860be75f4a /drivers/scsi | |
| parent | a50243b1ddcdd766d0d17fbfeeb1a22e62fdc461 (diff) | |
| parent | 26af1a368e40618d67956b1f883fbcfec292c5d8 (diff) | |
Merge tag 'scsi-misc' of git://git.kernel.org/pub/scm/linux/kernel/git/jejb/scsi
Pull SCSI updates from James Bottomley:
"This is mostly update of the usual drivers: arcmsr, qla2xxx, lpfc,
hisi_sas, target/iscsi and target/core.
Additionally Christoph refactored gdth as part of the dma changes. The
major mid-layer change this time is the removal of bidi commands and
with them the whole of the osd/exofs driver and filesystem. This is a
major simplification for block and mq in particular"
* tag 'scsi-misc' of git://git.kernel.org/pub/scm/linux/kernel/git/jejb/scsi: (240 commits)
scsi: cxgb4i: validate tcp sequence number only if chip version <= T5
scsi: cxgb4i: get pf number from lldi->pf
scsi: core: replace GFP_ATOMIC with GFP_KERNEL in scsi_scan.c
scsi: mpt3sas: Add missing breaks in switch statements
scsi: aacraid: Fix missing break in switch statement
scsi: kill command serial number
scsi: csiostor: drop serial_number usage
scsi: mvumi: use request tag instead of serial_number
scsi: dpt_i2o: remove serial number usage
scsi: st: osst: Remove negative constant left-shifts
scsi: ufs-bsg: Allow reading descriptors
scsi: ufs: Allow reading descriptor via raw upiu
scsi: ufs-bsg: Change the calling convention for write descriptor
scsi: ufs: Remove unused device quirks
Revert "scsi: ufs: disable vccq if it's not needed by UFS device"
scsi: megaraid_sas: Remove a bunch of set but not used variables
scsi: clean obsolete return values of eh_timed_out
scsi: sd: Optimal I/O size should be a multiple of physical block size
scsi: MAINTAINERS: SCSI initiator and target tweaks
scsi: fcoe: make use of fip_mode enum complete
...
Diffstat (limited to 'drivers/scsi')
166 files changed, 9712 insertions, 10288 deletions
diff --git a/drivers/scsi/Kconfig b/drivers/scsi/Kconfig index 8f9d9e9fa695..d528018e6fa8 100644 --- a/drivers/scsi/Kconfig +++ b/drivers/scsi/Kconfig @@ -665,7 +665,7 @@ config SCSI_DMX3191D config SCSI_GDTH tristate "Intel/ICP (former GDT SCSI Disk Array) RAID Controller support" - depends on (ISA || EISA || PCI) && SCSI && ISA_DMA_API + depends on PCI && SCSI ---help--- Formerly called GDT SCSI Disk Array Controller Support. @@ -1196,8 +1196,6 @@ config SCSI_AM53C974 PCscsi/PCnet (Am53/79C974) solutions. This is a new implementation base on the generic esp_scsi driver. - Documentation can be found in <file:Documentation/scsi/tmscsim.txt>. - Note that this driver does NOT support Tekram DC390W/U/F, which are based on NCR/Symbios chips. Use "NCR53C8XX SCSI support" for those. @@ -1517,6 +1515,4 @@ source "drivers/scsi/pcmcia/Kconfig" source "drivers/scsi/device_handler/Kconfig" -source "drivers/scsi/osd/Kconfig" - endmenu diff --git a/drivers/scsi/Makefile b/drivers/scsi/Makefile index fcb41ae329c4..8826111fdf4a 100644 --- a/drivers/scsi/Makefile +++ b/drivers/scsi/Makefile @@ -150,7 +150,6 @@ obj-$(CONFIG_CHR_DEV_SG) += sg.o obj-$(CONFIG_CHR_DEV_SCH) += ch.o obj-$(CONFIG_SCSI_ENCLOSURE) += ses.o -obj-$(CONFIG_SCSI_OSD_INITIATOR) += osd/ obj-$(CONFIG_SCSI_HISI_SAS) += hisi_sas/ # This goes last, so that "real" scsi devices probe earlier diff --git a/drivers/scsi/aacraid/Makefile b/drivers/scsi/aacraid/Makefile index 1bd9fd18f7f3..3893b95b140b 100644 --- a/drivers/scsi/aacraid/Makefile +++ b/drivers/scsi/aacraid/Makefile @@ -4,5 +4,3 @@ obj-$(CONFIG_SCSI_AACRAID) := aacraid.o aacraid-objs := linit.o aachba.o commctrl.o comminit.o commsup.o \ dpcsup.o rx.o sa.o rkt.o nark.o src.o - -ccflags-y := -Idrivers/scsi diff --git a/drivers/scsi/aacraid/aachba.c b/drivers/scsi/aacraid/aachba.c index 75ab5ff6b78c..6085aa087a2f 100644 --- a/drivers/scsi/aacraid/aachba.c +++ b/drivers/scsi/aacraid/aachba.c @@ -3455,7 +3455,7 @@ static int delete_disk(struct aac_dev *dev, void __user *arg) } } -int aac_dev_ioctl(struct aac_dev *dev, int cmd, void __user *arg) +int aac_dev_ioctl(struct aac_dev *dev, unsigned int cmd, void __user *arg) { switch (cmd) { case FSACTL_QUERY_DISK: diff --git a/drivers/scsi/aacraid/aacraid.h b/drivers/scsi/aacraid/aacraid.h index 3291d1c16864..1df5171594b8 100644 --- a/drivers/scsi/aacraid/aacraid.h +++ b/drivers/scsi/aacraid/aacraid.h @@ -2706,12 +2706,12 @@ void aac_set_intx_mode(struct aac_dev *dev); int aac_get_config_status(struct aac_dev *dev, int commit_flag); int aac_get_containers(struct aac_dev *dev); int aac_scsi_cmd(struct scsi_cmnd *cmd); -int aac_dev_ioctl(struct aac_dev *dev, int cmd, void __user *arg); +int aac_dev_ioctl(struct aac_dev *dev, unsigned int cmd, void __user *arg); #ifndef shost_to_class #define shost_to_class(shost) &shost->shost_dev #endif ssize_t aac_get_serial_number(struct device *dev, char *buf); -int aac_do_ioctl(struct aac_dev * dev, int cmd, void __user *arg); +int aac_do_ioctl(struct aac_dev *dev, unsigned int cmd, void __user *arg); int aac_rx_init(struct aac_dev *dev); int aac_rkt_init(struct aac_dev *dev); int aac_nark_init(struct aac_dev *dev); diff --git a/drivers/scsi/aacraid/commctrl.c b/drivers/scsi/aacraid/commctrl.c index e2899ff7913e..f0ff40332753 100644 --- a/drivers/scsi/aacraid/commctrl.c +++ b/drivers/scsi/aacraid/commctrl.c @@ -1060,7 +1060,7 @@ static int aac_send_reset_adapter(struct aac_dev *dev, void __user *arg) return retval; } -int aac_do_ioctl(struct aac_dev * dev, int cmd, void __user *arg) +int aac_do_ioctl(struct aac_dev *dev, unsigned int cmd, void __user *arg) { int status; diff --git a/drivers/scsi/aacraid/commsup.c b/drivers/scsi/aacraid/commsup.c index d5a6aa9676c8..e67e032936ef 100644 --- a/drivers/scsi/aacraid/commsup.c +++ b/drivers/scsi/aacraid/commsup.c @@ -1303,8 +1303,9 @@ static void aac_handle_aif(struct aac_dev * dev, struct fib * fibptr) ADD : DELETE; break; } - case AifBuManagerEvent: - aac_handle_aif_bu(dev, aifcmd); + break; + case AifBuManagerEvent: + aac_handle_aif_bu(dev, aifcmd); break; } @@ -1376,18 +1377,19 @@ static void aac_handle_aif(struct aac_dev * dev, struct fib * fibptr) container = 0; retry_next: - if (device_config_needed == NOTHING) - for (; container < dev->maximum_num_containers; ++container) { - if ((dev->fsa_dev[container].config_waiting_on == 0) && - (dev->fsa_dev[container].config_needed != NOTHING) && - time_before(jiffies, dev->fsa_dev[container].config_waiting_stamp + AIF_SNIFF_TIMEOUT)) { - device_config_needed = - dev->fsa_dev[container].config_needed; - dev->fsa_dev[container].config_needed = NOTHING; - channel = CONTAINER_TO_CHANNEL(container); - id = CONTAINER_TO_ID(container); - lun = CONTAINER_TO_LUN(container); - break; + if (device_config_needed == NOTHING) { + for (; container < dev->maximum_num_containers; ++container) { + if ((dev->fsa_dev[container].config_waiting_on == 0) && + (dev->fsa_dev[container].config_needed != NOTHING) && + time_before(jiffies, dev->fsa_dev[container].config_waiting_stamp + AIF_SNIFF_TIMEOUT)) { + device_config_needed = + dev->fsa_dev[container].config_needed; + dev->fsa_dev[container].config_needed = NOTHING; + channel = CONTAINER_TO_CHANNEL(container); + id = CONTAINER_TO_ID(container); + lun = CONTAINER_TO_LUN(container); + break; + } } } if (device_config_needed == NOTHING) diff --git a/drivers/scsi/aacraid/linit.c b/drivers/scsi/aacraid/linit.c index 7e56a11836c1..a45f81ec80ce 100644 --- a/drivers/scsi/aacraid/linit.c +++ b/drivers/scsi/aacraid/linit.c @@ -616,7 +616,8 @@ static struct device_attribute *aac_dev_attrs[] = { NULL, }; -static int aac_ioctl(struct scsi_device *sdev, int cmd, void __user * arg) +static int aac_ioctl(struct scsi_device *sdev, unsigned int cmd, + void __user *arg) { struct aac_dev *dev = (struct aac_dev *)sdev->host->hostdata; if (!capable(CAP_SYS_RAWIO)) @@ -852,8 +853,7 @@ static u8 aac_eh_tmf_hard_reset_fib(struct aac_hba_map_info *info, address = (u64)fib->hw_error_pa; rst->error_ptr_hi = cpu_to_le32((u32)(address >> 32)); - rst->error_ptr_lo = cpu_to_le32 - ((u32)(address & 0xffffffff)); + rst->error_ptr_lo = cpu_to_le32((u32)(address & 0xffffffff)); rst->error_length = cpu_to_le32(FW_ERROR_BUFFER_SIZE); fib->hbacmd_size = sizeof(*rst); @@ -1206,7 +1206,8 @@ static long aac_compat_do_ioctl(struct aac_dev *dev, unsigned cmd, unsigned long return ret; } -static int aac_compat_ioctl(struct scsi_device *sdev, int cmd, void __user *arg) +static int aac_compat_ioctl(struct scsi_device *sdev, unsigned int cmd, + void __user *arg) { struct aac_dev *dev = (struct aac_dev *)sdev->host->hostdata; if (!capable(CAP_SYS_RAWIO)) diff --git a/drivers/scsi/aacraid/src.c b/drivers/scsi/aacraid/src.c index 8377aec0649d..97bb9e9d201c 100644 --- a/drivers/scsi/aacraid/src.c +++ b/drivers/scsi/aacraid/src.c @@ -1157,7 +1157,7 @@ out: dev_err(&dev->pdev->dev, "%s: %s status = %d", __func__, state_str[state], rc); -return rc; + return rc; } /** * aac_srcv_init - initialize an SRCv card diff --git a/drivers/scsi/aic7xxx/Makefile b/drivers/scsi/aic7xxx/Makefile index c15be2590d1c..e0188ecd85b2 100644 --- a/drivers/scsi/aic7xxx/Makefile +++ b/drivers/scsi/aic7xxx/Makefile @@ -34,7 +34,6 @@ aic79xx-y += aic79xx_osm.o \ aic79xx_proc.o \ aic79xx_osm_pci.o -ccflags-y += -Idrivers/scsi ifdef WARNINGS_BECOME_ERRORS ccflags-y += -Werror endif diff --git a/drivers/scsi/aic7xxx/aic79xx_core.c b/drivers/scsi/aic7xxx/aic79xx_core.c index 9ee75c9a9aa1..7e5044bf05c0 100644 --- a/drivers/scsi/aic7xxx/aic79xx_core.c +++ b/drivers/scsi/aic7xxx/aic79xx_core.c @@ -2285,6 +2285,7 @@ ahd_handle_seqint(struct ahd_softc *ahd, u_int intstat) switch (scb->hscb->task_management) { case SIU_TASKMGMT_ABORT_TASK: tag = SCB_GET_TAG(scb); + /* fall through */ case SIU_TASKMGMT_ABORT_TASK_SET: case SIU_TASKMGMT_CLEAR_TASK_SET: lun = scb->hscb->lun; @@ -2295,6 +2296,7 @@ ahd_handle_seqint(struct ahd_softc *ahd, u_int intstat) break; case SIU_TASKMGMT_LUN_RESET: lun = scb->hscb->lun; + /* fall through */ case SIU_TASKMGMT_TARGET_RESET: { struct ahd_devinfo devinfo; @@ -6550,8 +6552,8 @@ ahd_fini_scbdata(struct ahd_softc *ahd) kfree(sns_map); } ahd_dma_tag_destroy(ahd, scb_data->sense_dmat); - /* FALLTHROUGH */ } + /* fall through */ case 6: { struct map_node *sg_map; @@ -6565,8 +6567,8 @@ ahd_fini_scbdata(struct ahd_softc *ahd) kfree(sg_map); } ahd_dma_tag_destroy(ahd, scb_data->sg_dmat); - /* FALLTHROUGH */ } + /* fall through */ case 5: { struct map_node *hscb_map; @@ -7209,6 +7211,7 @@ ahd_init(struct ahd_softc *ahd) case FLX_CSTAT_OVER: case FLX_CSTAT_UNDER: warn_user++; + /* fall through */ case FLX_CSTAT_INVALID: case FLX_CSTAT_OKAY: if (warn_user == 0 && bootverbose == 0) @@ -8413,7 +8416,7 @@ ahd_search_scb_list(struct ahd_softc *ahd, int target, char channel, if ((scb->flags & SCB_ACTIVE) == 0) printk("Inactive SCB in Waiting List\n"); ahd_done_with_status(ahd, scb, status); - /* FALLTHROUGH */ + /* fall through */ case SEARCH_REMOVE: ahd_rem_wscb(ahd, scbid, prev, next, tid); *list_tail = prev; @@ -8422,6 +8425,7 @@ ahd_search_scb_list(struct ahd_softc *ahd, int target, char channel, break; case SEARCH_PRINT: printk("0x%x ", scbid); + /* fall through */ case SEARCH_COUNT: prev = scbid; break; @@ -9547,8 +9551,8 @@ ahd_download_instr(struct ahd_softc *ahd, u_int instrptr, uint8_t *dconsts) { fmt3_ins = &instr.format3; fmt3_ins->address = ahd_resolve_seqaddr(ahd, fmt3_ins->address); - /* FALLTHROUGH */ } + /* fall through */ case AIC_OP_OR: case AIC_OP_AND: case AIC_OP_XOR: @@ -9559,7 +9563,7 @@ ahd_download_instr(struct ahd_softc *ahd, u_int instrptr, uint8_t *dconsts) fmt1_ins->immediate = dconsts[fmt1_ins->immediate]; } fmt1_ins->parity = 0; - /* FALLTHROUGH */ + /* fall through */ case AIC_OP_ROL: { int i, count; diff --git a/drivers/scsi/arcmsr/arcmsr.h b/drivers/scsi/arcmsr/arcmsr.h index 9c397a2794d6..9220bcf8388f 100644 --- a/drivers/scsi/arcmsr/arcmsr.h +++ b/drivers/scsi/arcmsr/arcmsr.h @@ -49,7 +49,7 @@ struct device_attribute; #define ARCMSR_MAX_OUTSTANDING_CMD 1024 #define ARCMSR_DEFAULT_OUTSTANDING_CMD 128 #define ARCMSR_MIN_OUTSTANDING_CMD 32 -#define ARCMSR_DRIVER_VERSION "v1.40.00.09-20180709" +#define ARCMSR_DRIVER_VERSION "v1.40.00.10-20190116" #define ARCMSR_SCSI_INITIATOR_ID 255 #define ARCMSR_MAX_XFER_SECTORS 512 #define ARCMSR_MAX_XFER_SECTORS_B 4096 @@ -739,7 +739,7 @@ struct AdapterControlBlock #define ACB_ADAPTER_TYPE_C 0x00000002 /* hbc L IOP */ #define ACB_ADAPTER_TYPE_D 0x00000003 /* hbd M IOP */ #define ACB_ADAPTER_TYPE_E 0x00000004 /* hba L IOP */ - u32 roundup_ccbsize; + u32 ioqueue_size; struct pci_dev * pdev; struct Scsi_Host * host; unsigned long vir2phy_offset; @@ -747,6 +747,7 @@ struct AdapterControlBlock uint32_t outbound_int_enable; uint32_t cdb_phyaddr_hi32; uint32_t reg_mu_acc_handle0; + uint64_t cdb_phyadd_hipart; spinlock_t eh_lock; spinlock_t ccblist_lock; spinlock_t postq_lock; @@ -855,11 +856,11 @@ struct AdapterControlBlock ******************************************************************************* */ struct CommandControlBlock{ - /*x32:sizeof struct_CCB=(32+60)byte, x64:sizeof struct_CCB=(64+60)byte*/ + /*x32:sizeof struct_CCB=(64+60)byte, x64:sizeof struct_CCB=(64+60)byte*/ struct list_head list; /*x32: 8byte, x64: 16byte*/ struct scsi_cmnd *pcmd; /*8 bytes pointer of linux scsi command */ struct AdapterControlBlock *acb; /*x32: 4byte, x64: 8byte*/ - uint32_t cdb_phyaddr; /*x32: 4byte, x64: 4byte*/ + unsigned long cdb_phyaddr; /*x32: 4byte, x64: 8byte*/ uint32_t arc_cdb_size; /*x32:4byte,x64:4byte*/ uint16_t ccb_flags; /*x32: 2byte, x64: 2byte*/ #define CCB_FLAG_READ 0x0000 @@ -875,10 +876,10 @@ struct CommandControlBlock{ uint32_t smid; #if BITS_PER_LONG == 64 /* ======================512+64 bytes======================== */ - uint32_t reserved[4]; /*16 byte*/ + uint32_t reserved[3]; /*12 byte*/ #else /* ======================512+32 bytes======================== */ - // uint32_t reserved; /*4 byte*/ + uint32_t reserved[8]; /*32 byte*/ #endif /* ======================================================= */ struct ARCMSR_CDB arcmsr_cdb; diff --git a/drivers/scsi/arcmsr/arcmsr_hba.c b/drivers/scsi/arcmsr/arcmsr_hba.c index 57c6fa388bf6..88053b15c363 100644 --- a/drivers/scsi/arcmsr/arcmsr_hba.c +++ b/drivers/scsi/arcmsr/arcmsr_hba.c @@ -91,6 +91,10 @@ static int cmd_per_lun = ARCMSR_DEFAULT_CMD_PERLUN; module_param(cmd_per_lun, int, S_IRUGO); MODULE_PARM_DESC(cmd_per_lun, " device queue depth(1 ~ 128), default is 32"); +static int dma_mask_64 = 0; +module_param(dma_mask_64, int, S_IRUGO); +MODULE_PARM_DESC(dma_mask_64, " set DMA mask to 64 bits(0 ~ 1), dma_mask_64=1(64 bits), =0(32 bits)"); + static int set_date_time = 0; module_param(set_date_time, int, S_IRUGO); MODULE_PARM_DESC(set_date_time, " send date, time to iop(0 ~ 1), set_date_time=1(enable), default(=0) is disable"); @@ -223,13 +227,13 @@ static struct pci_driver arcmsr_pci_driver = { **************************************************************************** */ -static void arcmsr_free_mu(struct AdapterControlBlock *acb) +static void arcmsr_free_io_queue(struct AdapterControlBlock *acb) { switch (acb->adapter_type) { case ACB_ADAPTER_TYPE_B: case ACB_ADAPTER_TYPE_D: case ACB_ADAPTER_TYPE_E: { - dma_free_coherent(&acb->pdev->dev, acb->roundup_ccbsize, + dma_free_coherent(&acb->pdev->dev, acb->ioqueue_size, acb->dma_coherent2, acb->dma_coherent_handle2); break; } @@ -576,6 +580,58 @@ static void arcmsr_flush_adapter_cache(struct AdapterControlBlock *acb) } } +static void arcmsr_hbaB_assign_regAddr(struct AdapterControlBlock *acb) +{ + struct MessageUnit_B *reg = acb->pmuB; + + if (acb->pdev->device == PCI_DEVICE_ID_ARECA_1203) { + reg->drv2iop_doorbell = MEM_BASE0(ARCMSR_DRV2IOP_DOORBELL_1203); + reg->drv2iop_doorbell_mask = MEM_BASE0(ARCMSR_DRV2IOP_DOORBELL_MASK_1203); + reg->iop2drv_doorbell = MEM_BASE0(ARCMSR_IOP2DRV_DOORBELL_1203); + reg->iop2drv_doorbell_mask = MEM_BASE0(ARCMSR_IOP2DRV_DOORBELL_MASK_1203); + } else { + reg->drv2iop_doorbell= MEM_BASE0(ARCMSR_DRV2IOP_DOORBELL); + reg->drv2iop_doorbell_mask = MEM_BASE0(ARCMSR_DRV2IOP_DOORBELL_MASK); + reg->iop2drv_doorbell = MEM_BASE0(ARCMSR_IOP2DRV_DOORBELL); + reg->iop2drv_doorbell_mask = MEM_BASE0(ARCMSR_IOP2DRV_DOORBELL_MASK); + } + reg->message_wbuffer = MEM_BASE1(ARCMSR_MESSAGE_WBUFFER); + reg->message_rbuffer = MEM_BASE1(ARCMSR_MESSAGE_RBUFFER); + reg->message_rwbuffer = MEM_BASE1(ARCMSR_MESSAGE_RWBUFFER); +} + +static void arcmsr_hbaD_assign_regAddr(struct AdapterControlBlock *acb) +{ + struct MessageUnit_D *reg = acb->pmuD; + + reg->chip_id = MEM_BASE0(ARCMSR_ARC1214_CHIP_ID); + reg->cpu_mem_config = MEM_BASE0(ARCMSR_ARC1214_CPU_MEMORY_CONFIGURATION); + reg->i2o_host_interrupt_mask = MEM_BASE0(ARCMSR_ARC1214_I2_HOST_INTERRUPT_MASK); + reg->sample_at_reset = MEM_BASE0(ARCMSR_ARC1214_SAMPLE_RESET); + reg->reset_request = MEM_BASE0(ARCMSR_ARC1214_RESET_REQUEST); + reg->host_int_status = MEM_BASE0(ARCMSR_ARC1214_MAIN_INTERRUPT_STATUS); + reg->pcief0_int_enable = MEM_BASE0(ARCMSR_ARC1214_PCIE_F0_INTERRUPT_ENABLE); + reg->inbound_msgaddr0 = MEM_BASE0(ARCMSR_ARC1214_INBOUND_MESSAGE0); + reg->inbound_msgaddr1 = MEM_BASE0(ARCMSR_ARC1214_INBOUND_MESSAGE1); + reg->outbound_msgaddr0 = MEM_BASE0(ARCMSR_ARC1214_OUTBOUND_MESSAGE0); + reg->outbound_msgaddr1 = MEM_BASE0(ARCMSR_ARC1214_OUTBOUND_MESSAGE1); + reg->inbound_doorbell = MEM_BASE0(ARCMSR_ARC1214_INBOUND_DOORBELL); + reg->outbound_doorbell = MEM_BASE0(ARCMSR_ARC1214_OUTBOUND_DOORBELL); + reg->outbound_doorbell_enable = MEM_BASE0(ARCMSR_ARC1214_OUTBOUND_DOORBELL_ENABLE); + reg->inboundlist_base_low = MEM_BASE0(ARCMSR_ARC1214_INBOUND_LIST_BASE_LOW); + reg->inboundlist_base_high = MEM_BASE0(ARCMSR_ARC1214_INBOUND_LIST_BASE_HIGH); + reg->inboundlist_write_pointer = MEM_BASE0(ARCMSR_ARC1214_INBOUND_LIST_WRITE_POINTER); + reg->outboundlist_base_low = MEM_BASE0(ARCMSR_ARC1214_OUTBOUND_LIST_BASE_LOW); + reg->outboundlist_base_high = MEM_BASE0(ARCMSR_ARC1214_OUTBOUND_LIST_BASE_HIGH); + reg->outboundlist_copy_pointer = MEM_BASE0(ARCMSR_ARC1214_OUTBOUND_LIST_COPY_POINTER); + reg->outboundlist_read_pointer = MEM_BASE0(ARCMSR_ARC1214_OUTBOUND_LIST_READ_POINTER); + reg->outboundlist_interrupt_cause = MEM_BASE0(ARCMSR_ARC1214_OUTBOUND_INTERRUPT_CAUSE); + reg->outboundlist_interrupt_enable = MEM_BASE0(ARCMSR_ARC1214_OUTBOUND_INTERRUPT_ENABLE); + reg->message_wbuffer = MEM_BASE0(ARCMSR_ARC1214_MESSAGE_WBUFFER); + reg->message_rbuffer = MEM_BASE0(ARCMSR_ARC1214_MESSAGE_RBUFFER); + reg->msgcode_rwbuffer = MEM_BASE0(ARCMSR_ARC1214_MESSAGE_RWBUFFER); +} + static bool arcmsr_alloc_io_queue(struct AdapterControlBlock *acb) { bool rtn = true; @@ -585,88 +641,39 @@ static bool arcmsr_alloc_io_queue(struct AdapterControlBlock *acb) switch (acb->adapter_type) { case ACB_ADAPTER_TYPE_B: { - struct MessageUnit_B *reg; - acb->roundup_ccbsize = roundup(sizeof(struct MessageUnit_B), 32); - dma_coherent = dma_alloc_coherent(&pdev->dev, - acb->roundup_ccbsize, - &dma_coherent_handle, - GFP_KERNEL); + acb->ioqueue_size = roundup(sizeof(struct MessageUnit_B), 32); + dma_coherent = dma_alloc_coherent(&pdev->dev, acb->ioqueue_size, + &dma_coherent_handle, GFP_KERNEL); if (!dma_coherent) { pr_notice("arcmsr%d: DMA allocation failed\n", acb->host->host_no); return false; } acb->dma_coherent_handle2 = dma_coherent_handle; acb->dma_coherent2 = dma_coherent; - reg = (struct MessageUnit_B *)dma_coherent; - acb->pmuB = reg; - if (acb->pdev->device == PCI_DEVICE_ID_ARECA_1203) { - reg->drv2iop_doorbell = MEM_BASE0(ARCMSR_DRV2IOP_DOORBELL_1203); - reg->drv2iop_doorbell_mask = MEM_BASE0(ARCMSR_DRV2IOP_DOORBELL_MASK_1203); - reg->iop2drv_doorbell = MEM_BASE0(ARCMSR_IOP2DRV_DOORBELL_1203); - reg->iop2drv_doorbell_mask = MEM_BASE0(ARCMSR_IOP2DRV_DOORBELL_MASK_1203); - } else { - reg->drv2iop_doorbell = MEM_BASE0(ARCMSR_DRV2IOP_DOORBELL); - reg->drv2iop_doorbell_mask = MEM_BASE0(ARCMSR_DRV2IOP_DOORBELL_MASK); - reg->iop2drv_doorbell = MEM_BASE0(ARCMSR_IOP2DRV_DOORBELL); - reg->iop2drv_doorbell_mask = MEM_BASE0(ARCMSR_IOP2DRV_DOORBELL_MASK); - } - reg->message_wbuffer = MEM_BASE1(ARCMSR_MESSAGE_WBUFFER); - reg->message_rbuffer = MEM_BASE1(ARCMSR_MESSAGE_RBUFFER); - reg->message_rwbuffer = MEM_BASE1(ARCMSR_MESSAGE_RWBUFFER); + acb->pmuB = (struct MessageUnit_B *)dma_coherent; + arcmsr_hbaB_assign_regAddr(acb); } break; case ACB_ADAPTER_TYPE_D: { - struct MessageUnit_D *reg; - - acb->roundup_ccbsize = roundup(sizeof(struct MessageUnit_D), 32); - dma_coherent = dma_alloc_coherent(&pdev->dev, - acb->roundup_ccbsize, - &dma_coherent_handle, - GFP_KERNEL); + acb->ioqueue_size = roundup(sizeof(struct MessageUnit_D), 32); + dma_coherent = dma_alloc_coherent(&pdev->dev, acb->ioqueue_size, + &dma_coherent_handle, GFP_KERNEL); if (!dma_coherent) { pr_notice("arcmsr%d: DMA allocation failed\n", acb->host->host_no); return false; } acb->dma_coherent_handle2 = dma_coherent_handle; acb->dma_coherent2 = dma_coherent; - reg = (struct MessageUnit_D *)dma_coherent; - acb->pmuD = reg; - reg->chip_id = MEM_BASE0(ARCMSR_ARC1214_CHIP_ID); - reg->cpu_mem_config = MEM_BASE0(ARCMSR_ARC1214_CPU_MEMORY_CONFIGURATION); - reg->i2o_host_interrupt_mask = MEM_BASE0(ARCMSR_ARC1214_I2_HOST_INTERRUPT_MASK); - reg->sample_at_reset = MEM_BASE0(ARCMSR_ARC1214_SAMPLE_RESET); - reg->reset_request = MEM_BASE0(ARCMSR_ARC1214_RESET_REQUEST); - reg->host_int_status = MEM_BASE0(ARCMSR_ARC1214_MAIN_INTERRUPT_STATUS); - reg->pcief0_int_enable = MEM_BASE0(ARCMSR_ARC1214_PCIE_F0_INTERRUPT_ENABLE); - reg->inbound_msgaddr0 = MEM_BASE0(ARCMSR_ARC1214_INBOUND_MESSAGE0); - reg->inbound_msgaddr1 = MEM_BASE0(ARCMSR_ARC1214_INBOUND_MESSAGE1); - reg->outbound_msgaddr0 = MEM_BASE0(ARCMSR_ARC1214_OUTBOUND_MESSAGE0); - reg->outbound_msgaddr1 = MEM_BASE0(ARCMSR_ARC1214_OUTBOUND_MESSAGE1); - reg->inbound_doorbell = MEM_BASE0(ARCMSR_ARC1214_INBOUND_DOORBELL); - reg->outbound_doorbell = MEM_BASE0(ARCMSR_ARC1214_OUTBOUND_DOORBELL); - reg->outbound_doorbell_enable = MEM_BASE0(ARCMSR_ARC1214_OUTBOUND_DOORBELL_ENABLE); - reg->inboundlist_base_low = MEM_BASE0(ARCMSR_ARC1214_INBOUND_LIST_BASE_LOW); - reg->inboundlist_base_high = MEM_BASE0(ARCMSR_ARC1214_INBOUND_LIST_BASE_HIGH); - reg->inboundlist_write_pointer = MEM_BASE0(ARCMSR_ARC1214_INBOUND_LIST_WRITE_POINTER); - reg->outboundlist_base_low = MEM_BASE0(ARCMSR_ARC1214_OUTBOUND_LIST_BASE_LOW); - reg->outboundlist_base_high = MEM_BASE0(ARCMSR_ARC1214_OUTBOUND_LIST_BASE_HIGH); - reg->outboundlist_copy_pointer = MEM_BASE0(ARCMSR_ARC1214_OUTBOUND_LIST_COPY_POINTER); - reg->outboundlist_read_pointer = MEM_BASE0(ARCMSR_ARC1214_OUTBOUND_LIST_READ_POINTER); - reg->outboundlist_interrupt_cause = MEM_BASE0(ARCMSR_ARC1214_OUTBOUND_INTERRUPT_CAUSE); - reg->outboundlist_interrupt_enable = MEM_BASE0(ARCMSR_ARC1214_OUTBOUND_INTERRUPT_ENABLE); - reg->message_wbuffer = MEM_BASE0(ARCMSR_ARC1214_MESSAGE_WBUFFER); - reg->message_rbuffer = MEM_BASE0(ARCMSR_ARC1214_MESSAGE_RBUFFER); - reg->msgcode_rwbuffer = MEM_BASE0(ARCMSR_ARC1214_MESSAGE_RWBUFFER); + acb->pmuD = (struct MessageUnit_D *)dma_coherent; + arcmsr_hbaD_assign_regAddr(acb); } break; case ACB_ADAPTER_TYPE_E: { uint32_t completeQ_size; completeQ_size = sizeof(struct deliver_completeQ) * ARCMSR_MAX_HBE_DONEQUEUE + 128; - acb->roundup_ccbsize = roundup(completeQ_size, 32); - dma_coherent = dma_alloc_coherent(&pdev->dev, - acb->roundup_ccbsize, - &dma_coherent_handle, - GFP_KERNEL); + acb->ioqueue_size = roundup(completeQ_size, 32); + dma_coherent = dma_alloc_coherent(&pdev->dev, acb->ioqueue_size, + &dma_coherent_handle, GFP_KERNEL); if (!dma_coherent){ pr_notice("arcmsr%d: DMA allocation failed\n", acb->host->host_no); return false; @@ -674,7 +681,7 @@ static bool arcmsr_alloc_io_queue(struct AdapterControlBlock *acb) acb->dma_coherent_handle2 = dma_coherent_handle; acb->dma_coherent2 = dma_coherent; acb->pCompletionQ = dma_coherent; - acb->completionQ_entry = acb->roundup_ccbsize / sizeof(struct deliver_completeQ); + acb->completionQ_entry = acb->ioqueue_size / sizeof(struct deliver_completeQ); acb->doneq_index = 0; } break; @@ -691,11 +698,11 @@ static int arcmsr_alloc_ccb_pool(struct AdapterControlBlock *acb) dma_addr_t dma_coherent_handle; struct CommandControlBlock *ccb_tmp; int i = 0, j = 0; - dma_addr_t cdb_phyaddr; + unsigned long cdb_phyaddr, next_ccb_phy; unsigned long roundup_ccbsize; unsigned long max_xfer_len; unsigned long max_sg_entrys; - uint32_t firm_config_version; + uint32_t firm_config_version, curr_phy_upper32; for (i = 0; i < ARCMSR_MAX_TARGETID; i++) for (j = 0; j < ARCMSR_MAX_TARGETLUN; j++) @@ -712,6 +719,7 @@ static int arcmsr_alloc_ccb_pool(struct AdapterControlBlock *acb) acb->host->sg_tablesize = max_sg_entrys; roundup_ccbsize = roundup(sizeof(struct CommandControlBlock) + (max_sg_entrys - 1) * sizeof(struct SG64ENTRY), 32); acb->uncache_size = roundup_ccbsize * acb->maxFreeCCB; + acb->uncache_size += acb->ioqueue_size; dma_coherent = dma_alloc_coherent(&pdev->dev, acb->uncache_size, &dma_coherent_handle, GFP_KERNEL); if(!dma_coherent){ printk(KERN_NOTICE "arcmsr%d: dma_alloc_coherent got error\n", acb->host->host_no); @@ -722,9 +730,10 @@ static int arcmsr_alloc_ccb_pool(struct AdapterControlBlock *acb) memset(dma_coherent, 0, acb->uncache_size); acb->ccbsize = roundup_ccbsize; ccb_tmp = dma_coherent; + curr_phy_upper32 = upper_32_bits(dma_coherent_handle); acb->vir2phy_offset = (unsigned long)dma_coherent - (unsigned long)dma_coherent_handle; for(i = 0; i < acb->maxFreeCCB; i++){ - cdb_phyaddr = dma_coherent_handle + offsetof(struct CommandControlBlock, arcmsr_cdb); + cdb_phyaddr = (unsigned long)dma_coherent_handle + offsetof(struct CommandControlBlock, arcmsr_cdb); switch (acb->adapter_type) { case ACB_ADAPTER_TYPE_A: case ACB_ADAPTER_TYPE_B: @@ -740,10 +749,34 @@ static int arcmsr_alloc_ccb_pool(struct AdapterControlBlock *acb) ccb_tmp->acb = acb; ccb_tmp->smid = (u32)i << 16; INIT_LIST_HEAD(&ccb_tmp->list); - list_add_tail(&ccb_tmp->list, &acb->ccb_free_list); + next_ccb_phy = dma_coherent_handle + roundup_ccbsize; + if (upper_32_bits(next_ccb_phy) != curr_phy_upper32) { + acb->maxFreeCCB = i; + acb->host->can_queue = i; + break; + } + else + list_add_tail(&ccb_tmp->list, &acb->ccb_free_list); ccb_tmp = (struct CommandControlBlock *)((unsigned long)ccb_tmp + roundup_ccbsize); - dma_coherent_handle = dma_coherent_handle + roundup_ccbsize; + dma_coherent_handle = next_ccb_phy; } + acb->dma_coherent_handle2 = dma_coherent_handle; + acb->dma_coherent2 = ccb_tmp; + switch (acb->adapter_type) { + case ACB_ADAPTER_TYPE_B: + acb->pmuB = (struct MessageUnit_B *)acb->dma_coherent2; + arcmsr_hbaB_assign_regAddr(acb); + break; + case ACB_ADAPTER_TYPE_D: + acb->pmuD = (struct MessageUnit_D *)acb->dma_coherent2; + arcmsr_hbaD_assign_regAddr(acb); + break; + case ACB_ADAPTER_TYPE_E: + acb->pCompletionQ = acb->dma_coherent2; + acb->completionQ_entry = acb->ioqueue_size / sizeof(struct deliver_completeQ); + acb->doneq_index = 0; + break; + } return 0; } @@ -894,6 +927,31 @@ static void arcmsr_init_set_datetime_timer(struct AdapterControlBlock *pacb) add_timer(&pacb->refresh_timer); } +static int arcmsr_set_dma_mask(struct AdapterControlBlock *acb) +{ + struct pci_dev *pcidev = acb->pdev; + + if (IS_DMA64) { + if (((acb->adapter_type == ACB_ADAPTER_TYPE_A) && !dma_mask_64) || + dma_set_mask(&pcidev->dev, DMA_BIT_MASK(64))) + goto dma32; + if (dma_set_coherent_mask(&pcidev->dev, DMA_BIT_MASK(64)) || + dma_set_mask_and_coherent(&pcidev->dev, DMA_BIT_MASK(64))) { + printk("arcmsr: set DMA 64 mask failed\n"); + return -ENXIO; + } + } else { +dma32: + if (dma_set_mask(&pcidev->dev, DMA_BIT_MASK(32)) || + dma_set_coherent_mask(&pcidev->dev, DMA_BIT_MASK(32)) || + dma_set_mask_and_coherent(&pcidev->dev, DMA_BIT_MASK(32))) { + printk("arcmsr: set DMA 32-bit mask failed\n"); + return -ENXIO; + } + } + return 0; +} + static int arcmsr_probe(struct pci_dev *pdev, const struct pci_device_id *id) { struct Scsi_Host *host; @@ -908,22 +966,15 @@ static int arcmsr_probe(struct pci_dev *pdev, const struct pci_device_id *id) if(!host){ goto pci_disable_dev; } - error = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)); - if(error){ - error = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32)); - if(error){ - printk(KERN_WARNING - "scsi%d: No suitable DMA mask available\n", - host->host_no); - goto scsi_host_release; - } - } init_waitqueue_head(&wait_q); bus = pdev->bus->number; dev_fun = pdev->devfn; acb = (struct AdapterControlBlock *) host->hostdata; memset(acb,0,sizeof(struct AdapterControlBlock)); acb->pdev = pdev; + acb->adapter_type = id->driver_data; + if (arcmsr_set_dma_mask(acb)) + goto scsi_host_release; acb->host = host; host->max_lun = ARCMSR_MAX_TARGETLUN; host->max_id = ARCMSR_MAX_TARGETID; /*16:8*/ @@ -953,7 +1004,6 @@ static int arcmsr_probe(struct pci_dev *pdev, const struct pci_device_id *id) ACB_F_MESSAGE_WQBUFFER_READED); acb->acb_flags &= ~ACB_F_SCSISTOPADAPTER; INIT_LIST_HEAD(&acb->ccb_free_list); - acb->adapter_type = id->driver_data; error = arcmsr_remap_pciregion(acb); if(!error){ goto pci_release_regs; @@ -965,9 +1015,10 @@ static int arcmsr_probe(struct pci_dev *pdev, const struct pci_device_id *id) if(!error){ goto free_hbb_mu; } + arcmsr_free_io_queue(acb); error = arcmsr_alloc_ccb_pool(acb); if(error){ - goto free_hbb_mu; + goto unmap_pci_region; } error = scsi_add_host(host, &pdev->dev); if(error){ @@ -995,8 +1046,9 @@ scsi_host_remove: scsi_remove_host(host); free_ccb_pool: arcmsr_free_ccb_pool(acb); + goto unmap_pci_region; free_hbb_mu: - arcmsr_free_mu(acb); + arcmsr_free_io_queue(acb); unmap_pci_region: arcmsr_unmap_pciregion(acb); pci_release_regs: @@ -1042,7 +1094,6 @@ static int arcmsr_suspend(struct pci_dev *pdev, pm_message_t state) static int arcmsr_resume(struct pci_dev *pdev) { - int error; struct Scsi_Host *host = pci_get_drvdata(pdev); struct AdapterControlBlock *acb = (struct AdapterControlBlock *)host->hostdata; @@ -1054,24 +1105,30 @@ static int arcmsr_resume(struct pci_dev *pdev) pr_warn("%s: pci_enable_device error\n", __func__); return -ENODEV; } - error = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)); - if (error) { - error = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32)); - if (error) { - pr_warn("scsi%d: No suitable DMA mask available\n", - host->host_no); - goto controller_unregister; - } - } + if (arcmsr_set_dma_mask(acb)) + goto controller_unregister; pci_set_master(pdev); if (arcmsr_request_irq(pdev, acb) == FAILED) goto controller_stop; - if (acb->adapter_type == ACB_ADAPTER_TYPE_E) { + switch (acb->adapter_type) { + case ACB_ADAPTER_TYPE_B: { + struct MessageUnit_B *reg = acb->pmuB; + uint32_t i; + for (i = 0; i < ARCMSR_MAX_HBB_POSTQUEUE; i++) { + reg->post_qbuffer[i] = 0; + reg->done_qbuffer[i] = 0; + } + reg->postq_index = 0; + reg->doneq_index = 0; + break; + } + case ACB_ADAPTER_TYPE_E: writel(0, &acb->pmuE->host_int_status); writel(ARCMSR_HBEMU_DOORBELL_SYNC, &acb->pmuE->iobound_doorbell); acb->in_doorbell = 0; acb->out_doorbell = 0; acb->doneq_index = 0; + break; } arcmsr_iop_init(acb); arcmsr_init_get_devmap_timer(acb); @@ -1351,10 +1408,12 @@ static void arcmsr_drain_donequeue(struct AdapterControlBlock *acb, struct Comma static void arcmsr_done4abort_postqueue(struct AdapterControlBlock *acb) { int i = 0; - uint32_t flag_ccb, ccb_cdb_phy; + uint32_t flag_ccb; struct ARCMSR_CDB *pARCMSR_CDB; bool error; struct CommandControlBlock *pCCB; + unsigned long ccb_cdb_phy, cdb_phy_hipart; + switch (acb->adapter_type) { case ACB_ADAPTER_TYPE_A: { @@ -1366,7 +1425,10 @@ static void arcmsr_done4abort_postqueue(struct AdapterControlBlock *acb) writel(outbound_intstatus, ®->outbound_intstatus);/*clear interrupt*/ while(((flag_ccb = readl(®->outbound_queueport)) != 0xFFFFFFFF) && (i++ < acb->maxOutstanding)) { - pARCMSR_CDB = (struct ARCMSR_CDB *)(acb->vir2phy_offset + (flag_ccb << 5));/*frame must be 32 bytes aligned*/ + ccb_cdb_phy = (flag_ccb << 5) & 0xffffffff; + if (acb->cdb_phyadd_hipart) + ccb_cdb_phy = ccb_cdb_phy | acb->cdb_phyadd_hipart; + pARCMSR_CDB = (struct ARCMSR_CDB *)(acb->vir2phy_offset + ccb_cdb_phy); pCCB = container_of(pARCMSR_CDB, struct CommandControlBlock, arcmsr_cdb); error = (flag_ccb & ARCMSR_CCBREPLY_FLAG_ERROR_MODE0) ? true : false; arcmsr_drain_donequeue(acb, pCCB, error); @@ -1382,7 +1444,10 @@ static void arcmsr_done4abort_postqueue(struct AdapterControlBlock *acb) flag_ccb = reg->done_qbuffer[i]; if (flag_ccb != 0) { reg->done_qbuffer[i] = 0; - pARCMSR_CDB = (struct ARCMSR_CDB *)(acb->vir2phy_offset+(flag_ccb << 5));/*frame must be 32 bytes aligned*/ + ccb_cdb_phy = (flag_ccb << 5) & 0xffffffff; + if (acb->cdb_phyadd_hipart) + ccb_cdb_phy = ccb_cdb_phy | acb->cdb_phyadd_hipart; + pARCMSR_CDB = (struct ARCMSR_CDB *)(acb->vir2phy_offset + ccb_cdb_phy); pCCB = container_of(pARCMSR_CDB, struct CommandControlBlock, arcmsr_cdb); error = (flag_ccb & ARCMSR_CCBREPLY_FLAG_ERROR_MODE0) ? true : false; arcmsr_drain_donequeue(acb, pCCB, error); @@ -1399,7 +1464,9 @@ static void arcmsr_done4abort_postqueue(struct AdapterControlBlock *acb) /*need to do*/ flag_ccb = readl(®->outbound_queueport_low); ccb_cdb_phy = (flag_ccb & 0xFFFFFFF0); - pARCMSR_CDB = (struct ARCMSR_CDB *)(acb->vir2phy_offset+ccb_cdb_phy);/*frame must be 32 bytes aligned*/ + if (acb->cdb_phyadd_hipart) + ccb_cdb_phy = ccb_cdb_phy | acb->cdb_phyadd_hipart; + pARCMSR_CDB = (struct ARCMSR_CDB *)(acb->vir2phy_offset + ccb_cdb_phy); pCCB = container_of(pARCMSR_CDB, struct CommandControlBlock, arcmsr_cdb); error = (flag_ccb & ARCMSR_CCBREPLY_FLAG_ERROR_MODE1) ? true : false; arcmsr_drain_donequeue(acb, pCCB, error); @@ -1427,9 +1494,13 @@ static void arcmsr_done4abort_postqueue(struct AdapterControlBlock *acb) ((toggle ^ 0x4000) + 1); doneq_index = pmu->doneq_index; spin_unlock_irqrestore(&acb->doneq_lock, flags); + cdb_phy_hipart = pmu->done_qbuffer[doneq_index & + 0xFFF].addressHigh; addressLow = pmu->done_qbuffer[doneq_index & 0xFFF].addressLow; ccb_cdb_phy = (addressLow & 0xFFFFFFF0); + if (acb->cdb_phyadd_hipart) + ccb_cdb_phy = ccb_cdb_phy | acb->cdb_phyadd_hipart; pARCMSR_CDB = (struct ARCMSR_CDB *) (acb->vir2phy_offset + ccb_cdb_phy); pCCB = container_of(pARCMSR_CDB, @@ -1506,7 +1577,6 @@ static void arcmsr_free_pcidev(struct AdapterControlBlock *acb) pdev = acb->pdev; arcmsr_free_irq(pdev, acb); arcmsr_free_ccb_pool(acb); - arcmsr_free_mu(acb); arcmsr_unmap_pciregion(acb); pci_release_regions(pdev); scsi_host_put(host); @@ -1564,7 +1634,6 @@ static void arcmsr_remove(struct pci_dev *pdev) } arcmsr_free_irq(pdev, acb); arcmsr_free_ccb_pool(acb); - arcmsr_free_mu(acb); arcmsr_unmap_pciregion(acb); pci_release_regions(pdev); scsi_host_put(host); @@ -1749,12 +1818,8 @@ static void arcmsr_post_ccb(struct AdapterControlBlock *acb, struct CommandContr arc_cdb_size = (ccb->arc_cdb_size > 0x300) ? 0x300 : ccb->arc_cdb_size; ccb_post_stamp = (cdb_phyaddr | ((arc_cdb_size - 1) >> 6) | 1); - if (acb->cdb_phyaddr_hi32) { - writel(acb->cdb_phyaddr_hi32, &phbcmu->inbound_queueport_high); - writel(ccb_post_stamp, &phbcmu->inbound_queueport_low); - } else { - writel(ccb_post_stamp, &phbcmu->inbound_queueport_low); - } + writel(upper_32_bits(ccb->cdb_phyaddr), &phbcmu->inbound_queueport_high); + writel(ccb_post_stamp, &phbcmu->inbound_queueport_low); } break; case ACB_ADAPTER_TYPE_D: { @@ -1767,8 +1832,8 @@ static void arcmsr_post_ccb(struct AdapterControlBlock *acb, struct CommandContr spin_lock_irqsave(&acb->postq_lock, flags); postq_index = pmu->postq_index; pinbound_srb = (struct InBound_SRB *)&(pmu->post_qbuffer[postq_index & 0xFF]); - pinbound_srb->addressHigh = dma_addr_hi32(cdb_phyaddr); - pinbound_srb->addressLow = dma_addr_lo32(cdb_phyaddr); + pinbound_srb->addressHigh = upper_32_bits(ccb->cdb_phyaddr); + pinbound_srb->addressLow = cdb_phyaddr; pinbound_srb->length = ccb->arc_cdb_size >> 2; arcmsr_cdb->msgContext = dma_addr_lo32(cdb_phyaddr); toggle = postq_index & 0x4000; @@ -2304,8 +2369,13 @@ static void arcmsr_hbaA_postqueue_isr(struct AdapterControlBlock *acb) struct ARCMSR_CDB *pARCMSR_CDB; struct CommandControlBlock *pCCB; bool error; + unsigned long cdb_phy_addr; + while ((flag_ccb = readl(®->outbound_queueport)) != 0xFFFFFFFF) { - pARCMSR_CDB = (struct ARCMSR_CDB *)(acb->vir2phy_offset + (flag_ccb << 5));/*frame must be 32 bytes aligned*/ + cdb_phy_addr = (flag_ccb << 5) & 0xffffffff; + if (acb->cdb_phyadd_hipart) + cdb_phy_addr = cdb_phy_addr | acb->cdb_phyadd_hipart; + pARCMSR_CDB = (struct ARCMSR_CDB *)(acb->vir2phy_offset + cdb_phy_addr); pCCB = container_of(pARCMSR_CDB, struct CommandControlBlock, arcmsr_cdb); error = (flag_ccb & ARCMSR_CCBREPLY_FLAG_ERROR_MODE0) ? true : false; arcmsr_drain_donequeue(acb, pCCB, error); @@ -2319,13 +2389,18 @@ static void arcmsr_hbaB_postqueue_isr(struct AdapterControlBlock *acb) struct ARCMSR_CDB *pARCMSR_CDB; struct CommandControlBlock *pCCB; bool error; + unsigned long cdb_phy_addr; + index = reg->doneq_index; while ((flag_ccb = reg->done_qbuffer[index]) != 0) { - reg->done_qbuffer[index] = 0; - pARCMSR_CDB = (struct ARCMSR_CDB *)(acb->vir2phy_offset+(flag_ccb << 5));/*frame must be 32 bytes aligned*/ + cdb_phy_addr = (flag_ccb << 5) & 0xffffffff; + if (acb->cdb_phyadd_hipart) + cdb_phy_addr = cdb_phy_addr | acb->cdb_phyadd_hipart; + pARCMSR_CDB = (struct ARCMSR_CDB *)(acb->vir2phy_offset + cdb_phy_addr); pCCB = container_of(pARCMSR_CDB, struct CommandControlBlock, arcmsr_cdb); error = (flag_ccb & ARCMSR_CCBREPLY_FLAG_ERROR_MODE0) ? true : false; arcmsr_drain_donequeue(acb, pCCB, error); + reg->done_qbuffer[index] = 0; index++; index %= ARCMSR_MAX_HBB_POSTQUEUE; reg->doneq_index = index; @@ -2337,7 +2412,8 @@ static void arcmsr_hbaC_postqueue_isr(struct AdapterControlBlock *acb) struct MessageUnit_C __iomem *phbcmu; struct ARCMSR_CDB *arcmsr_cdb; struct CommandControlBlock *ccb; - uint32_t flag_ccb, ccb_cdb_phy, throttling = 0; + uint32_t flag_ccb, throttling = 0; + unsigned long ccb_cdb_phy; int error; phbcmu = acb->pmuC; @@ -2347,6 +2423,8 @@ static void arcmsr_hbaC_postqueue_isr(struct AdapterControlBlock *acb) while ((flag_ccb = readl(&phbcmu->outbound_queueport_low)) != 0xFFFFFFFF) { ccb_cdb_phy = (flag_ccb & 0xFFFFFFF0); + if (acb->cdb_phyadd_hipart) + ccb_cdb_phy = ccb_cdb_phy | acb->cdb_phyadd_hipart; arcmsr_cdb = (struct ARCMSR_CDB *)(acb->vir2phy_offset + ccb_cdb_phy); ccb = container_of(arcmsr_cdb, struct CommandControlBlock, @@ -2367,12 +2445,12 @@ static void arcmsr_hbaC_postqueue_isr(struct AdapterControlBlock *acb) static void arcmsr_hbaD_postqueue_isr(struct AdapterControlBlock *acb) { u32 outbound_write_pointer, doneq_index, index_stripped, toggle; - uint32_t addressLow, ccb_cdb_phy; + uint32_t addressLow; int error; struct MessageUnit_D *pmu; struct ARCMSR_CDB *arcmsr_cdb; struct CommandControlBlock *ccb; - unsigned long flags; + unsigned long flags, ccb_cdb_phy, cdb_phy_hipart; spin_lock_irqsave(&acb->doneq_lock, flags); pmu = acb->pmuD; @@ -2386,9 +2464,13 @@ static void arcmsr_hbaD_postqueue_isr(struct AdapterControlBlock *acb) pmu->doneq_index = index_stripped ? (index_stripped | toggle) : ((toggle ^ 0x4000) + 1); doneq_index = pmu->doneq_index; + cdb_phy_hipart = pmu->done_qbuffer[doneq_index & + 0xFFF].addressHigh; addressLow = pmu->done_qbuffer[doneq_index & 0xFFF].addressLow; ccb_cdb_phy = (addressLow & 0xFFFFFFF0); + if (acb->cdb_phyadd_hipart) + ccb_cdb_phy = ccb_cdb_phy | acb->cdb_phyadd_hipart; arcmsr_cdb = (struct ARCMSR_CDB *)(acb->vir2phy_offset + ccb_cdb_phy); ccb = container_of(arcmsr_cdb, @@ -3229,7 +3311,9 @@ static int arcmsr_hbaA_polling_ccbdone(struct AdapterControlBlock *acb, uint32_t flag_ccb, outbound_intstatus, poll_ccb_done = 0, poll_count = 0; int rtn; bool error; - polling_hba_ccb_retry: + unsigned long ccb_cdb_phy; + +polling_hba_ccb_retry: poll_count++; outbound_intstatus = readl(®->outbound_intstatus) & acb->outbound_int_enable; writel(outbound_intstatus, ®->outbound_intstatus);/*clear interrupt*/ @@ -3247,7 +3331,10 @@ static int arcmsr_hbaA_polling_ccbdone(struct AdapterControlBlock *acb, goto polling_hba_ccb_retry; } } - arcmsr_cdb = (struct ARCMSR_CDB *)(acb->vir2phy_offset + (flag_ccb << 5)); + ccb_cdb_phy = (flag_ccb << 5) & 0xffffffff; + if (acb->cdb_phyadd_hipart) + ccb_cdb_phy = ccb_cdb_phy | acb->cdb_phyadd_hipart; + arcmsr_cdb = (struct ARCMSR_CDB *)(acb->vir2phy_offset + ccb_cdb_phy); ccb = container_of(arcmsr_cdb, struct CommandControlBlock, arcmsr_cdb); poll_ccb_done |= (ccb == poll_ccb) ? 1 : 0; if ((ccb->acb != acb) || (ccb->startdone != ARCMSR_CCB_START)) { @@ -3285,8 +3372,9 @@ static int arcmsr_hbaB_polling_ccbdone(struct AdapterControlBlock *acb, uint32_t flag_ccb, poll_ccb_done = 0, poll_count = 0; int index, rtn; bool error; - polling_hbb_ccb_retry: + unsigned long ccb_cdb_phy; +polling_hbb_ccb_retry: poll_count++; /* clear doorbell interrupt */ writel(ARCMSR_DOORBELL_INT_CLEAR_PATTERN, reg->iop2drv_doorbell); @@ -3312,7 +3400,10 @@ static int arcmsr_hbaB_polling_ccbdone(struct AdapterControlBlock *acb, index %= ARCMSR_MAX_HBB_POSTQUEUE; reg->doneq_index = index; /* check if command done with no error*/ - arcmsr_cdb = (struct ARCMSR_CDB *)(acb->vir2phy_offset + (flag_ccb << 5)); + ccb_cdb_phy = (flag_ccb << 5) & 0xffffffff; + if (acb->cdb_phyadd_hipart) + ccb_cdb_phy = ccb_cdb_phy | acb->cdb_phyadd_hipart; + arcmsr_cdb = (struct ARCMSR_CDB *)(acb->vir2phy_offset + ccb_cdb_phy); ccb = container_of(arcmsr_cdb, struct CommandControlBlock, arcmsr_cdb); poll_ccb_done |= (ccb == poll_ccb) ? 1 : 0; if ((ccb->acb != acb) || (ccb->startdone != ARCMSR_CCB_START)) { @@ -3345,12 +3436,14 @@ static int arcmsr_hbaC_polling_ccbdone(struct AdapterControlBlock *acb, struct CommandControlBlock *poll_ccb) { struct MessageUnit_C __iomem *reg = acb->pmuC; - uint32_t flag_ccb, ccb_cdb_phy; + uint32_t flag_ccb; struct ARCMSR_CDB *arcmsr_cdb; bool error; struct CommandControlBlock *pCCB; uint32_t poll_ccb_done = 0, poll_count = 0; int rtn; + unsigned long ccb_cdb_phy; + polling_hbc_ccb_retry: poll_count++; while (1) { @@ -3369,7 +3462,9 @@ polling_hbc_ccb_retry: } flag_ccb = readl(®->outbound_queueport_low); ccb_cdb_phy = (flag_ccb & 0xFFFFFFF0); - arcmsr_cdb = (struct ARCMSR_CDB *)(acb->vir2phy_offset + ccb_cdb_phy);/*frame must be 32 bytes aligned*/ + if (acb->cdb_phyadd_hipart) + ccb_cdb_phy = ccb_cdb_phy | acb->cdb_phyadd_hipart; + arcmsr_cdb = (struct ARCMSR_CDB *)(acb->vir2phy_offset + ccb_cdb_phy); pCCB = container_of(arcmsr_cdb, struct CommandControlBlock, arcmsr_cdb); poll_ccb_done |= (pCCB == poll_ccb) ? 1 : 0; /* check ifcommand done with no error*/ @@ -3403,9 +3498,9 @@ static int arcmsr_hbaD_polling_ccbdone(struct AdapterControlBlock *acb, struct CommandControlBlock *poll_ccb) { bool error; - uint32_t poll_ccb_done = 0, poll_count = 0, flag_ccb, ccb_cdb_phy; + uint32_t poll_ccb_done = 0, poll_count = 0, flag_ccb; int rtn, doneq_index, index_stripped, outbound_write_pointer, toggle; - unsigned long flags; + unsigned long flags, ccb_cdb_phy, cdb_phy_hipart; struct ARCMSR_CDB *arcmsr_cdb; struct CommandControlBlock *pCCB; struct MessageUnit_D *pmu = acb->pmuD; @@ -3437,8 +3532,12 @@ polling_hbaD_ccb_retry: ((toggle ^ 0x4000) + 1); doneq_index = pmu->doneq_index; spin_unlock_irqrestore(&acb->doneq_lock, flags); + cdb_phy_hipart = pmu->done_qbuffer[doneq_index & + 0xFFF].addressHigh; flag_ccb = pmu->done_qbuffer[doneq_index & 0xFFF].addressLow; ccb_cdb_phy = (flag_ccb & 0xFFFFFFF0); + if (acb->cdb_phyadd_hipart) + ccb_cdb_phy = ccb_cdb_phy | acb->cdb_phyadd_hipart; arcmsr_cdb = (struct ARCMSR_CDB *)(acb->vir2phy_offset + ccb_cdb_phy); pCCB = container_of(arcmsr_cdb, struct CommandControlBlock, @@ -3680,6 +3779,7 @@ static int arcmsr_iop_confirm(struct AdapterControlBlock *acb) cdb_phyaddr = lower_32_bits(dma_coherent_handle); cdb_phyaddr_hi32 = upper_32_bits(dma_coherent_handle); acb->cdb_phyaddr_hi32 = cdb_phyaddr_hi32; + acb->cdb_phyadd_hipart = ((uint64_t)cdb_phyaddr_hi32) << 32; /* *********************************************************************** ** if adapter type B, set window of "post command Q" @@ -3744,7 +3844,6 @@ static int arcmsr_iop_confirm(struct AdapterControlBlock *acb) } break; case ACB_ADAPTER_TYPE_C: { - if (cdb_phyaddr_hi32 != 0) { struct MessageUnit_C __iomem *reg = acb->pmuC; printk(KERN_NOTICE "arcmsr%d: cdb_phyaddr_hi32=0x%x\n", @@ -3759,7 +3858,6 @@ static int arcmsr_iop_confirm(struct AdapterControlBlock *acb) return 1; } } - } break; case ACB_ADAPTER_TYPE_D: { uint32_t __iomem *rwbuffer; @@ -3793,7 +3891,7 @@ static int arcmsr_iop_confirm(struct AdapterControlBlock *acb) cdb_phyaddr_hi32 = (uint32_t)((dma_coherent_handle >> 16) >> 16); writel(cdb_phyaddr, ®->msgcode_rwbuffer[5]); writel(cdb_phyaddr_hi32, ®->msgcode_rwbuffer[6]); - writel(acb->roundup_ccbsize, ®->msgcode_rwbuffer[7]); + writel(acb->ioqueue_size, ®->msgcode_rwbuffer[7]); writel(ARCMSR_INBOUND_MESG0_SET_CONFIG, ®->inbound_msgaddr0); acb->out_doorbell ^= ARCMSR_HBEMU_DRV2IOP_MESSAGE_CMD_DONE; writel(acb->out_doorbell, ®->iobound_doorbell); diff --git a/drivers/scsi/bfa/bfa_fcs_lport.c b/drivers/scsi/bfa/bfa_fcs_lport.c index b4f2c1d8742e..646f09f66443 100644 --- a/drivers/scsi/bfa/bfa_fcs_lport.c +++ b/drivers/scsi/bfa/bfa_fcs_lport.c @@ -6430,9 +6430,7 @@ bfa_fcs_vport_sm_logo_for_stop(struct bfa_fcs_vport_s *vport, switch (event) { case BFA_FCS_VPORT_SM_OFFLINE: bfa_sm_send_event(vport->lps, BFA_LPS_SM_OFFLINE); - /* - * !!! fall through !!! - */ + /* fall through */ case BFA_FCS_VPORT_SM_RSP_OK: case BFA_FCS_VPORT_SM_RSP_ERROR: @@ -6458,9 +6456,7 @@ bfa_fcs_vport_sm_logo(struct bfa_fcs_vport_s *vport, switch (event) { case BFA_FCS_VPORT_SM_OFFLINE: bfa_sm_send_event(vport->lps, BFA_LPS_SM_OFFLINE); - /* - * !!! fall through !!! - */ + /* fall through */ case BFA_FCS_VPORT_SM_RSP_OK: case BFA_FCS_VPORT_SM_RSP_ERROR: diff --git a/drivers/scsi/bfa/bfa_fcs_rport.c b/drivers/scsi/bfa/bfa_fcs_rport.c index de50349a39ce..1e400f2aaece 100644 --- a/drivers/scsi/bfa/bfa_fcs_rport.c +++ b/drivers/scsi/bfa/bfa_fcs_rport.c @@ -427,17 +427,13 @@ bfa_fcs_rport_sm_plogi(struct bfa_fcs_rport_s *rport, enum rport_event event) case RPSM_EVENT_LOGO_RCVD: bfa_fcs_rport_send_logo_acc(rport); - /* - * !! fall through !! - */ + /* fall through */ case RPSM_EVENT_PRLO_RCVD: if (rport->prlo == BFA_TRUE) bfa_fcs_rport_send_prlo_acc(rport); bfa_fcxp_discard(rport->fcxp); - /* - * !! fall through !! - */ + /* fall through */ case RPSM_EVENT_FAILED: if (rport->plogi_retries < BFA_FCS_RPORT_MAX_RETRIES) { rport->plogi_retries++; @@ -868,9 +864,7 @@ bfa_fcs_rport_sm_adisc_online(struct bfa_fcs_rport_s *rport, * At least go offline when a PLOGI is received. */ bfa_fcxp_discard(rport->fcxp); - /* - * !!! fall through !!! - */ + /* fall through */ case RPSM_EVENT_FAILED: case RPSM_EVENT_ADDRESS_CHANGE: @@ -1056,6 +1050,7 @@ bfa_fcs_rport_sm_fc4_logosend(struct bfa_fcs_rport_s *rport, case RPSM_EVENT_LOGO_RCVD: bfa_fcs_rport_send_logo_acc(rport); + /* fall through */ case RPSM_EVENT_PRLO_RCVD: if (rport->prlo == BFA_TRUE) bfa_fcs_rport_send_prlo_acc(rport); @@ -1144,9 +1139,7 @@ bfa_fcs_rport_sm_hcb_offline(struct bfa_fcs_rport_s *rport, bfa_fcs_rport_send_plogiacc(rport, NULL); break; } - /* - * !! fall through !! - */ + /* fall through */ case RPSM_EVENT_ADDRESS_CHANGE: if (!bfa_fcs_lport_is_online(rport->port)) { @@ -1303,6 +1296,7 @@ bfa_fcs_rport_sm_hcb_logosend(struct bfa_fcs_rport_s *rport, case RPSM_EVENT_LOGO_RCVD: bfa_fcs_rport_send_logo_acc(rport); + /* fall through */ case RPSM_EVENT_PRLO_RCVD: if (rport->prlo == BFA_TRUE) bfa_fcs_rport_send_prlo_acc(rport); @@ -1346,6 +1340,7 @@ bfa_fcs_rport_sm_logo_sending(struct bfa_fcs_rport_s *rport, case RPSM_EVENT_LOGO_RCVD: bfa_fcs_rport_send_logo_acc(rport); + /* fall through */ case RPSM_EVENT_PRLO_RCVD: if (rport->prlo == BFA_TRUE) bfa_fcs_rport_send_prlo_acc(rport); diff --git a/drivers/scsi/bfa/bfa_ioc.c b/drivers/scsi/bfa/bfa_ioc.c index 9631877aba4f..79a55c3615be 100644 --- a/drivers/scsi/bfa/bfa_ioc.c +++ b/drivers/scsi/bfa/bfa_ioc.c @@ -978,9 +978,7 @@ bfa_iocpf_sm_enabling(struct bfa_iocpf_s *iocpf, enum iocpf_event event) case IOCPF_E_INITFAIL: bfa_iocpf_timer_stop(ioc); - /* - * !!! fall through !!! - */ + /* fall through */ case IOCPF_E_TIMEOUT: writel(1, ioc->ioc_regs.ioc_sem_reg); @@ -1056,9 +1054,7 @@ bfa_iocpf_sm_disabling(struct bfa_iocpf_s *iocpf, enum iocpf_event event) case IOCPF_E_FAIL: bfa_iocpf_timer_stop(ioc); - /* - * !!! fall through !!! - */ + /* fall through */ case IOCPF_E_TIMEOUT: bfa_ioc_set_cur_ioc_fwstate(ioc, BFI_IOC_FAIL); @@ -6007,6 +6003,7 @@ bfa_dconf_sm_final_sync(struct bfa_dconf_mod_s *dconf, case BFA_DCONF_SM_IOCDISABLE: case BFA_DCONF_SM_FLASH_COMP: bfa_timer_stop(&dconf->timer); + /* fall through */ case BFA_DCONF_SM_TIMEOUT: bfa_sm_set_state(dconf, bfa_dconf_sm_uninit); bfa_fsm_send_event(&dconf->bfa->iocfc, IOCFC_E_DCONF_DONE); diff --git a/drivers/scsi/bfa/bfad_debugfs.c b/drivers/scsi/bfa/bfad_debugfs.c index 349cfe7d055e..bfcd87c0dc74 100644 --- a/drivers/scsi/bfa/bfad_debugfs.c +++ b/drivers/scsi/bfa/bfad_debugfs.c @@ -460,11 +460,6 @@ bfad_debugfs_init(struct bfad_port_s *port) if (!bfa_debugfs_root) { bfa_debugfs_root = debugfs_create_dir("bfa", NULL); atomic_set(&bfa_debugfs_port_count, 0); - if (!bfa_debugfs_root) { - printk(KERN_WARNING - "BFA debugfs root dir creation failed\n"); - goto err; - } } /* Setup the pci_dev debugfs directory for the port */ @@ -472,12 +467,6 @@ bfad_debugfs_init(struct bfad_port_s *port) if (!port->port_debugfs_root) { port->port_debugfs_root = debugfs_create_dir(name, bfa_debugfs_root); - if (!port->port_debugfs_root) { - printk(KERN_WARNING - "bfa %s: debugfs root creation failed\n", - bfad->pci_name); - goto err; - } atomic_inc(&bfa_debugfs_port_count); @@ -489,16 +478,9 @@ bfad_debugfs_init(struct bfad_port_s *port) port->port_debugfs_root, port, file->fops); - if (!bfad->bfad_dentry_files[i]) { - printk(KERN_WARNING - "bfa %s: debugfs %s creation failed\n", - bfad->pci_name, file->name); - goto err; - } } } -err: return; } diff --git a/drivers/scsi/bnx2fc/bnx2fc_fcoe.c b/drivers/scsi/bnx2fc/bnx2fc_fcoe.c index 2e4e7159ebf9..a75e74ad1698 100644 --- a/drivers/scsi/bnx2fc/bnx2fc_fcoe.c +++ b/drivers/scsi/bnx2fc/bnx2fc_fcoe.c @@ -1438,7 +1438,7 @@ bind_err: static struct bnx2fc_interface * bnx2fc_interface_create(struct bnx2fc_hba *hba, struct net_device *netdev, - enum fip_state fip_mode) + enum fip_mode fip_mode) { struct fcoe_ctlr_device *ctlr_dev; struct bnx2fc_interface *interface; diff --git a/drivers/scsi/bnx2i/bnx2i_iscsi.c b/drivers/scsi/bnx2i/bnx2i_iscsi.c index 69c75426c5eb..c5fa5f3b00e9 100644 --- a/drivers/scsi/bnx2i/bnx2i_iscsi.c +++ b/drivers/scsi/bnx2i/bnx2i_iscsi.c @@ -577,7 +577,7 @@ static void bnx2i_free_mp_bdt(struct bnx2i_hba *hba) hba->dummy_buffer, hba->dummy_buf_dma); hba->dummy_buffer = NULL; } - return; + return; } /** diff --git a/drivers/scsi/csiostor/csio_attr.c b/drivers/scsi/csiostor/csio_attr.c index 9bd2bd8dc2be..200e50089711 100644 --- a/drivers/scsi/csiostor/csio_attr.c +++ b/drivers/scsi/csiostor/csio_attr.c @@ -497,7 +497,6 @@ out: static int csio_fcoe_free_vnp(struct csio_hw *hw, struct csio_lnode *ln) { - struct csio_lnode *pln; struct csio_mb *mbp; struct fw_fcoe_vnp_cmd *rsp; int ret = 0; @@ -514,8 +513,6 @@ csio_fcoe_free_vnp(struct csio_hw *hw, struct csio_lnode *ln) goto out; } - pln = ln->pln; - csio_fcoe_vnp_free_init_mb(ln, mbp, CSIO_MB_DEFAULT_TMO, ln->fcf_flowid, ln->vnp_flowid, NULL); diff --git a/drivers/scsi/csiostor/csio_init.c b/drivers/scsi/csiostor/csio_init.c index 616b25bf7941..a6dd704d7f2d 100644 --- a/drivers/scsi/csiostor/csio_init.c +++ b/drivers/scsi/csiostor/csio_init.c @@ -167,14 +167,10 @@ csio_dfs_destroy(struct csio_hw *hw) * csio_dfs_init - Debug filesystem initialization for the module. * */ -static int +static void csio_dfs_init(void) { csio_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL); - if (!csio_debugfs_root) - pr_warn("Could not create debugfs entry, continuing\n"); - - return 0; } /* diff --git a/drivers/scsi/csiostor/csio_scsi.c b/drivers/scsi/csiostor/csio_scsi.c index bc5547a62c00..462560b2855e 100644 --- a/drivers/scsi/csiostor/csio_scsi.c +++ b/drivers/scsi/csiostor/csio_scsi.c @@ -1984,15 +1984,15 @@ inval_scmnd: /* FW successfully aborted the request */ if (host_byte(cmnd->result) == DID_REQUEUE) { csio_info(hw, - "Aborted SCSI command to (%d:%llu) serial#:0x%lx\n", + "Aborted SCSI command to (%d:%llu) tag %u\n", cmnd->device->id, cmnd->device->lun, - cmnd->serial_number); + cmnd->request->tag); return SUCCESS; } else { csio_info(hw, - "Failed to abort SCSI command, (%d:%llu) serial#:0x%lx\n", + "Failed to abort SCSI command, (%d:%llu) tag %u\n", cmnd->device->id, cmnd->device->lun, - cmnd->serial_number); + cmnd->request->tag); return FAILED; } } diff --git a/drivers/scsi/cxgbi/Makefile b/drivers/scsi/cxgbi/Makefile index a73781ac1800..f78c9cc460a2 100644 --- a/drivers/scsi/cxgbi/Makefile +++ b/drivers/scsi/cxgbi/Makefile @@ -1,4 +1,4 @@ -ccflags-y += -Idrivers/net/ethernet/chelsio/libcxgb +ccflags-y += -I $(srctree)/drivers/net/ethernet/chelsio/libcxgb obj-$(CONFIG_SCSI_CXGB3_ISCSI) += libcxgbi.o cxgb3i/ obj-$(CONFIG_SCSI_CXGB4_ISCSI) += libcxgbi.o cxgb4i/ diff --git a/drivers/scsi/cxgbi/cxgb4i/cxgb4i.c b/drivers/scsi/cxgbi/cxgb4i/cxgb4i.c index d26f50af00ea..d44914e5e415 100644 --- a/drivers/scsi/cxgbi/cxgb4i/cxgb4i.c +++ b/drivers/scsi/cxgbi/cxgb4i/cxgb4i.c @@ -1210,7 +1210,8 @@ static void do_rx_iscsi_hdr(struct cxgbi_device *cdev, struct sk_buff *skb) csk->skb_ulp_lhdr = skb; cxgbi_skcb_set_flag(skb, SKCBF_RX_HDR); - if (cxgbi_skcb_tcp_seq(skb) != csk->rcv_nxt) { + if ((CHELSIO_CHIP_VERSION(lldi->adapter_type) <= CHELSIO_T5) && + (cxgbi_skcb_tcp_seq(skb) != csk->rcv_nxt)) { pr_info("tid %u, CPL_ISCSI_HDR, bad seq, 0x%x/0x%x.\n", csk->tid, cxgbi_skcb_tcp_seq(skb), csk->rcv_nxt); @@ -2134,8 +2135,7 @@ static void *t4_uld_add(const struct cxgb4_lld_info *lldi) cdev->itp = &cxgb4i_iscsi_transport; cdev->owner = THIS_MODULE; - cdev->pfvf = FW_VIID_PFN_G(cxgb4_port_viid(lldi->ports[0])) - << FW_VIID_PFN_S; + cdev->pfvf = FW_PFVF_CMD_PFN_V(lldi->pf); pr_info("cdev 0x%p,%s, pfvf %u.\n", cdev, lldi->ports[0]->name, cdev->pfvf); diff --git a/drivers/scsi/cxgbi/libcxgbi.c b/drivers/scsi/cxgbi/libcxgbi.c index 245742557c03..006372b3fba2 100644 --- a/drivers/scsi/cxgbi/libcxgbi.c +++ b/drivers/scsi/cxgbi/libcxgbi.c @@ -1212,7 +1212,7 @@ scmd_get_params(struct scsi_cmnd *sc, struct scatterlist **sgl, unsigned int *sgcnt, unsigned int *dlen, unsigned int prot) { - struct scsi_data_buffer *sdb = prot ? scsi_prot(sc) : scsi_out(sc); + struct scsi_data_buffer *sdb = prot ? scsi_prot(sc) : &sc->sdb; *sgl = sdb->table.sgl; *sgcnt = sdb->table.nents; @@ -1428,8 +1428,7 @@ static void task_release_itt(struct iscsi_task *task, itt_t hdr_itt) log_debug(1 << CXGBI_DBG_DDP, "cdev 0x%p, task 0x%p, release tag 0x%x.\n", cdev, task, tag); - if (sc && - (scsi_bidi_cmnd(sc) || sc->sc_data_direction == DMA_FROM_DEVICE) && + if (sc && sc->sc_data_direction == DMA_FROM_DEVICE && cxgbi_ppm_is_ddp_tag(ppm, tag)) { struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task); struct cxgbi_task_tag_info *ttinfo = &tdata->ttinfo; @@ -1461,9 +1460,7 @@ static int task_reserve_itt(struct iscsi_task *task, itt_t *hdr_itt) u32 tag = 0; int err = -EINVAL; - if (sc && - (scsi_bidi_cmnd(sc) || sc->sc_data_direction == DMA_FROM_DEVICE) - ) { + if (sc && sc->sc_data_direction == DMA_FROM_DEVICE) { struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task); struct cxgbi_task_tag_info *ttinfo = &tdata->ttinfo; @@ -1897,7 +1894,7 @@ int cxgbi_conn_alloc_pdu(struct iscsi_task *task, u8 opcode) if (SKB_MAX_HEAD(cdev->skb_tx_rsvd) > (512 * MAX_SKB_FRAGS) && (opcode == ISCSI_OP_SCSI_DATA_OUT || (opcode == ISCSI_OP_SCSI_CMD && - (scsi_bidi_cmnd(sc) || sc->sc_data_direction == DMA_TO_DEVICE)))) + sc->sc_data_direction == DMA_TO_DEVICE))) /* data could goes into skb head */ headroom += min_t(unsigned int, SKB_MAX_HEAD(cdev->skb_tx_rsvd), @@ -1972,7 +1969,7 @@ int cxgbi_conn_init_pdu(struct iscsi_task *task, unsigned int offset, return 0; if (task->sc) { - struct scsi_data_buffer *sdb = scsi_out(task->sc); + struct scsi_data_buffer *sdb = &task->sc->sdb; struct scatterlist *sg = NULL; int err; diff --git a/drivers/scsi/cxlflash/common.h b/drivers/scsi/cxlflash/common.h index 8908a20065c8..4d90106fcb37 100644 --- a/drivers/scsi/cxlflash/common.h +++ b/drivers/scsi/cxlflash/common.h @@ -334,7 +334,8 @@ int cxlflash_afu_sync(struct afu *afu, ctx_hndl_t c, res_hndl_t r, u8 mode); void cxlflash_list_init(void); void cxlflash_term_global_luns(void); void cxlflash_free_errpage(void); -int cxlflash_ioctl(struct scsi_device *sdev, int cmd, void __user *arg); +int cxlflash_ioctl(struct scsi_device *sdev, unsigned int cmd, + void __user *arg); void cxlflash_stop_term_user_contexts(struct cxlflash_cfg *cfg); int cxlflash_mark_contexts_error(struct cxlflash_cfg *cfg); void cxlflash_term_local_luns(struct cxlflash_cfg *cfg); diff --git a/drivers/scsi/cxlflash/main.c b/drivers/scsi/cxlflash/main.c index c8bad2c093b8..7096810fd222 100644 --- a/drivers/scsi/cxlflash/main.c +++ b/drivers/scsi/cxlflash/main.c @@ -3282,7 +3282,7 @@ static int cxlflash_chr_open(struct inode *inode, struct file *file) * * Return: A string identifying the decoded host ioctl. */ -static char *decode_hioctl(int cmd) +static char *decode_hioctl(unsigned int cmd) { switch (cmd) { case HT_CXLFLASH_LUN_PROVISION: diff --git a/drivers/scsi/cxlflash/superpipe.c b/drivers/scsi/cxlflash/superpipe.c index acac6152f50b..1a94a469051e 100644 --- a/drivers/scsi/cxlflash/superpipe.c +++ b/drivers/scsi/cxlflash/superpipe.c @@ -1924,7 +1924,7 @@ out: * * Return: A string identifying the decoded ioctl. */ -static char *decode_ioctl(int cmd) +static char *decode_ioctl(unsigned int cmd) { switch (cmd) { case DK_CXLFLASH_ATTACH: @@ -2051,7 +2051,7 @@ err1: * * Return: 0 on success, -errno on failure */ -static int ioctl_common(struct scsi_device *sdev, int cmd) +static int ioctl_common(struct scsi_device *sdev, unsigned int cmd) { struct cxlflash_cfg *cfg = shost_priv(sdev->host); struct device *dev = &cfg->dev->dev; @@ -2096,7 +2096,7 @@ out: * * Return: 0 on success, -errno on failure */ -int cxlflash_ioctl(struct scsi_device *sdev, int cmd, void __user *arg) +int cxlflash_ioctl(struct scsi_device *sdev, unsigned int cmd, void __user *arg) { typedef int (*sioctl) (struct scsi_device *, void *); @@ -2179,8 +2179,7 @@ int cxlflash_ioctl(struct scsi_device *sdev, int cmd, void __user *arg) } if (unlikely(copy_from_user(&buf, arg, size))) { - dev_err(dev, "%s: copy_from_user() fail " - "size=%lu cmd=%d (%s) arg=%p\n", + dev_err(dev, "%s: copy_from_user() fail size=%lu cmd=%u (%s) arg=%p\n", __func__, size, cmd, decode_ioctl(cmd), arg); rc = -EFAULT; goto cxlflash_ioctl_exit; @@ -2203,8 +2202,7 @@ int cxlflash_ioctl(struct scsi_device *sdev, int cmd, void __user *arg) rc = do_ioctl(sdev, (void *)&buf); if (likely(!rc)) if (unlikely(copy_to_user(arg, &buf, size))) { - dev_err(dev, "%s: copy_to_user() fail " - "size=%lu cmd=%d (%s) arg=%p\n", + dev_err(dev, "%s: copy_to_user() fail size=%lu cmd=%u (%s) arg=%p\n", __func__, size, cmd, decode_ioctl(cmd), arg); rc = -EFAULT; } diff --git a/drivers/scsi/dpt_i2o.c b/drivers/scsi/dpt_i2o.c index 70d1a18278af..abdc34affdf6 100644 --- a/drivers/scsi/dpt_i2o.c +++ b/drivers/scsi/dpt_i2o.c @@ -589,46 +589,6 @@ static int adpt_show_info(struct seq_file *m, struct Scsi_Host *host) } /* - * Turn a struct scsi_cmnd * into a unique 32 bit 'context'. - */ -static u32 adpt_cmd_to_context(struct scsi_cmnd *cmd) -{ - return (u32)cmd->serial_number; -} - -/* - * Go from a u32 'context' to a struct scsi_cmnd * . - * This could probably be made more efficient. - */ -static struct scsi_cmnd * - adpt_cmd_from_context(adpt_hba * pHba, u32 context) -{ - struct scsi_cmnd * cmd; - struct scsi_device * d; - - if (context == 0) - return NULL; - - spin_unlock(pHba->host->host_lock); - shost_for_each_device(d, pHba->host) { - unsigned long flags; - spin_lock_irqsave(&d->list_lock, flags); - list_for_each_entry(cmd, &d->cmd_list, list) { - if (((u32)cmd->serial_number == context)) { - spin_unlock_irqrestore(&d->list_lock, flags); - scsi_device_put(d); - spin_lock(pHba->host->host_lock); - return cmd; - } - } - spin_unlock_irqrestore(&d->list_lock, flags); - } - spin_lock(pHba->host->host_lock); - - return NULL; -} - -/* * Turn a pointer to ioctl reply data into an u32 'context' */ static u32 adpt_ioctl_to_context(adpt_hba * pHba, void *reply) @@ -685,9 +645,6 @@ static int adpt_abort(struct scsi_cmnd * cmd) u32 msg[5]; int rcode; - if(cmd->serial_number == 0){ - return FAILED; - } pHba = (adpt_hba*) cmd->device->host->hostdata[0]; printk(KERN_INFO"%s: Trying to Abort\n",pHba->name); if ((dptdevice = (void*) (cmd->device->hostdata)) == NULL) { @@ -699,8 +656,9 @@ static int adpt_abort(struct scsi_cmnd * cmd) msg[0] = FIVE_WORD_MSG_SIZE|SGL_OFFSET_0; msg[1] = I2O_CMD_SCSI_ABORT<<24|HOST_TID<<12|dptdevice->tid; msg[2] = 0; - msg[3]= 0; - msg[4] = adpt_cmd_to_context(cmd); + msg[3]= 0; + /* Add 1 to avoid firmware treating it as invalid command */ + msg[4] = cmd->request->tag + 1; if (pHba->host) spin_lock_irq(pHba->host->host_lock); rcode = adpt_i2o_post_wait(pHba, msg, sizeof(msg), FOREVER); @@ -2198,20 +2156,27 @@ static irqreturn_t adpt_isr(int irq, void *dev_id) status = I2O_POST_WAIT_OK; } if(!(context & 0x40000000)) { - cmd = adpt_cmd_from_context(pHba, - readl(reply+12)); + /* + * The request tag is one less than the command tag + * as the firmware might treat a 0 tag as invalid + */ + cmd = scsi_host_find_tag(pHba->host, + readl(reply + 12) - 1); if(cmd != NULL) { printk(KERN_WARNING"%s: Apparent SCSI cmd in Post Wait Context - cmd=%p context=%x\n", pHba->name, cmd, context); } } adpt_i2o_post_wait_complete(context, status); } else { // SCSI message - cmd = adpt_cmd_from_context (pHba, readl(reply+12)); + /* + * The request tag is one less than the command tag + * as the firmware might treat a 0 tag as invalid + */ + cmd = scsi_host_find_tag(pHba->host, + readl(reply + 12) - 1); if(cmd != NULL){ scsi_dma_unmap(cmd); - if(cmd->serial_number != 0) { // If not timedout - adpt_i2o_to_scsi(reply, cmd); - } + adpt_i2o_to_scsi(reply, cmd); } } writel(m, pHba->reply_port); @@ -2277,7 +2242,8 @@ static s32 adpt_scsi_to_i2o(adpt_hba* pHba, struct scsi_cmnd* cmd, struct adpt_d // I2O_CMD_SCSI_EXEC msg[1] = ((0xff<<24)|(HOST_TID<<12)|d->tid); msg[2] = 0; - msg[3] = adpt_cmd_to_context(cmd); /* Want SCSI control block back */ + /* Add 1 to avoid firmware treating it as invalid command */ + msg[3] = cmd->request->tag + 1; // Our cards use the transaction context as the tag for queueing // Adaptec/DPT Private stuff msg[4] = I2O_CMD_SCSI_EXEC|(DPT_ORGANIZATION_ID<<16); @@ -2693,9 +2659,6 @@ static void adpt_fail_posted_scbs(adpt_hba* pHba) unsigned long flags; spin_lock_irqsave(&d->list_lock, flags); list_for_each_entry(cmd, &d->cmd_list, list) { - if(cmd->serial_number == 0){ - continue; - } cmd->result = (DID_OK << 16) | (QUEUE_FULL <<1); cmd->scsi_done(cmd); } diff --git a/drivers/scsi/esas2r/esas2r.h b/drivers/scsi/esas2r/esas2r.h index 858c3b33db78..7f43b95f4e94 100644 --- a/drivers/scsi/esas2r/esas2r.h +++ b/drivers/scsi/esas2r/esas2r.h @@ -965,8 +965,8 @@ struct esas2r_adapter { const char *esas2r_info(struct Scsi_Host *); int esas2r_write_params(struct esas2r_adapter *a, struct esas2r_request *rq, struct esas2r_sas_nvram *data); -int esas2r_ioctl_handler(void *hostdata, int cmd, void __user *arg); -int esas2r_ioctl(struct scsi_device *dev, int cmd, void __user *arg); +int esas2r_ioctl_handler(void *hostdata, unsigned int cmd, void __user *arg); +int esas2r_ioctl(struct scsi_device *dev, unsigned int cmd, void __user *arg); u8 handle_hba_ioctl(struct esas2r_adapter *a, struct atto_ioctl *ioctl_hba); int esas2r_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd); diff --git a/drivers/scsi/esas2r/esas2r_init.c b/drivers/scsi/esas2r/esas2r_init.c index 46b2c83ba21f..950cd92df2ff 100644 --- a/drivers/scsi/esas2r/esas2r_init.c +++ b/drivers/scsi/esas2r/esas2r_init.c @@ -1241,6 +1241,7 @@ static bool esas2r_format_init_msg(struct esas2r_adapter *a, a->init_msg = ESAS2R_INIT_MSG_GET_INIT; break; } + /* fall through */ case ESAS2R_INIT_MSG_GET_INIT: if (msg == ESAS2R_INIT_MSG_GET_INIT) { @@ -1254,7 +1255,7 @@ static bool esas2r_format_init_msg(struct esas2r_adapter *a, esas2r_hdebug("FAILED"); } } - /* fall through */ + /* fall through */ default: rq->req_stat = RS_SUCCESS; diff --git a/drivers/scsi/esas2r/esas2r_ioctl.c b/drivers/scsi/esas2r/esas2r_ioctl.c index 34bcc8c04ff4..3d130523c288 100644 --- a/drivers/scsi/esas2r/esas2r_ioctl.c +++ b/drivers/scsi/esas2r/esas2r_ioctl.c @@ -1274,7 +1274,7 @@ int esas2r_write_params(struct esas2r_adapter *a, struct esas2r_request *rq, /* This function only cares about ATTO-specific ioctls (atto_express_ioctl) */ -int esas2r_ioctl_handler(void *hostdata, int cmd, void __user *arg) +int esas2r_ioctl_handler(void *hostdata, unsigned int cmd, void __user *arg) { struct atto_express_ioctl *ioctl = NULL; struct esas2r_adapter *a; @@ -1292,9 +1292,8 @@ int esas2r_ioctl_handler(void *hostdata, int cmd, void __user *arg) ioctl = memdup_user(arg, sizeof(struct atto_express_ioctl)); if (IS_ERR(ioctl)) { esas2r_log(ESAS2R_LOG_WARN, - "ioctl_handler access_ok failed for cmd %d, " - "address %p", cmd, - arg); + "ioctl_handler access_ok failed for cmd %u, address %p", + cmd, arg); return PTR_ERR(ioctl); } @@ -1493,7 +1492,7 @@ int esas2r_ioctl_handler(void *hostdata, int cmd, void __user *arg) ioctl_done: if (err < 0) { - esas2r_log(ESAS2R_LOG_WARN, "err %d on ioctl cmd %d", err, + esas2r_log(ESAS2R_LOG_WARN, "err %d on ioctl cmd %u", err, cmd); switch (err) { @@ -1518,9 +1517,8 @@ ioctl_done: err = __copy_to_user(arg, ioctl, sizeof(struct atto_express_ioctl)); if (err != 0) { esas2r_log(ESAS2R_LOG_WARN, - "ioctl_handler copy_to_user didn't copy " - "everything (err %d, cmd %d)", err, - cmd); + "ioctl_handler copy_to_user didn't copy everything (err %d, cmd %u)", + err, cmd); kfree(ioctl); return -EFAULT; @@ -1531,7 +1529,7 @@ ioctl_done: return 0; } -int esas2r_ioctl(struct scsi_device *sd, int cmd, void __user *arg) +int esas2r_ioctl(struct scsi_device *sd, unsigned int cmd, void __user *arg) { return esas2r_ioctl_handler(sd->host->hostdata, cmd, arg); } diff --git a/drivers/scsi/esas2r/esas2r_main.c b/drivers/scsi/esas2r/esas2r_main.c index 64397d441bae..fdbda5c05aa0 100644 --- a/drivers/scsi/esas2r/esas2r_main.c +++ b/drivers/scsi/esas2r/esas2r_main.c @@ -623,7 +623,7 @@ static int esas2r_proc_major; long esas2r_proc_ioctl(struct file *fp, unsigned int cmd, unsigned long arg) { return esas2r_ioctl_handler(esas2r_proc_host->hostdata, - (int)cmd, (void __user *)arg); + cmd, (void __user *)arg); } static void __exit esas2r_exit(void) diff --git a/drivers/scsi/fcoe/fcoe.c b/drivers/scsi/fcoe/fcoe.c index cd19be3f3405..8ba8862d3292 100644 --- a/drivers/scsi/fcoe/fcoe.c +++ b/drivers/scsi/fcoe/fcoe.c @@ -389,7 +389,7 @@ static int fcoe_interface_setup(struct fcoe_interface *fcoe, * Returns: pointer to a struct fcoe_interface or NULL on error */ static struct fcoe_interface *fcoe_interface_create(struct net_device *netdev, - enum fip_state fip_mode) + enum fip_mode fip_mode) { struct fcoe_ctlr_device *ctlr_dev; struct fcoe_ctlr *ctlr; diff --git a/drivers/scsi/fcoe/fcoe_ctlr.c b/drivers/scsi/fcoe/fcoe_ctlr.c index 54da3166da8d..7dc4ffa24430 100644 --- a/drivers/scsi/fcoe/fcoe_ctlr.c +++ b/drivers/scsi/fcoe/fcoe_ctlr.c @@ -147,7 +147,7 @@ static void fcoe_ctlr_map_dest(struct fcoe_ctlr *fip) * fcoe_ctlr_init() - Initialize the FCoE Controller instance * @fip: The FCoE controller to initialize */ -void fcoe_ctlr_init(struct fcoe_ctlr *fip, enum fip_state mode) +void fcoe_ctlr_init(struct fcoe_ctlr *fip, enum fip_mode mode) { fcoe_ctlr_set_state(fip, FIP_ST_LINK_WAIT); fip->mode = mode; @@ -454,7 +454,10 @@ void fcoe_ctlr_link_up(struct fcoe_ctlr *fip) mutex_unlock(&fip->ctlr_mutex); fc_linkup(fip->lp); } else if (fip->state == FIP_ST_LINK_WAIT) { - fcoe_ctlr_set_state(fip, fip->mode); + if (fip->mode == FIP_MODE_NON_FIP) + fcoe_ctlr_set_state(fip, FIP_ST_NON_FIP); + else + fcoe_ctlr_set_state(fip, FIP_ST_AUTO); switch (fip->mode) { default: LIBFCOE_FIP_DBG(fip, "invalid mode %d\n", fip->mode); diff --git a/drivers/scsi/fcoe/fcoe_sysfs.c b/drivers/scsi/fcoe/fcoe_sysfs.c index 5c8310bade61..c3dcbdc3aa64 100644 --- a/drivers/scsi/fcoe/fcoe_sysfs.c +++ b/drivers/scsi/fcoe/fcoe_sysfs.c @@ -671,8 +671,19 @@ static const struct device_type fcoe_fcf_device_type = { .release = fcoe_fcf_device_release, }; -static BUS_ATTR(ctlr_create, S_IWUSR, NULL, fcoe_ctlr_create_store); -static BUS_ATTR(ctlr_destroy, S_IWUSR, NULL, fcoe_ctlr_destroy_store); +static ssize_t ctlr_create_store(struct bus_type *bus, const char *buf, + size_t count) +{ + return fcoe_ctlr_create_store(bus, buf, count); +} +static BUS_ATTR_WO(ctlr_create); + +static ssize_t ctlr_destroy_store(struct bus_type *bus, const char *buf, + size_t count) +{ + return fcoe_ctlr_destroy_store(bus, buf, count); +} +static BUS_ATTR_WO(ctlr_destroy); static struct attribute *fcoe_bus_attrs[] = { &bus_attr_ctlr_create.attr, diff --git a/drivers/scsi/fcoe/fcoe_transport.c b/drivers/scsi/fcoe/fcoe_transport.c index f4909cd206d3..29fe3426f9f2 100644 --- a/drivers/scsi/fcoe/fcoe_transport.c +++ b/drivers/scsi/fcoe/fcoe_transport.c @@ -855,7 +855,6 @@ out_nodev: mutex_unlock(&ft_mutex); return rc; } -EXPORT_SYMBOL(fcoe_ctlr_destroy_store); /** * fcoe_transport_create() - Create a fcoe interface @@ -873,7 +872,7 @@ static int fcoe_transport_create(const char *buffer, int rc = -ENODEV; struct net_device *netdev = NULL; struct fcoe_transport *ft = NULL; - enum fip_state fip_mode = (enum fip_state)(long)kp->arg; + enum fip_mode fip_mode = (enum fip_mode)kp->arg; mutex_lock(&ft_mutex); diff --git a/drivers/scsi/fnic/fnic.h b/drivers/scsi/fnic/fnic.h index d094ba59ed15..477513dc23b7 100644 --- a/drivers/scsi/fnic/fnic.h +++ b/drivers/scsi/fnic/fnic.h @@ -39,7 +39,7 @@ #define DRV_NAME "fnic" #define DRV_DESCRIPTION "Cisco FCoE HBA Driver" -#define DRV_VERSION "1.6.0.34" +#define DRV_VERSION "1.6.0.47" #define PFX DRV_NAME ": " #define DFX DRV_NAME "%d: " @@ -49,7 +49,7 @@ #define FNIC_MAX_IO_REQ 1024 /* scsi_cmnd tag map entries */ #define FNIC_DFLT_IO_REQ 256 /* Default scsi_cmnd tag map entries */ #define FNIC_IO_LOCKS 64 /* IO locks: power of 2 */ -#define FNIC_DFLT_QUEUE_DEPTH 32 +#define FNIC_DFLT_QUEUE_DEPTH 256 #define FNIC_STATS_RATE_LIMIT 4 /* limit rate at which stats are pulled up */ /* @@ -128,6 +128,7 @@ __fnic_set_state_flags(fnicp, st_flags, 1) extern unsigned int fnic_log_level; +extern unsigned int io_completions; #define FNIC_MAIN_LOGGING 0x01 #define FNIC_FCS_LOGGING 0x02 @@ -196,6 +197,7 @@ enum fnic_state { #define FNIC_WQ_MAX 1 #define FNIC_RQ_MAX 1 #define FNIC_CQ_MAX (FNIC_WQ_COPY_MAX + FNIC_WQ_MAX + FNIC_RQ_MAX) +#define FNIC_DFLT_IO_COMPLETIONS 256 struct mempool; diff --git a/drivers/scsi/fnic/fnic_debugfs.c b/drivers/scsi/fnic/fnic_debugfs.c index 139fffa3658a..21991c99db7c 100644 --- a/drivers/scsi/fnic/fnic_debugfs.c +++ b/drivers/scsi/fnic/fnic_debugfs.c @@ -54,23 +54,9 @@ int fnic_debugfs_init(void) { int rc = -1; fnic_trace_debugfs_root = debugfs_create_dir("fnic", NULL); - if (!fnic_trace_debugfs_root) { - printk(KERN_DEBUG "Cannot create debugfs root\n"); - return rc; - } - - if (!fnic_trace_debugfs_root) { - printk(KERN_DEBUG - "fnic root directory doesn't exist in debugfs\n"); - return rc; - } fnic_stats_debugfs_root = debugfs_create_dir("statistics", fnic_trace_debugfs_root); - if (!fnic_stats_debugfs_root) { - printk(KERN_DEBUG "Cannot create Statistics directory\n"); - return rc; - } /* Allocate memory to structure */ fc_trc_flag = (struct fc_trace_flag_type *) @@ -356,39 +342,19 @@ static const struct file_operations fnic_trace_debugfs_fops = { * it will also create file trace_enable to control enable/disable of * trace logging into trace buffer. */ -int fnic_trace_debugfs_init(void) +void fnic_trace_debugfs_init(void) { - int rc = -1; - if (!fnic_trace_debugfs_root) { - printk(KERN_DEBUG - "FNIC Debugfs root directory doesn't exist\n"); - return rc; - } fnic_trace_enable = debugfs_create_file("tracing_enable", S_IFREG|S_IRUGO|S_IWUSR, fnic_trace_debugfs_root, &(fc_trc_flag->fnic_trace), &fnic_trace_ctrl_fops); - if (!fnic_trace_enable) { - printk(KERN_DEBUG - "Cannot create trace_enable file under debugfs\n"); - return rc; - } - fnic_trace_debugfs_file = debugfs_create_file("trace", S_IFREG|S_IRUGO|S_IWUSR, fnic_trace_debugfs_root, &(fc_trc_flag->fnic_trace), &fnic_trace_debugfs_fops); - - if (!fnic_trace_debugfs_file) { - printk(KERN_DEBUG - "Cannot create trace file under debugfs\n"); - return rc; - } - rc = 0; - return rc; } /* @@ -419,37 +385,20 @@ void fnic_trace_debugfs_terminate(void) * trace logging into trace buffer. */ -int fnic_fc_trace_debugfs_init(void) +void fnic_fc_trace_debugfs_init(void) { - int rc = -1; - - if (!fnic_trace_debugfs_root) { - pr_err("fnic:Debugfs root directory doesn't exist\n"); - return rc; - } - fnic_fc_trace_enable = debugfs_create_file("fc_trace_enable", S_IFREG|S_IRUGO|S_IWUSR, fnic_trace_debugfs_root, &(fc_trc_flag->fc_trace), &fnic_trace_ctrl_fops); - if (!fnic_fc_trace_enable) { - pr_err("fnic: Failed create fc_trace_enable file\n"); - return rc; - } - fnic_fc_trace_clear = debugfs_create_file("fc_trace_clear", S_IFREG|S_IRUGO|S_IWUSR, fnic_trace_debugfs_root, &(fc_trc_flag->fc_clear), &fnic_trace_ctrl_fops); - if (!fnic_fc_trace_clear) { - pr_err("fnic: Failed to create fc_trace_enable file\n"); - return rc; - } - fnic_fc_rdata_trace_debugfs_file = debugfs_create_file("fc_trace_rdata", S_IFREG|S_IRUGO|S_IWUSR, @@ -457,24 +406,12 @@ int fnic_fc_trace_debugfs_init(void) &(fc_trc_flag->fc_normal_file), &fnic_trace_debugfs_fops); - if (!fnic_fc_rdata_trace_debugfs_file) { - pr_err("fnic: Failed create fc_rdata_trace file\n"); - return rc; - } - fnic_fc_trace_debugfs_file = debugfs_create_file("fc_trace", S_IFREG|S_IRUGO|S_IWUSR, fnic_trace_debugfs_root, &(fc_trc_flag->fc_row_file), &fnic_trace_debugfs_fops); - - if (!fnic_fc_trace_debugfs_file) { - pr_err("fnic: Failed to create fc_trace file\n"); - return rc; - } - rc = 0; - return rc; } /* @@ -757,45 +694,26 @@ static const struct file_operations fnic_reset_debugfs_fops = { * It will create file stats and reset_stats under statistics/host# directory * to log per fnic stats. */ -int fnic_stats_debugfs_init(struct fnic *fnic) +void fnic_stats_debugfs_init(struct fnic *fnic) { - int rc = -1; char name[16]; snprintf(name, sizeof(name), "host%d", fnic->lport->host->host_no); - if (!fnic_stats_debugfs_root) { - printk(KERN_DEBUG "fnic_stats root doesn't exist\n"); - return rc; - } fnic->fnic_stats_debugfs_host = debugfs_create_dir(name, fnic_stats_debugfs_root); - if (!fnic->fnic_stats_debugfs_host) { - printk(KERN_DEBUG "Cannot create host directory\n"); - return rc; - } fnic->fnic_stats_debugfs_file = debugfs_create_file("stats", S_IFREG|S_IRUGO|S_IWUSR, fnic->fnic_stats_debugfs_host, fnic, &fnic_stats_debugfs_fops); - if (!fnic->fnic_stats_debugfs_file) { - printk(KERN_DEBUG "Cannot create host stats file\n"); - return rc; - } fnic->fnic_reset_debugfs_file = debugfs_create_file("reset_stats", S_IFREG|S_IRUGO|S_IWUSR, fnic->fnic_stats_debugfs_host, fnic, &fnic_reset_debugfs_fops); - if (!fnic->fnic_reset_debugfs_file) { - printk(KERN_DEBUG "Cannot create host stats file\n"); - return rc; - } - rc = 0; - return rc; } /* diff --git a/drivers/scsi/fnic/fnic_fcs.c b/drivers/scsi/fnic/fnic_fcs.c index 844ef688fa91..911a5adc289c 100644 --- a/drivers/scsi/fnic/fnic_fcs.c +++ b/drivers/scsi/fnic/fnic_fcs.c @@ -65,11 +65,21 @@ void fnic_handle_link(struct work_struct *work) fnic->link_status = vnic_dev_link_status(fnic->vdev); fnic->link_down_cnt = vnic_dev_link_down_cnt(fnic->vdev); + atomic64_set(&fnic->fnic_stats.misc_stats.current_port_speed, + vnic_dev_port_speed(fnic->vdev)); + shost_printk(KERN_INFO, fnic->lport->host, "Current vnic speed set to : %llu\n", + (u64)atomic64_read( + &fnic->fnic_stats.misc_stats.current_port_speed)); + switch (vnic_dev_port_speed(fnic->vdev)) { case DCEM_PORTSPEED_10G: fc_host_speed(fnic->lport->host) = FC_PORTSPEED_10GBIT; fnic->lport->link_supported_speeds = FC_PORTSPEED_10GBIT; break; + case DCEM_PORTSPEED_20G: + fc_host_speed(fnic->lport->host) = FC_PORTSPEED_20GBIT; + fnic->lport->link_supported_speeds = FC_PORTSPEED_20GBIT; + break; case DCEM_PORTSPEED_25G: fc_host_speed(fnic->lport->host) = FC_PORTSPEED_25GBIT; fnic->lport->link_supported_speeds = FC_PORTSPEED_25GBIT; diff --git a/drivers/scsi/fnic/fnic_io.h b/drivers/scsi/fnic/fnic_io.h index e0bc659ed71f..1cb6a68c8e4e 100644 --- a/drivers/scsi/fnic/fnic_io.h +++ b/drivers/scsi/fnic/fnic_io.h @@ -70,9 +70,10 @@ enum fnic_port_speeds { DCEM_PORTSPEED_NONE = 0, DCEM_PORTSPEED_1G = 1000, DCEM_PORTSPEED_10G = 10000, + DCEM_PORTSPEED_20G = 20000, + DCEM_PORTSPEED_25G = 25000, DCEM_PORTSPEED_40G = 40000, DCEM_PORTSPEED_4x10G = 41000, - DCEM_PORTSPEED_25G = 25000, DCEM_PORTSPEED_100G = 100000, }; #endif /* _FNIC_IO_H_ */ diff --git a/drivers/scsi/fnic/fnic_isr.c b/drivers/scsi/fnic/fnic_isr.c index 4e3a50202e8c..da4602b63495 100644 --- a/drivers/scsi/fnic/fnic_isr.c +++ b/drivers/scsi/fnic/fnic_isr.c @@ -51,7 +51,7 @@ static irqreturn_t fnic_isr_legacy(int irq, void *data) } if (pba & (1 << FNIC_INTX_WQ_RQ_COPYWQ)) { - work_done += fnic_wq_copy_cmpl_handler(fnic, -1); + work_done += fnic_wq_copy_cmpl_handler(fnic, io_completions); work_done += fnic_wq_cmpl_handler(fnic, -1); work_done += fnic_rq_cmpl_handler(fnic, -1); @@ -72,7 +72,7 @@ static irqreturn_t fnic_isr_msi(int irq, void *data) fnic->fnic_stats.misc_stats.last_isr_time = jiffies; atomic64_inc(&fnic->fnic_stats.misc_stats.isr_count); - work_done += fnic_wq_copy_cmpl_handler(fnic, -1); + work_done += fnic_wq_copy_cmpl_handler(fnic, io_completions); work_done += fnic_wq_cmpl_handler(fnic, -1); work_done += fnic_rq_cmpl_handler(fnic, -1); @@ -125,7 +125,7 @@ static irqreturn_t fnic_isr_msix_wq_copy(int irq, void *data) fnic->fnic_stats.misc_stats.last_isr_time = jiffies; atomic64_inc(&fnic->fnic_stats.misc_stats.isr_count); - wq_copy_work_done = fnic_wq_copy_cmpl_handler(fnic, -1); + wq_copy_work_done = fnic_wq_copy_cmpl_handler(fnic, io_completions); vnic_intr_return_credits(&fnic->intr[FNIC_MSIX_WQ_COPY], wq_copy_work_done, 1 /* unmask intr */, diff --git a/drivers/scsi/fnic/fnic_main.c b/drivers/scsi/fnic/fnic_main.c index 5b3534b0deda..18584ab27c32 100644 --- a/drivers/scsi/fnic/fnic_main.c +++ b/drivers/scsi/fnic/fnic_main.c @@ -69,6 +69,11 @@ unsigned int fnic_log_level; module_param(fnic_log_level, int, S_IRUGO|S_IWUSR); MODULE_PARM_DESC(fnic_log_level, "bit mask of fnic logging levels"); + +unsigned int io_completions = FNIC_DFLT_IO_COMPLETIONS; +module_param(io_completions, int, S_IRUGO|S_IWUSR); +MODULE_PARM_DESC(io_completions, "Max CQ entries to process at a time"); + unsigned int fnic_trace_max_pages = 16; module_param(fnic_trace_max_pages, uint, S_IRUGO|S_IWUSR); MODULE_PARM_DESC(fnic_trace_max_pages, "Total allocated memory pages " @@ -178,6 +183,9 @@ static void fnic_get_host_speed(struct Scsi_Host *shost) case DCEM_PORTSPEED_10G: fc_host_speed(shost) = FC_PORTSPEED_10GBIT; break; + case DCEM_PORTSPEED_20G: + fc_host_speed(shost) = FC_PORTSPEED_20GBIT; + break; case DCEM_PORTSPEED_25G: fc_host_speed(shost) = FC_PORTSPEED_25GBIT; break; @@ -500,7 +508,7 @@ static int fnic_cleanup(struct fnic *fnic) } /* Clean up completed IOs and FCS frames */ - fnic_wq_copy_cmpl_handler(fnic, -1); + fnic_wq_copy_cmpl_handler(fnic, io_completions); fnic_wq_cmpl_handler(fnic, -1); fnic_rq_cmpl_handler(fnic, -1); @@ -578,12 +586,7 @@ static int fnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) host->transportt = fnic_fc_transport; - err = fnic_stats_debugfs_init(fnic); - if (err) { - shost_printk(KERN_ERR, fnic->lport->host, - "Failed to initialize debugfs for stats\n"); - fnic_stats_debugfs_remove(fnic); - } + fnic_stats_debugfs_init(fnic); /* Setup PCI resources */ pci_set_drvdata(pdev, fnic); @@ -650,12 +653,20 @@ static int fnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto err_out_iounmap; } + err = vnic_dev_cmd_init(fnic->vdev); + if (err) { + shost_printk(KERN_ERR, fnic->lport->host, + "vnic_dev_cmd_init() returns %d, aborting\n", + err); + goto err_out_vnic_unregister; + } + err = fnic_dev_wait(fnic->vdev, vnic_dev_open, - vnic_dev_open_done, 0); + vnic_dev_open_done, CMD_OPENF_RQ_ENABLE_THEN_POST); if (err) { shost_printk(KERN_ERR, fnic->lport->host, "vNIC dev open failed, aborting.\n"); - goto err_out_vnic_unregister; + goto err_out_dev_cmd_deinit; } err = vnic_dev_init(fnic->vdev, 0); @@ -796,6 +807,7 @@ static int fnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) /* allocate RQ buffers and post them to RQ*/ for (i = 0; i < fnic->rq_count; i++) { + vnic_rq_enable(&fnic->rq[i]); err = vnic_rq_fill(&fnic->rq[i], fnic_alloc_rq_frame); if (err) { shost_printk(KERN_ERR, fnic->lport->host, @@ -870,15 +882,11 @@ static int fnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) /* Enable all queues */ for (i = 0; i < fnic->raw_wq_count; i++) vnic_wq_enable(&fnic->wq[i]); - for (i = 0; i < fnic->rq_count; i++) - vnic_rq_enable(&fnic->rq[i]); for (i = 0; i < fnic->wq_copy_count; i++) vnic_wq_copy_enable(&fnic->wq_copy[i]); fc_fabric_login(lp); - vnic_dev_enable(fnic->vdev); - err = fnic_request_intr(fnic); if (err) { shost_printk(KERN_ERR, fnic->lport->host, @@ -886,6 +894,8 @@ static int fnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto err_out_free_exch_mgr; } + vnic_dev_enable(fnic->vdev); + for (i = 0; i < fnic->intr_count; i++) vnic_intr_unmask(&fnic->intr[i]); @@ -914,6 +924,7 @@ err_out_clear_intr: fnic_clear_intr_mode(fnic); err_out_dev_close: vnic_dev_close(fnic->vdev); +err_out_dev_cmd_deinit: err_out_vnic_unregister: vnic_dev_unregister(fnic->vdev); err_out_iounmap: diff --git a/drivers/scsi/fnic/fnic_scsi.c b/drivers/scsi/fnic/fnic_scsi.c index cafbcfb85bfa..80608b53897b 100644 --- a/drivers/scsi/fnic/fnic_scsi.c +++ b/drivers/scsi/fnic/fnic_scsi.c @@ -180,20 +180,19 @@ void __fnic_set_state_flags(struct fnic *fnic, unsigned long st_flags, unsigned long clearbits) { - struct Scsi_Host *host = fnic->lport->host; - int sh_locked = spin_is_locked(host->host_lock); unsigned long flags = 0; + unsigned long host_lock_flags = 0; - if (!sh_locked) - spin_lock_irqsave(host->host_lock, flags); + spin_lock_irqsave(&fnic->fnic_lock, flags); + spin_lock_irqsave(fnic->lport->host->host_lock, host_lock_flags); if (clearbits) fnic->state_flags &= ~st_flags; else fnic->state_flags |= st_flags; - if (!sh_locked) - spin_unlock_irqrestore(host->host_lock, flags); + spin_unlock_irqrestore(fnic->lport->host->host_lock, host_lock_flags); + spin_unlock_irqrestore(&fnic->fnic_lock, flags); return; } @@ -1326,13 +1325,32 @@ int fnic_wq_copy_cmpl_handler(struct fnic *fnic, int copy_work_to_do) unsigned int wq_work_done = 0; unsigned int i, cq_index; unsigned int cur_work_done; + struct misc_stats *misc_stats = &fnic->fnic_stats.misc_stats; + u64 start_jiffies = 0; + u64 end_jiffies = 0; + u64 delta_jiffies = 0; + u64 delta_ms = 0; for (i = 0; i < fnic->wq_copy_count; i++) { cq_index = i + fnic->raw_wq_count + fnic->rq_count; + + start_jiffies = jiffies; cur_work_done = vnic_cq_copy_service(&fnic->cq[cq_index], fnic_fcpio_cmpl_handler, copy_work_to_do); + end_jiffies = jiffies; + wq_work_done += cur_work_done; + delta_jiffies = end_jiffies - start_jiffies; + if (delta_jiffies > + (u64) atomic64_read(&misc_stats->max_isr_jiffies)) { + atomic64_set(&misc_stats->max_isr_jiffies, + delta_jiffies); + delta_ms = jiffies_to_msecs(delta_jiffies); + atomic64_set(&misc_stats->max_isr_time_ms, delta_ms); + atomic64_set(&misc_stats->corr_work_done, + cur_work_done); + } } return wq_work_done; } @@ -1397,8 +1415,9 @@ static void fnic_cleanup_io(struct fnic *fnic, int exclude_id) cleanup_scsi_cmd: sc->result = DID_TRANSPORT_DISRUPTED << 16; FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, - "%s: sc duration = %lu DID_TRANSPORT_DISRUPTED\n", - __func__, (jiffies - start_time)); + "%s: tag:0x%x : sc:0x%p duration = %lu DID_TRANSPORT_DISRUPTED\n", + __func__, sc->request->tag, sc, + (jiffies - start_time)); if (atomic64_read(&fnic->io_cmpl_skip)) atomic64_dec(&fnic->io_cmpl_skip); @@ -1407,6 +1426,11 @@ cleanup_scsi_cmd: /* Complete the command to SCSI */ if (sc->scsi_done) { + if (!(CMD_FLAGS(sc) & FNIC_IO_ISSUED)) + shost_printk(KERN_ERR, fnic->lport->host, + "Calling done for IO not issued to fw: tag:0x%x sc:0x%p\n", + sc->request->tag, sc); + FNIC_TRACE(fnic_cleanup_io, sc->device->host->host_no, i, sc, jiffies_to_msecs(jiffies - start_time), diff --git a/drivers/scsi/fnic/fnic_stats.h b/drivers/scsi/fnic/fnic_stats.h index 9daa6ada6fa0..086f729f3c46 100644 --- a/drivers/scsi/fnic/fnic_stats.h +++ b/drivers/scsi/fnic/fnic_stats.h @@ -97,6 +97,9 @@ struct vlan_stats { struct misc_stats { u64 last_isr_time; u64 last_ack_time; + atomic64_t max_isr_jiffies; + atomic64_t max_isr_time_ms; + atomic64_t corr_work_done; atomic64_t isr_count; atomic64_t max_cq_entries; atomic64_t ack_index_out_of_range; @@ -113,6 +116,7 @@ struct misc_stats { atomic64_t queue_fulls; atomic64_t rport_not_ready; atomic64_t frame_errors; + atomic64_t current_port_speed; }; struct fnic_stats { @@ -134,6 +138,6 @@ struct stats_debug_info { }; int fnic_get_stats_data(struct stats_debug_info *, struct fnic_stats *); -int fnic_stats_debugfs_init(struct fnic *); +void fnic_stats_debugfs_init(struct fnic *); void fnic_stats_debugfs_remove(struct fnic *); #endif /* _FNIC_STATS_H_ */ diff --git a/drivers/scsi/fnic/fnic_trace.c b/drivers/scsi/fnic/fnic_trace.c index bf0fd2aeb92e..9621831e17ba 100644 --- a/drivers/scsi/fnic/fnic_trace.c +++ b/drivers/scsi/fnic/fnic_trace.c @@ -409,6 +409,9 @@ int fnic_get_stats_data(struct stats_debug_info *debug, len += snprintf(debug->debug_buffer + len, buf_size - len, "Last ISR time: %llu (%8llu.%09lu)\n" "Last ACK time: %llu (%8llu.%09lu)\n" + "Max ISR jiffies: %llu\n" + "Max ISR time (ms) (0 denotes < 1 ms): %llu\n" + "Corr. work done: %llu\n" "Number of ISRs: %lld\n" "Maximum CQ Entries: %lld\n" "Number of ACK index out of range: %lld\n" @@ -428,6 +431,9 @@ int fnic_get_stats_data(struct stats_debug_info *debug, (s64)val1.tv_sec, val1.tv_nsec, (u64)stats->misc_stats.last_ack_time, (s64)val2.tv_sec, val2.tv_nsec, + (u64)atomic64_read(&stats->misc_stats.max_isr_jiffies), + (u64)atomic64_read(&stats->misc_stats.max_isr_time_ms), + (u64)atomic64_read(&stats->misc_stats.corr_work_done), (u64)atomic64_read(&stats->misc_stats.isr_count), (u64)atomic64_read(&stats->misc_stats.max_cq_entries), (u64)atomic64_read(&stats->misc_stats.ack_index_out_of_range), @@ -446,6 +452,11 @@ int fnic_get_stats_data(struct stats_debug_info *debug, (u64)atomic64_read(&stats->misc_stats.rport_not_ready), (u64)atomic64_read(&stats->misc_stats.frame_errors)); + len += snprintf(debug->debug_buffer + len, buf_size - len, + "Firmware reported port seed: %llu\n", + (u64)atomic64_read( + &stats->misc_stats.current_port_speed)); + return len; } @@ -503,15 +514,10 @@ int fnic_trace_buf_init(void) fnic_trace_entries.page_offset[i] = fnic_buf_head; fnic_buf_head += FNIC_ENTRY_SIZE_BYTES; } - err = fnic_trace_debugfs_init(); - if (err < 0) { - pr_err("fnic: Failed to initialize debugfs for tracing\n"); - goto err_fnic_trace_debugfs_init; - } + fnic_trace_debugfs_init(); pr_info("fnic: Successfully Initialized Trace Buffer\n"); return err; -err_fnic_trace_debugfs_init: - fnic_trace_free(); + err_fnic_trace_buf_init: return err; } @@ -596,16 +602,10 @@ int fnic_fc_trace_init(void) fc_trace_entries.page_offset[i] = fc_trace_buf_head; fc_trace_buf_head += FC_TRC_SIZE_BYTES; } - err = fnic_fc_trace_debugfs_init(); - if (err < 0) { - pr_err("fnic: Failed to initialize FC_CTLR tracing.\n"); - goto err_fnic_fc_ctlr_trace_debugfs_init; - } + fnic_fc_trace_debugfs_init(); pr_info("fnic: Successfully Initialized FC_CTLR Trace Buffer\n"); return err; -err_fnic_fc_ctlr_trace_debugfs_init: - fnic_fc_trace_free(); err_fnic_fc_ctlr_trace_buf_init: return err; } diff --git a/drivers/scsi/fnic/fnic_trace.h b/drivers/scsi/fnic/fnic_trace.h index e375d0c2eaaf..8aa55c1e2025 100644 --- a/drivers/scsi/fnic/fnic_trace.h +++ b/drivers/scsi/fnic/fnic_trace.h @@ -111,7 +111,7 @@ int fnic_trace_buf_init(void); void fnic_trace_free(void); int fnic_debugfs_init(void); void fnic_debugfs_terminate(void); -int fnic_trace_debugfs_init(void); +void fnic_trace_debugfs_init(void); void fnic_trace_debugfs_terminate(void); /* Fnic FC CTLR Trace releated function */ @@ -123,7 +123,7 @@ int fnic_fc_trace_get_data(fnic_dbgfs_t *fnic_dbgfs_prt, u8 rdata_flag); void copy_and_format_trace_data(struct fc_trace_hdr *tdata, fnic_dbgfs_t *fnic_dbgfs_prt, int *len, u8 rdata_flag); -int fnic_fc_trace_debugfs_init(void); +void fnic_fc_trace_debugfs_init(void); void fnic_fc_trace_debugfs_terminate(void); #endif diff --git a/drivers/scsi/fnic/vnic_dev.c b/drivers/scsi/fnic/vnic_dev.c index 434447ea24b8..78af9cc2009b 100644 --- a/drivers/scsi/fnic/vnic_dev.c +++ b/drivers/scsi/fnic/vnic_dev.c @@ -27,6 +27,24 @@ #include "vnic_devcmd.h" #include "vnic_dev.h" #include "vnic_stats.h" +#include "vnic_wq.h" + +struct devcmd2_controller { + struct vnic_wq_ctrl *wq_ctrl; + struct vnic_dev_ring results_ring; + struct vnic_wq wq; + struct vnic_devcmd2 *cmd_ring; + struct devcmd2_result *result; + u16 next_result; + u16 result_size; + int color; +}; + +enum vnic_proxy_type { + PROXY_NONE, + PROXY_BY_BDF, + PROXY_BY_INDEX, +}; struct vnic_res { void __iomem *vaddr; @@ -48,6 +66,12 @@ struct vnic_dev { dma_addr_t stats_pa; struct vnic_devcmd_fw_info *fw_info; dma_addr_t fw_info_pa; + enum vnic_proxy_type proxy; + u32 proxy_index; + u64 args[VNIC_DEVCMD_NARGS]; + struct devcmd2_controller *devcmd2; + int (*devcmd_rtn)(struct vnic_dev *vdev, enum vnic_devcmd_cmd cmd, + int wait); }; #define VNIC_MAX_RES_HDR_SIZE \ @@ -119,6 +143,7 @@ static int vnic_dev_discover_res(struct vnic_dev *vdev, } break; case RES_TYPE_INTR_PBA_LEGACY: + case RES_TYPE_DEVCMD2: case RES_TYPE_DEVCMD: len = count; break; @@ -229,8 +254,7 @@ void vnic_dev_free_desc_ring(struct vnic_dev *vdev, struct vnic_dev_ring *ring) } } -int vnic_dev_cmd(struct vnic_dev *vdev, enum vnic_devcmd_cmd cmd, - u64 *a0, u64 *a1, int wait) +int vnic_dev_cmd1(struct vnic_dev *vdev, enum vnic_devcmd_cmd cmd, int wait) { struct vnic_devcmd __iomem *devcmd = vdev->devcmd; int delay; @@ -244,6 +268,8 @@ int vnic_dev_cmd(struct vnic_dev *vdev, enum vnic_devcmd_cmd cmd, EBUSY, /* ERR_EBUSY */ }; int err; + u64 *a0 = &vdev->args[0]; + u64 *a1 = &vdev->args[1]; status = ioread32(&devcmd->status); if (status & STAT_BUSY) { @@ -290,6 +316,223 @@ int vnic_dev_cmd(struct vnic_dev *vdev, enum vnic_devcmd_cmd cmd, return -ETIMEDOUT; } +int vnic_dev_cmd2(struct vnic_dev *vdev, enum vnic_devcmd_cmd cmd, + int wait) +{ + struct devcmd2_controller *dc2c = vdev->devcmd2; + struct devcmd2_result *result; + u8 color; + unsigned int i; + int delay; + int err; + u32 fetch_index; + u32 posted; + u32 new_posted; + + posted = ioread32(&dc2c->wq_ctrl->posted_index); + fetch_index = ioread32(&dc2c->wq_ctrl->fetch_index); + + if (posted == 0xFFFFFFFF || fetch_index == 0xFFFFFFFF) { + /* Hardware surprise removal: return error */ + pr_err("%s: devcmd2 invalid posted or fetch index on cmd %d\n", + pci_name(vdev->pdev), _CMD_N(cmd)); + pr_err("%s: fetch index: %u, posted index: %u\n", + pci_name(vdev->pdev), fetch_index, posted); + + return -ENODEV; + + } + + new_posted = (posted + 1) % DEVCMD2_RING_SIZE; + + if (new_posted == fetch_index) { + pr_err("%s: devcmd2 wq full while issuing cmd %d\n", + pci_name(vdev->pdev), _CMD_N(cmd)); + pr_err("%s: fetch index: %u, posted index: %u\n", + pci_name(vdev->pdev), fetch_index, posted); + return -EBUSY; + + } + dc2c->cmd_ring[posted].cmd = cmd; + dc2c->cmd_ring[posted].flags = 0; + + if ((_CMD_FLAGS(cmd) & _CMD_FLAGS_NOWAIT)) + dc2c->cmd_ring[posted].flags |= DEVCMD2_FNORESULT; + if (_CMD_DIR(cmd) & _CMD_DIR_WRITE) { + for (i = 0; i < VNIC_DEVCMD_NARGS; i++) + dc2c->cmd_ring[posted].args[i] = vdev->args[i]; + + } + + /* Adding write memory barrier prevents compiler and/or CPU + * reordering, thus avoiding descriptor posting before + * descriptor is initialized. Otherwise, hardware can read + * stale descriptor fields. + */ + wmb(); + iowrite32(new_posted, &dc2c->wq_ctrl->posted_index); + + if (dc2c->cmd_ring[posted].flags & DEVCMD2_FNORESULT) + return 0; + + result = dc2c->result + dc2c->next_result; + color = dc2c->color; + + dc2c->next_result++; + if (dc2c->next_result == dc2c->result_size) { + dc2c->next_result = 0; + dc2c->color = dc2c->color ? 0 : 1; + } + + for (delay = 0; delay < wait; delay++) { + udelay(100); + if (result->color == color) { + if (result->error) { + err = -(int) result->error; + if (err != ERR_ECMDUNKNOWN || + cmd != CMD_CAPABILITY) + pr_err("%s:Error %d devcmd %d\n", + pci_name(vdev->pdev), + err, _CMD_N(cmd)); + return err; + } + if (_CMD_DIR(cmd) & _CMD_DIR_READ) { + rmb(); /*prevent reorder while reding result*/ + for (i = 0; i < VNIC_DEVCMD_NARGS; i++) + vdev->args[i] = result->results[i]; + } + return 0; + } + } + + pr_err("%s:Timed out devcmd %d\n", pci_name(vdev->pdev), _CMD_N(cmd)); + + return -ETIMEDOUT; +} + + +int vnic_dev_init_devcmd1(struct vnic_dev *vdev) +{ + vdev->devcmd = vnic_dev_get_res(vdev, RES_TYPE_DEVCMD, 0); + if (!vdev->devcmd) + return -ENODEV; + + vdev->devcmd_rtn = &vnic_dev_cmd1; + return 0; +} + + +int vnic_dev_init_devcmd2(struct vnic_dev *vdev) +{ + int err; + unsigned int fetch_index; + + if (vdev->devcmd2) + return 0; + + vdev->devcmd2 = kzalloc(sizeof(*vdev->devcmd2), GFP_ATOMIC); + if (!vdev->devcmd2) + return -ENOMEM; + + vdev->devcmd2->color = 1; + vdev->devcmd2->result_size = DEVCMD2_RING_SIZE; + err = vnic_wq_devcmd2_alloc(vdev, &vdev->devcmd2->wq, + DEVCMD2_RING_SIZE, DEVCMD2_DESC_SIZE); + if (err) + goto err_free_devcmd2; + + fetch_index = ioread32(&vdev->devcmd2->wq.ctrl->fetch_index); + if (fetch_index == 0xFFFFFFFF) { /* check for hardware gone */ + pr_err("error in devcmd2 init"); + return -ENODEV; + } + + /* + * Don't change fetch_index ever and + * set posted_index same as fetch_index + * when setting up the WQ for devcmd2. + */ + vnic_wq_init_start(&vdev->devcmd2->wq, 0, fetch_index, + fetch_index, 0, 0); + + vnic_wq_enable(&vdev->devcmd2->wq); + + err = vnic_dev_alloc_desc_ring(vdev, &vdev->devcmd2->results_ring, + DEVCMD2_RING_SIZE, DEVCMD2_DESC_SIZE); + if (err) + goto err_free_wq; + + vdev->devcmd2->result = + (struct devcmd2_result *) vdev->devcmd2->results_ring.descs; + vdev->devcmd2->cmd_ring = + (struct vnic_devcmd2 *) vdev->devcmd2->wq.ring.descs; + vdev->devcmd2->wq_ctrl = vdev->devcmd2->wq.ctrl; + vdev->args[0] = (u64) vdev->devcmd2->results_ring.base_addr | + VNIC_PADDR_TARGET; + vdev->args[1] = DEVCMD2_RING_SIZE; + + err = vnic_dev_cmd2(vdev, CMD_INITIALIZE_DEVCMD2, 1000); + if (err) + goto err_free_desc_ring; + + vdev->devcmd_rtn = &vnic_dev_cmd2; + + return 0; + +err_free_desc_ring: + vnic_dev_free_desc_ring(vdev, &vdev->devcmd2->results_ring); +err_free_wq: + vnic_wq_disable(&vdev->devcmd2->wq); + vnic_wq_free(&vdev->devcmd2->wq); +err_free_devcmd2: + kfree(vdev->devcmd2); + vdev->devcmd2 = NULL; + + return err; +} + + +void vnic_dev_deinit_devcmd2(struct vnic_dev *vdev) +{ + vnic_dev_free_desc_ring(vdev, &vdev->devcmd2->results_ring); + vnic_wq_disable(&vdev->devcmd2->wq); + vnic_wq_free(&vdev->devcmd2->wq); + kfree(vdev->devcmd2); + vdev->devcmd2 = NULL; + vdev->devcmd_rtn = &vnic_dev_cmd1; +} + + +int vnic_dev_cmd_no_proxy(struct vnic_dev *vdev, + enum vnic_devcmd_cmd cmd, u64 *a0, u64 *a1, int wait) +{ + int err; + + vdev->args[0] = *a0; + vdev->args[1] = *a1; + + err = (*vdev->devcmd_rtn)(vdev, cmd, wait); + + *a0 = vdev->args[0]; + *a1 = vdev->args[1]; + + return err; +} + + +int vnic_dev_cmd(struct vnic_dev *vdev, enum vnic_devcmd_cmd cmd, + u64 *a0, u64 *a1, int wait) +{ + memset(vdev->args, 0, sizeof(vdev->args)); + + switch (vdev->proxy) { + case PROXY_NONE: + default: + return vnic_dev_cmd_no_proxy(vdev, cmd, a0, a1, wait); + } +} + + int vnic_dev_fw_info(struct vnic_dev *vdev, struct vnic_devcmd_fw_info **fw_info) { @@ -664,6 +907,8 @@ void vnic_dev_unregister(struct vnic_dev *vdev) dma_free_coherent(&vdev->pdev->dev, sizeof(struct vnic_devcmd_fw_info), vdev->fw_info, vdev->fw_info_pa); + if (vdev->devcmd2) + vnic_dev_deinit_devcmd2(vdev); kfree(vdev); } } @@ -683,13 +928,26 @@ struct vnic_dev *vnic_dev_register(struct vnic_dev *vdev, if (vnic_dev_discover_res(vdev, bar)) goto err_out; - vdev->devcmd = vnic_dev_get_res(vdev, RES_TYPE_DEVCMD, 0); - if (!vdev->devcmd) - goto err_out; - return vdev; err_out: vnic_dev_unregister(vdev); return NULL; } + +int vnic_dev_cmd_init(struct vnic_dev *vdev) +{ + int err; + void *p; + + p = vnic_dev_get_res(vdev, RES_TYPE_DEVCMD2, 0); + if (p) { + pr_err("fnic: DEVCMD2 resource found!\n"); + err = vnic_dev_init_devcmd2(vdev); + } else { + pr_err("fnic: DEVCMD2 not found, fall back to Devcmd\n"); + err = vnic_dev_init_devcmd1(vdev); + } + + return err; +} diff --git a/drivers/scsi/fnic/vnic_dev.h b/drivers/scsi/fnic/vnic_dev.h index 40d4195f562b..ef5309a5df5d 100644 --- a/drivers/scsi/fnic/vnic_dev.h +++ b/drivers/scsi/fnic/vnic_dev.h @@ -36,6 +36,7 @@ #define vnic_dev_fw_info fnic_dev_fw_info #define vnic_dev_spec fnic_dev_spec #define vnic_dev_stats_clear fnic_dev_stats_clear +#define vnic_dev_cmd_init fnic_dev_cmd_init #define vnic_dev_stats_dump fnic_dev_stats_dump #define vnic_dev_hang_notify fnic_dev_hang_notify #define vnic_dev_packet_filter fnic_dev_packet_filter @@ -128,6 +129,7 @@ int vnic_dev_fw_info(struct vnic_dev *vdev, int vnic_dev_spec(struct vnic_dev *vdev, unsigned int offset, unsigned int size, void *value); int vnic_dev_stats_clear(struct vnic_dev *vdev); +int vnic_dev_cmd_init(struct vnic_dev *vdev); int vnic_dev_stats_dump(struct vnic_dev *vdev, struct vnic_stats **stats); int vnic_dev_hang_notify(struct vnic_dev *vdev); void vnic_dev_packet_filter(struct vnic_dev *vdev, int directed, int multicast, diff --git a/drivers/scsi/fnic/vnic_devcmd.h b/drivers/scsi/fnic/vnic_devcmd.h index 3e2fcbda6aed..c5dde556dc7c 100644 --- a/drivers/scsi/fnic/vnic_devcmd.h +++ b/drivers/scsi/fnic/vnic_devcmd.h @@ -170,7 +170,8 @@ enum vnic_devcmd_cmd { /* variant of CMD_INIT, with provisioning info * (u64)a0=paddr of vnic_devcmd_provinfo - * (u32)a1=sizeof provision info */ + * (u32)a1=sizeof provision info + */ CMD_INIT_PROV_INFO = _CMDC(_CMD_DIR_WRITE, _CMD_VTYPE_ENET, 27), /* enable virtual link */ @@ -262,12 +263,132 @@ enum vnic_devcmd_cmd { * non-zero for resetting vlan to the default * out: (u16)a0=old default vlan */ - CMD_SET_DEFAULT_VLAN = _CMDC(_CMD_DIR_RW, _CMD_VTYPE_ALL, 46) + CMD_SET_DEFAULT_VLAN = _CMDC(_CMD_DIR_RW, _CMD_VTYPE_ALL, 46), + + /* init_prov_info2: + * Variant of CMD_INIT_PROV_INFO, where it will not try to enable + * the vnic until CMD_ENABLE2 is issued. + * (u64)a0=paddr of vnic_devcmd_provinfo + * (u32)a1=sizeof provision info + */ + CMD_INIT_PROV_INFO2 = _CMDC(_CMD_DIR_WRITE, _CMD_VTYPE_ENET, 47), + + /* enable2: + * (u32)a0=0 ==> standby + * =CMD_ENABLE2_ACTIVE ==> active + */ + CMD_ENABLE2 = _CMDC(_CMD_DIR_WRITE, _CMD_VTYPE_ENET, 48), + + /* + * cmd_status: + * Returns the status of the specified command + * Input: + * a0 = command for which status is being queried. + * Possible values are: + * CMD_SOFT_RESET + * CMD_HANG_RESET + * CMD_OPEN + * CMD_INIT + * CMD_INIT_PROV_INFO + * CMD_DEINIT + * CMD_INIT_PROV_INFO2 + * CMD_ENABLE2 + * Output: + * if status == STAT_ERROR + * a0 = ERR_ENOTSUPPORTED - status for command in a0 is + * not supported + * if status == STAT_NONE + * a0 = status of the devcmd specified in a0 as follows. + * ERR_SUCCESS - command in a0 completed successfully + * ERR_EINPROGRESS - command in a0 is still in progress + */ + CMD_STATUS = _CMDC(_CMD_DIR_RW, _CMD_VTYPE_ALL, 49), + + /* + * Returns interrupt coalescing timer conversion factors. + * After calling this devcmd, ENIC driver can convert + * interrupt coalescing timer in usec into CPU cycles as follows: + * + * intr_timer_cycles = intr_timer_usec * multiplier / divisor + * + * Interrupt coalescing timer in usecs can be be converted/obtained + * from CPU cycles as follows: + * + * intr_timer_usec = intr_timer_cycles * divisor / multiplier + * + * in: none + * out: (u32)a0 = multiplier + * (u32)a1 = divisor + * (u32)a2 = maximum timer value in usec + */ + CMD_INTR_COAL_CONVERT = _CMDC(_CMD_DIR_READ, _CMD_VTYPE_ALL, 50), + + /* + * ISCSI DUMP API: + * in: (u64)a0=paddr of the param or param itself + * (u32)a1=ISCSI_CMD_xxx + */ + CMD_ISCSI_DUMP_REQ = _CMDC(_CMD_DIR_WRITE, _CMD_VTYPE_ALL, 51), + + /* + * ISCSI DUMP STATUS API: + * in: (u32)a0=cmd tag + * in: (u32)a1=ISCSI_CMD_xxx + * out: (u32)a0=cmd status + */ + CMD_ISCSI_DUMP_STATUS = _CMDC(_CMD_DIR_RW, _CMD_VTYPE_ALL, 52), + + /* + * Subvnic migration from MQ <--> VF. + * Enable the LIF migration from MQ to VF and vice versa. MQ and VF + * indexes are statically bound at the time of initialization. + * Based on the + * direction of migration, the resources of either MQ or the VF shall + * be attached to the LIF. + * in: (u32)a0=Direction of Migration + * 0=> Migrate to VF + * 1=> Migrate to MQ + * (u32)a1=VF index (MQ index) + */ + CMD_MIGRATE_SUBVNIC = _CMDC(_CMD_DIR_WRITE, _CMD_VTYPE_ENET, 53), + + /* + * Register / Deregister the notification block for MQ subvnics + * in: + * (u64)a0=paddr to notify (set paddr=0 to unset) + * (u32)a1 & 0x00000000ffffffff=sizeof(struct vnic_devcmd_notify) + * (u16)a1 & 0x0000ffff00000000=intr num (-1 for no intr) + * out: + * (u32)a1 = effective size + */ + CMD_SUBVNIC_NOTIFY = _CMDC(_CMD_DIR_RW, _CMD_VTYPE_ALL, 54), + + /* + * Set the predefined mac address as default + * in: + * (u48)a0=mac addr + */ + CMD_SET_MAC_ADDR = _CMDC(_CMD_DIR_WRITE, _CMD_VTYPE_ENET, 55), + + /* Update the provisioning info of the given VIF + * (u64)a0=paddr of vnic_devcmd_provinfo + * (u32)a1=sizeof provision info + */ + CMD_PROV_INFO_UPDATE = _CMDC(_CMD_DIR_WRITE, _CMD_VTYPE_ENET, 56), + + /* + * Initialization for the devcmd2 interface. + * in: (u64) a0=host result buffer physical address + * in: (u16) a1=number of entries in result buffer + */ + CMD_INITIALIZE_DEVCMD2 = _CMDC(_CMD_DIR_WRITE, _CMD_VTYPE_ALL, 57) }; /* flags for CMD_OPEN */ #define CMD_OPENF_OPROM 0x1 /* open coming from option rom */ +#define CMD_OPENF_RQ_ENABLE_THEN_POST 0x2 + /* flags for CMD_INIT */ #define CMD_INITF_DEFAULT_MAC 0x1 /* init with default mac addr */ @@ -345,4 +466,39 @@ struct vnic_devcmd { u64 args[VNIC_DEVCMD_NARGS]; /* RW cmd args (little-endian) */ }; +/* + * Version 2 of the interface. + * + * Some things are carried over, notably the vnic_devcmd_cmd enum. + */ + +/* + * Flags for vnic_devcmd2.flags + */ + +#define DEVCMD2_FNORESULT 0x1 /* Don't copy result to host */ + +#define VNIC_DEVCMD2_NARGS VNIC_DEVCMD_NARGS + +struct vnic_devcmd2 { + u16 pad; + u16 flags; + u32 cmd; /* same command #defines as original */ + u64 args[VNIC_DEVCMD2_NARGS]; +}; + +#define VNIC_DEVCMD2_NRESULTS VNIC_DEVCMD_NARGS +struct devcmd2_result { + u64 results[VNIC_DEVCMD2_NRESULTS]; + u32 pad; + u16 completed_index; /* into copy WQ */ + u8 error; /* same error codes as original */ + u8 color; /* 0 or 1 as with completion queues */ +}; + +#define DEVCMD2_RING_SIZE 32 +#define DEVCMD2_DESC_SIZE 128 + +#define DEVCMD2_RESULTS_SIZE_MAX ((1 << 16) - 1) + #endif /* _VNIC_DEVCMD_H_ */ diff --git a/drivers/scsi/fnic/vnic_resource.h b/drivers/scsi/fnic/vnic_resource.h index 2d842f79d41a..7c6163f73bd3 100644 --- a/drivers/scsi/fnic/vnic_resource.h +++ b/drivers/scsi/fnic/vnic_resource.h @@ -41,6 +41,13 @@ enum vnic_res_type { RES_TYPE_RSVD7, RES_TYPE_DEVCMD, /* Device command region */ RES_TYPE_PASS_THRU_PAGE, /* Pass-thru page */ + RES_TYPE_SUBVNIC, /* subvnic resource type */ + RES_TYPE_MQ_WQ, /* MQ Work queues */ + RES_TYPE_MQ_RQ, /* MQ Receive queues */ + RES_TYPE_MQ_CQ, /* MQ Completion queues */ + RES_TYPE_DEPRECATED1, /* Old version of devcmd 2 */ + RES_TYPE_DEPRECATED2, /* Old version of devcmd 2 */ + RES_TYPE_DEVCMD2, /* Device control region */ RES_TYPE_MAX, /* Count of resource types */ }; diff --git a/drivers/scsi/fnic/vnic_rq.c b/drivers/scsi/fnic/vnic_rq.c index fd2068f5ae16..6a35b1be0032 100644 --- a/drivers/scsi/fnic/vnic_rq.c +++ b/drivers/scsi/fnic/vnic_rq.c @@ -27,12 +27,9 @@ static int vnic_rq_alloc_bufs(struct vnic_rq *rq) { struct vnic_rq_buf *buf; - struct vnic_dev *vdev; unsigned int i, j, count = rq->ring.desc_count; unsigned int blks = VNIC_RQ_BUF_BLKS_NEEDED(count); - vdev = rq->vdev; - for (i = 0; i < blks; i++) { rq->bufs[i] = kzalloc(VNIC_RQ_BUF_BLK_SZ, GFP_ATOMIC); if (!rq->bufs[i]) { @@ -171,7 +168,7 @@ void vnic_rq_clean(struct vnic_rq *rq, struct vnic_rq_buf *buf; u32 fetch_index; - BUG_ON(ioread32(&rq->ctrl->enable)); + WARN_ON(ioread32(&rq->ctrl->enable)); buf = rq->to_clean; diff --git a/drivers/scsi/fnic/vnic_wq.c b/drivers/scsi/fnic/vnic_wq.c index a414135460db..015af2cdabaf 100644 --- a/drivers/scsi/fnic/vnic_wq.c +++ b/drivers/scsi/fnic/vnic_wq.c @@ -24,15 +24,32 @@ #include "vnic_dev.h" #include "vnic_wq.h" + +int vnic_wq_get_ctrl(struct vnic_dev *vdev, struct vnic_wq *wq, + unsigned int index, enum vnic_res_type res_type) +{ + wq->ctrl = vnic_dev_get_res(vdev, res_type, index); + + if (!wq->ctrl) + return -EINVAL; + + return 0; +} + + +int vnic_wq_alloc_ring(struct vnic_dev *vdev, struct vnic_wq *wq, + unsigned int desc_count, unsigned int desc_size) +{ + return vnic_dev_alloc_desc_ring(vdev, &wq->ring, desc_count, desc_size); +} + + static int vnic_wq_alloc_bufs(struct vnic_wq *wq) { struct vnic_wq_buf *buf; - struct vnic_dev *vdev; unsigned int i, j, count = wq->ring.desc_count; unsigned int blks = VNIC_WQ_BUF_BLKS_NEEDED(count); - vdev = wq->vdev; - for (i = 0; i < blks; i++) { wq->bufs[i] = kzalloc(VNIC_WQ_BUF_BLK_SZ, GFP_ATOMIC); if (!wq->bufs[i]) { @@ -111,6 +128,52 @@ int vnic_wq_alloc(struct vnic_dev *vdev, struct vnic_wq *wq, unsigned int index, return 0; } + +int vnic_wq_devcmd2_alloc(struct vnic_dev *vdev, struct vnic_wq *wq, + unsigned int desc_count, unsigned int desc_size) +{ + int err; + + wq->index = 0; + wq->vdev = vdev; + + err = vnic_wq_get_ctrl(vdev, wq, 0, RES_TYPE_DEVCMD2); + if (err) { + pr_err("Failed to get devcmd2 resource\n"); + return err; + } + vnic_wq_disable(wq); + + err = vnic_wq_alloc_ring(vdev, wq, desc_count, desc_size); + if (err) + return err; + return 0; +} + +void vnic_wq_init_start(struct vnic_wq *wq, unsigned int cq_index, + unsigned int fetch_index, unsigned int posted_index, + unsigned int error_interrupt_enable, + unsigned int error_interrupt_offset) +{ + u64 paddr; + unsigned int count = wq->ring.desc_count; + + paddr = (u64)wq->ring.base_addr | VNIC_PADDR_TARGET; + writeq(paddr, &wq->ctrl->ring_base); + iowrite32(count, &wq->ctrl->ring_size); + iowrite32(fetch_index, &wq->ctrl->fetch_index); + iowrite32(posted_index, &wq->ctrl->posted_index); + iowrite32(cq_index, &wq->ctrl->cq_index); + iowrite32(error_interrupt_enable, &wq->ctrl->error_interrupt_enable); + iowrite32(error_interrupt_offset, &wq->ctrl->error_interrupt_offset); + iowrite32(0, &wq->ctrl->error_status); + + wq->to_use = wq->to_clean = + &wq->bufs[fetch_index / VNIC_WQ_BUF_BLK_ENTRIES] + [fetch_index % VNIC_WQ_BUF_BLK_ENTRIES]; +} + + void vnic_wq_init(struct vnic_wq *wq, unsigned int cq_index, unsigned int error_interrupt_enable, unsigned int error_interrupt_offset) diff --git a/drivers/scsi/fnic/vnic_wq.h b/drivers/scsi/fnic/vnic_wq.h index 5cd094f79281..5d1e0a44d94a 100644 --- a/drivers/scsi/fnic/vnic_wq.h +++ b/drivers/scsi/fnic/vnic_wq.h @@ -33,6 +33,8 @@ #define vnic_wq_service fnic_wq_service #define vnic_wq_free fnic_wq_free #define vnic_wq_alloc fnic_wq_alloc +#define vnic_wq_devcmd2_alloc fnic_wq_devcmd2_alloc +#define vnic_wq_init_start fnic_wq_init_start #define vnic_wq_init fnic_wq_init #define vnic_wq_error_status fnic_wq_error_status #define vnic_wq_enable fnic_wq_enable @@ -163,6 +165,12 @@ static inline void vnic_wq_service(struct vnic_wq *wq, void vnic_wq_free(struct vnic_wq *wq); int vnic_wq_alloc(struct vnic_dev *vdev, struct vnic_wq *wq, unsigned int index, unsigned int desc_count, unsigned int desc_size); +int vnic_wq_devcmd2_alloc(struct vnic_dev *vdev, struct vnic_wq *wq, + unsigned int desc_count, unsigned int desc_size); +void vnic_wq_init_start(struct vnic_wq *wq, unsigned int cq_index, + unsigned int fetch_index, unsigned int posted_index, + unsigned int error_interrupt_enable, + unsigned int error_interrupt_offset); void vnic_wq_init(struct vnic_wq *wq, unsigned int cq_index, unsigned int error_interrupt_enable, unsigned int error_interrupt_offset); diff --git a/drivers/scsi/gdth.c b/drivers/scsi/gdth.c index 194c294f9b6c..e7f1dd4f3b66 100644 --- a/drivers/scsi/gdth.c +++ b/drivers/scsi/gdth.c @@ -1,6 +1,6 @@ /************************************************************************ * Linux driver for * - * ICP vortex GmbH: GDT ISA/EISA/PCI Disk Array Controllers * + * ICP vortex GmbH: GDT PCI Disk Array Controllers * * Intel Corporation: Storage RAID Controllers * * * * gdth.c * @@ -32,15 +32,10 @@ ************************************************************************/ /* All GDT Disk Array Controllers are fully supported by this driver. - * This includes the PCI/EISA/ISA SCSI Disk Array Controllers and the + * This includes the PCI SCSI Disk Array Controllers and the * PCI Fibre Channel Disk Array Controllers. See gdth.h for a complete * list of all controller types. * - * If you have one or more GDT3000/3020 EISA controllers with - * controller BIOS disabled, you have to set the IRQ values with the - * command line option "gdth=irq1,irq2,...", where the irq1,irq2,... are - * the IRQ values for the EISA controllers. - * * After the optional list of IRQ values, other possible * command line options are: * disable:Y disable driver @@ -61,14 +56,12 @@ * access a shared resource from several nodes, * appropriate controller firmware required * shared_access:N enable driver reserve/release protocol - * probe_eisa_isa:Y scan for EISA/ISA controllers - * probe_eisa_isa:N do not scan for EISA/ISA controllers * force_dma32:Y use only 32 bit DMA mode * force_dma32:N use 64 bit DMA mode, if supported * * The default values are: "gdth=disable:N,reserve_mode:1,reverse_scan:N, * max_ids:127,rescan:N,hdr_channel:0, - * shared_access:Y,probe_eisa_isa:N,force_dma32:N". + * shared_access:Y,force_dma32:N". * Here is another example: "gdth=reserve_list:0,1,2,0,0,1,3,0,rescan:Y". * * When loading the gdth driver as a module, the same options are available. @@ -79,7 +72,7 @@ * * Default: "modprobe gdth disable=0 reserve_mode=1 reverse_scan=0 * max_ids=127 rescan=0 hdr_channel=0 shared_access=0 - * probe_eisa_isa=0 force_dma32=0" + * force_dma32=0" * The other example: "modprobe gdth reserve_list=0,1,2,0,0,1,3,0 rescan=1". */ @@ -96,10 +89,6 @@ * phase: unused */ - -/* interrupt coalescing */ -/* #define INT_COAL */ - /* statistics */ #define GDTH_STATISTICS @@ -122,10 +111,6 @@ #include <linux/list.h> #include <linux/mutex.h> #include <linux/slab.h> - -#ifdef GDTH_RTC -#include <linux/mc146818rtc.h> -#endif #include <linux/reboot.h> #include <asm/dma.h> @@ -192,79 +177,9 @@ static void gdth_scsi_done(struct scsi_cmnd *scp); #ifdef DEBUG_GDTH static u8 DebugState = DEBUG_GDTH; - -#ifdef __SERIAL__ -#define MAX_SERBUF 160 -static void ser_init(void); -static void ser_puts(char *str); -static void ser_putc(char c); -static int ser_printk(const char *fmt, ...); -static char strbuf[MAX_SERBUF+1]; -#ifdef __COM2__ -#define COM_BASE 0x2f8 -#else -#define COM_BASE 0x3f8 -#endif -static void ser_init() -{ - unsigned port=COM_BASE; - - outb(0x80,port+3); - outb(0,port+1); - /* 19200 Baud, if 9600: outb(12,port) */ - outb(6, port); - outb(3,port+3); - outb(0,port+1); - /* - ser_putc('I'); - ser_putc(' '); - */ -} - -static void ser_puts(char *str) -{ - char *ptr; - - ser_init(); - for (ptr=str;*ptr;++ptr) - ser_putc(*ptr); -} - -static void ser_putc(char c) -{ - unsigned port=COM_BASE; - - while ((inb(port+5) & 0x20)==0); - outb(c,port); - if (c==0x0a) - { - while ((inb(port+5) & 0x20)==0); - outb(0x0d,port); - } -} - -static int ser_printk(const char *fmt, ...) -{ - va_list args; - int i; - - va_start(args,fmt); - i = vsprintf(strbuf,fmt,args); - ser_puts(strbuf); - va_end(args); - return i; -} - -#define TRACE(a) {if (DebugState==1) {ser_printk a;}} -#define TRACE2(a) {if (DebugState==1 || DebugState==2) {ser_printk a;}} -#define TRACE3(a) {if (DebugState!=0) {ser_printk a;}} - -#else /* !__SERIAL__ */ #define TRACE(a) {if (DebugState==1) {printk a;}} #define TRACE2(a) {if (DebugState==1 || DebugState==2) {printk a;}} #define TRACE3(a) {if (DebugState!=0) {printk a;}} -#endif - #else /* !DEBUG */ #define TRACE(a) #define TRACE2(a) @@ -273,9 +188,6 @@ static int ser_printk(const char *fmt, ...) #ifdef GDTH_STATISTICS static u32 max_rq=0, max_index=0, max_sg=0; -#ifdef INT_COAL -static u32 max_int_coal=0; -#endif static u32 act_ints=0, act_ios=0, act_stats=0, act_rq=0; static struct timer_list gdth_timer; #endif @@ -286,12 +198,6 @@ static struct timer_list gdth_timer; #define BUS_L2P(a,b) ((b)>(a)->virt_bus ? (b-1):(b)) -#ifdef CONFIG_ISA -static u8 gdth_drq_tab[4] = {5,6,7,7}; /* DRQ table */ -#endif -#if defined(CONFIG_EISA) || defined(CONFIG_ISA) -static u8 gdth_irq_tab[6] = {0,10,11,12,14,0}; /* IRQ table */ -#endif static u8 gdth_polling; /* polling if TRUE */ static int gdth_ctr_count = 0; /* controller count */ static LIST_HEAD(gdth_instances); /* controller list */ @@ -325,10 +231,6 @@ static u8 gdth_direction_tab[0x100] = { }; /* LILO and modprobe/insmod parameters */ -/* IRQ list for GDT3000/3020 EISA controllers */ -static int irq[MAXHA] __initdata = -{0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, - 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff}; /* disable driver flag */ static int disable __initdata = 0; /* reserve flag */ @@ -348,13 +250,10 @@ static int max_ids = MAXID; static int rescan = 0; /* shared access */ static int shared_access = 1; -/* enable support for EISA and ISA controllers */ -static int probe_eisa_isa = 0; /* 64 bit DMA mode, support for drives > 2 TB, if force_dma32 = 0 */ static int force_dma32 = 0; /* parameters for modprobe/insmod */ -module_param_hw_array(irq, int, irq, NULL, 0); module_param(disable, int, 0); module_param(reserve_mode, int, 0); module_param_array(reserve_list, int, NULL, 0); @@ -363,7 +262,6 @@ module_param(hdr_channel, int, 0); module_param(max_ids, int, 0); module_param(rescan, int, 0); module_param(shared_access, int, 0); -module_param(probe_eisa_isa, int, 0); module_param(force_dma32, int, 0); MODULE_AUTHOR("Achim Leubner"); MODULE_LICENSE("GPL"); @@ -515,45 +413,6 @@ static void gdth_eval_mapping(u32 size, u32 *cyls, int *heads, int *secs) } } -/* controller search and initialization functions */ -#ifdef CONFIG_EISA -static int __init gdth_search_eisa(u16 eisa_adr) -{ - u32 id; - - TRACE(("gdth_search_eisa() adr. %x\n",eisa_adr)); - id = inl(eisa_adr+ID0REG); - if (id == GDT3A_ID || id == GDT3B_ID) { /* GDT3000A or GDT3000B */ - if ((inb(eisa_adr+EISAREG) & 8) == 0) - return 0; /* not EISA configured */ - return 1; - } - if (id == GDT3_ID) /* GDT3000 */ - return 1; - - return 0; -} -#endif /* CONFIG_EISA */ - -#ifdef CONFIG_ISA -static int __init gdth_search_isa(u32 bios_adr) -{ - void __iomem *addr; - u32 id; - - TRACE(("gdth_search_isa() bios adr. %x\n",bios_adr)); - if ((addr = ioremap(bios_adr+BIOS_ID_OFFS, sizeof(u32))) != NULL) { - id = readl(addr); - iounmap(addr); - if (id == GDT2_ID) /* GDT2000 */ - return 1; - } - return 0; -} -#endif /* CONFIG_ISA */ - -#ifdef CONFIG_PCI - static bool gdth_search_vortex(u16 device) { if (device <= PCI_DEVICE_ID_VORTEX_GDT6555) @@ -656,204 +515,7 @@ static int gdth_pci_init_one(struct pci_dev *pdev, return 0; } -#endif /* CONFIG_PCI */ -#ifdef CONFIG_EISA -static int __init gdth_init_eisa(u16 eisa_adr,gdth_ha_str *ha) -{ - u32 retries,id; - u8 prot_ver,eisacf,i,irq_found; - - TRACE(("gdth_init_eisa() adr. %x\n",eisa_adr)); - - /* disable board interrupts, deinitialize services */ - outb(0xff,eisa_adr+EDOORREG); - outb(0x00,eisa_adr+EDENABREG); - outb(0x00,eisa_adr+EINTENABREG); - - outb(0xff,eisa_adr+LDOORREG); - retries = INIT_RETRIES; - gdth_delay(20); - while (inb(eisa_adr+EDOORREG) != 0xff) { - if (--retries == 0) { - printk("GDT-EISA: Initialization error (DEINIT failed)\n"); - return 0; - } - gdth_delay(1); - TRACE2(("wait for DEINIT: retries=%d\n",retries)); - } - prot_ver = inb(eisa_adr+MAILBOXREG); - outb(0xff,eisa_adr+EDOORREG); - if (prot_ver != PROTOCOL_VERSION) { - printk("GDT-EISA: Illegal protocol version\n"); - return 0; - } - ha->bmic = eisa_adr; - ha->brd_phys = (u32)eisa_adr >> 12; - - outl(0,eisa_adr+MAILBOXREG); - outl(0,eisa_adr+MAILBOXREG+4); - outl(0,eisa_adr+MAILBOXREG+8); - outl(0,eisa_adr+MAILBOXREG+12); - - /* detect IRQ */ - if ((id = inl(eisa_adr+ID0REG)) == GDT3_ID) { - ha->oem_id = OEM_ID_ICP; - ha->type = GDT_EISA; - ha->stype = id; - outl(1,eisa_adr+MAILBOXREG+8); - outb(0xfe,eisa_adr+LDOORREG); - retries = INIT_RETRIES; - gdth_delay(20); - while (inb(eisa_adr+EDOORREG) != 0xfe) { - if (--retries == 0) { - printk("GDT-EISA: Initialization error (get IRQ failed)\n"); - return 0; - } - gdth_delay(1); - } - ha->irq = inb(eisa_adr+MAILBOXREG); - outb(0xff,eisa_adr+EDOORREG); - TRACE2(("GDT3000/3020: IRQ=%d\n",ha->irq)); - /* check the result */ - if (ha->irq == 0) { - TRACE2(("Unknown IRQ, use IRQ table from cmd line !\n")); - for (i = 0, irq_found = FALSE; - i < MAXHA && irq[i] != 0xff; ++i) { - if (irq[i]==10 || irq[i]==11 || irq[i]==12 || irq[i]==14) { - irq_found = TRUE; - break; - } - } - if (irq_found) { - ha->irq = irq[i]; - irq[i] = 0; - printk("GDT-EISA: Can not detect controller IRQ,\n"); - printk("Use IRQ setting from command line (IRQ = %d)\n", - ha->irq); - } else { - printk("GDT-EISA: Initialization error (unknown IRQ), Enable\n"); - printk("the controller BIOS or use command line parameters\n"); - return 0; - } - } - } else { - eisacf = inb(eisa_adr+EISAREG) & 7; - if (eisacf > 4) /* level triggered */ - eisacf -= 4; - ha->irq = gdth_irq_tab[eisacf]; - ha->oem_id = OEM_ID_ICP; - ha->type = GDT_EISA; - ha->stype = id; - } - - ha->dma64_support = 0; - return 1; -} -#endif /* CONFIG_EISA */ - -#ifdef CONFIG_ISA -static int __init gdth_init_isa(u32 bios_adr,gdth_ha_str *ha) -{ - register gdt2_dpram_str __iomem *dp2_ptr; - int i; - u8 irq_drq,prot_ver; - u32 retries; - - TRACE(("gdth_init_isa() bios adr. %x\n",bios_adr)); - - ha->brd = ioremap(bios_adr, sizeof(gdt2_dpram_str)); - if (ha->brd == NULL) { - printk("GDT-ISA: Initialization error (DPMEM remap error)\n"); - return 0; - } - dp2_ptr = ha->brd; - writeb(1, &dp2_ptr->io.memlock); /* switch off write protection */ - /* reset interface area */ - memset_io(&dp2_ptr->u, 0, sizeof(dp2_ptr->u)); - if (readl(&dp2_ptr->u) != 0) { - printk("GDT-ISA: Initialization error (DPMEM write error)\n"); - iounmap(ha->brd); - return 0; - } - - /* disable board interrupts, read DRQ and IRQ */ - writeb(0xff, &dp2_ptr->io.irqdel); - writeb(0x00, &dp2_ptr->io.irqen); - writeb(0x00, &dp2_ptr->u.ic.S_Status); - writeb(0x00, &dp2_ptr->u.ic.Cmd_Index); - - irq_drq = readb(&dp2_ptr->io.rq); - for (i=0; i<3; ++i) { - if ((irq_drq & 1)==0) - break; - irq_drq >>= 1; - } - ha->drq = gdth_drq_tab[i]; - - irq_drq = readb(&dp2_ptr->io.rq) >> 3; - for (i=1; i<5; ++i) { - if ((irq_drq & 1)==0) - break; - irq_drq >>= 1; - } - ha->irq = gdth_irq_tab[i]; - - /* deinitialize services */ - writel(bios_adr, &dp2_ptr->u.ic.S_Info[0]); - writeb(0xff, &dp2_ptr->u.ic.S_Cmd_Indx); - writeb(0, &dp2_ptr->io.event); - retries = INIT_RETRIES; - gdth_delay(20); - while (readb(&dp2_ptr->u.ic.S_Status) != 0xff) { - if (--retries == 0) { - printk("GDT-ISA: Initialization error (DEINIT failed)\n"); - iounmap(ha->brd); - return 0; - } - gdth_delay(1); - } - prot_ver = (u8)readl(&dp2_ptr->u.ic.S_Info[0]); - writeb(0, &dp2_ptr->u.ic.Status); - writeb(0xff, &dp2_ptr->io.irqdel); - if (prot_ver != PROTOCOL_VERSION) { - printk("GDT-ISA: Illegal protocol version\n"); - iounmap(ha->brd); - return 0; - } - - ha->oem_id = OEM_ID_ICP; - ha->type = GDT_ISA; - ha->ic_all_size = sizeof(dp2_ptr->u); - ha->stype= GDT2_ID; - ha->brd_phys = bios_adr >> 4; - - /* special request to controller BIOS */ - writel(0x00, &dp2_ptr->u.ic.S_Info[0]); - writel(0x00, &dp2_ptr->u.ic.S_Info[1]); - writel(0x01, &dp2_ptr->u.ic.S_Info[2]); - writel(0x00, &dp2_ptr->u.ic.S_Info[3]); - writeb(0xfe, &dp2_ptr->u.ic.S_Cmd_Indx); - writeb(0, &dp2_ptr->io.event); - retries = INIT_RETRIES; - gdth_delay(20); - while (readb(&dp2_ptr->u.ic.S_Status) != 0xfe) { - if (--retries == 0) { - printk("GDT-ISA: Initialization error\n"); - iounmap(ha->brd); - return 0; - } - gdth_delay(1); - } - writeb(0, &dp2_ptr->u.ic.Status); - writeb(0xff, &dp2_ptr->io.irqdel); - - ha->dma64_support = 0; - return 1; -} -#endif /* CONFIG_ISA */ - -#ifdef CONFIG_PCI static int gdth_init_pci(struct pci_dev *pdev, gdth_pci_str *pcistr, gdth_ha_str *ha) { @@ -1228,30 +890,19 @@ static int gdth_init_pci(struct pci_dev *pdev, gdth_pci_str *pcistr, return 1; } -#endif /* CONFIG_PCI */ /* controller protocol functions */ static void gdth_enable_int(gdth_ha_str *ha) { unsigned long flags; - gdt2_dpram_str __iomem *dp2_ptr; gdt6_dpram_str __iomem *dp6_ptr; gdt6m_dpram_str __iomem *dp6m_ptr; TRACE(("gdth_enable_int() hanum %d\n",ha->hanum)); spin_lock_irqsave(&ha->smp_lock, flags); - if (ha->type == GDT_EISA) { - outb(0xff, ha->bmic + EDOORREG); - outb(0xff, ha->bmic + EDENABREG); - outb(0x01, ha->bmic + EINTENABREG); - } else if (ha->type == GDT_ISA) { - dp2_ptr = ha->brd; - writeb(1, &dp2_ptr->io.irqdel); - writeb(0, &dp2_ptr->u.ic.Cmd_Index); - writeb(1, &dp2_ptr->io.irqen); - } else if (ha->type == GDT_PCI) { + if (ha->type == GDT_PCI) { dp6_ptr = ha->brd; writeb(1, &dp6_ptr->io.irqdel); writeb(0, &dp6_ptr->u.ic.Cmd_Index); @@ -1275,12 +926,7 @@ static u8 gdth_get_status(gdth_ha_str *ha) TRACE(("gdth_get_status() irq %d ctr_count %d\n", ha->irq, gdth_ctr_count)); - if (ha->type == GDT_EISA) - IStatus = inb((u16)ha->bmic + EDOORREG); - else if (ha->type == GDT_ISA) - IStatus = - readb(&((gdt2_dpram_str __iomem *)ha->brd)->u.ic.Cmd_Index); - else if (ha->type == GDT_PCI) + if (ha->type == GDT_PCI) IStatus = readb(&((gdt6_dpram_str __iomem *)ha->brd)->u.ic.Cmd_Index); else if (ha->type == GDT_PCINEW) @@ -1298,11 +944,7 @@ static int gdth_test_busy(gdth_ha_str *ha) TRACE(("gdth_test_busy() hanum %d\n", ha->hanum)); - if (ha->type == GDT_EISA) - gdtsema0 = (int)inb(ha->bmic + SEMA0REG); - else if (ha->type == GDT_ISA) - gdtsema0 = (int)readb(&((gdt2_dpram_str __iomem *)ha->brd)->u.ic.Sema0); - else if (ha->type == GDT_PCI) + if (ha->type == GDT_PCI) gdtsema0 = (int)readb(&((gdt6_dpram_str __iomem *)ha->brd)->u.ic.Sema0); else if (ha->type == GDT_PCINEW) gdtsema0 = (int)inb(PTR2USHORT(&ha->plx->sema0_reg)); @@ -1336,11 +978,7 @@ static void gdth_set_sema0(gdth_ha_str *ha) { TRACE(("gdth_set_sema0() hanum %d\n", ha->hanum)); - if (ha->type == GDT_EISA) { - outb(1, ha->bmic + SEMA0REG); - } else if (ha->type == GDT_ISA) { - writeb(1, &((gdt2_dpram_str __iomem *)ha->brd)->u.ic.Sema0); - } else if (ha->type == GDT_PCI) { + if (ha->type == GDT_PCI) { writeb(1, &((gdt6_dpram_str __iomem *)ha->brd)->u.ic.Sema0); } else if (ha->type == GDT_PCINEW) { outb(1, PTR2USHORT(&ha->plx->sema0_reg)); @@ -1356,7 +994,6 @@ static void gdth_copy_command(gdth_ha_str *ha) register gdt6m_dpram_str __iomem *dp6m_ptr; register gdt6c_dpram_str __iomem *dp6c_ptr; gdt6_dpram_str __iomem *dp6_ptr; - gdt2_dpram_str __iomem *dp2_ptr; u16 cp_count,dp_offset,cmd_no; TRACE(("gdth_copy_command() hanum %d\n", ha->hanum)); @@ -1367,8 +1004,6 @@ static void gdth_copy_command(gdth_ha_str *ha) cmd_ptr = ha->pccb; ++ha->cmd_cnt; - if (ha->type == GDT_EISA) - return; /* no DPMEM, no copy */ /* set cpcount dword aligned */ if (cp_count & 3) @@ -1377,14 +1012,7 @@ static void gdth_copy_command(gdth_ha_str *ha) ha->cmd_offs_dpmem += cp_count; /* set offset and service, copy command to DPMEM */ - if (ha->type == GDT_ISA) { - dp2_ptr = ha->brd; - writew(dp_offset + DPMEM_COMMAND_OFFSET, - &dp2_ptr->u.ic.comm_queue[cmd_no].offset); - writew((u16)cmd_ptr->Service, - &dp2_ptr->u.ic.comm_queue[cmd_no].serv_id); - memcpy_toio(&dp2_ptr->u.ic.gdt_dpr_cmd[dp_offset],cmd_ptr,cp_count); - } else if (ha->type == GDT_PCI) { + if (ha->type == GDT_PCI) { dp6_ptr = ha->brd; writew(dp_offset + DPMEM_COMMAND_OFFSET, &dp6_ptr->u.ic.comm_queue[cmd_no].offset); @@ -1430,13 +1058,7 @@ static void gdth_release_event(gdth_ha_str *ha) if (ha->pccb->OpCode == GDT_INIT) ha->pccb->Service |= 0x80; - if (ha->type == GDT_EISA) { - if (ha->pccb->OpCode == GDT_INIT) /* store DMA buffer */ - outl(ha->ccb_phys, ha->bmic + MAILBOXREG); - outb(ha->pccb->Service, ha->bmic + LDOORREG); - } else if (ha->type == GDT_ISA) { - writeb(0, &((gdt2_dpram_str __iomem *)ha->brd)->io.event); - } else if (ha->type == GDT_PCI) { + if (ha->type == GDT_PCI) { writeb(0, &((gdt6_dpram_str __iomem *)ha->brd)->io.event); } else if (ha->type == GDT_PCINEW) { outb(1, PTR2USHORT(&ha->plx->ldoor_reg)); @@ -1560,15 +1182,7 @@ static int gdth_search_drives(gdth_ha_str *ha) gdth_arcdl_str *alst; gdth_alist_str *alst2; gdth_oem_str_ioctl *oemstr; -#ifdef INT_COAL - gdth_perf_modes *pmod; -#endif -#ifdef GDTH_RTC - u8 rtc[12]; - unsigned long flags; -#endif - TRACE(("gdth_search_drives() hanum %d\n", ha->hanum)); ok = 0; @@ -1588,29 +1202,6 @@ static int gdth_search_drives(gdth_ha_str *ha) } TRACE2(("gdth_search_drives(): SCREENSERVICE initialized\n")); -#ifdef GDTH_RTC - /* read realtime clock info, send to controller */ - /* 1. wait for the falling edge of update flag */ - spin_lock_irqsave(&rtc_lock, flags); - for (j = 0; j < 1000000; ++j) - if (CMOS_READ(RTC_FREQ_SELECT) & RTC_UIP) - break; - for (j = 0; j < 1000000; ++j) - if (!(CMOS_READ(RTC_FREQ_SELECT) & RTC_UIP)) - break; - /* 2. read info */ - do { - for (j = 0; j < 12; ++j) - rtc[j] = CMOS_READ(j); - } while (rtc[0] != CMOS_READ(0)); - spin_unlock_irqrestore(&rtc_lock, flags); - TRACE2(("gdth_search_drives(): RTC: %x/%x/%x\n",*(u32 *)&rtc[0], - *(u32 *)&rtc[4], *(u32 *)&rtc[8])); - /* 3. send to controller firmware */ - gdth_internal_cmd(ha, SCREENSERVICE, GDT_REALTIME, *(u32 *)&rtc[0], - *(u32 *)&rtc[4], *(u32 *)&rtc[8]); -#endif - /* unfreeze all IOs */ gdth_internal_cmd(ha, CACHESERVICE, GDT_UNFREEZE_IO, 0, 0, 0); @@ -1633,35 +1224,6 @@ static int gdth_search_drives(gdth_ha_str *ha) cdev_cnt = (u16)ha->info; ha->fw_vers = ha->service; -#ifdef INT_COAL - if (ha->type == GDT_PCIMPR) { - /* set perf. modes */ - pmod = (gdth_perf_modes *)ha->pscratch; - pmod->version = 1; - pmod->st_mode = 1; /* enable one status buffer */ - *((u64 *)&pmod->st_buff_addr1) = ha->coal_stat_phys; - pmod->st_buff_indx1 = COALINDEX; - pmod->st_buff_addr2 = 0; - pmod->st_buff_u_addr2 = 0; - pmod->st_buff_indx2 = 0; - pmod->st_buff_size = sizeof(gdth_coal_status) * MAXOFFSETS; - pmod->cmd_mode = 0; // disable all cmd buffers - pmod->cmd_buff_addr1 = 0; - pmod->cmd_buff_u_addr1 = 0; - pmod->cmd_buff_indx1 = 0; - pmod->cmd_buff_addr2 = 0; - pmod->cmd_buff_u_addr2 = 0; - pmod->cmd_buff_indx2 = 0; - pmod->cmd_buff_size = 0; - pmod->reserved1 = 0; - pmod->reserved2 = 0; - if (gdth_internal_cmd(ha, CACHESERVICE, GDT_IOCTL, SET_PERF_MODES, - INVALID_CHANNEL,sizeof(gdth_perf_modes))) { - printk("GDT-HA %d: Interrupt coalescing activated\n", ha->hanum); - } - } -#endif - /* detect number of buses - try new IOCTL */ iocr = (gdth_raw_iochan_str *)ha->pscratch; iocr->hdr.version = 0xffffffff; @@ -2433,9 +1995,6 @@ static int gdth_fill_cache_cmd(gdth_ha_str *ha, struct scsi_cmnd *scp, TRACE(("gdth_fill_cache_cmd() cmd 0x%x cmdsize %d hdrive %d\n", scp->cmnd[0],scp->cmd_len,hdrive)); - if (ha->type==GDT_EISA && ha->cmd_cnt>0) - return 0; - mode64 = (ha->cache_feat & GDT_64BIT) ? TRUE : FALSE; /* test for READ_16, WRITE_16 if !mode64 ? --- not required, should not occur due to error return on @@ -2518,9 +2077,9 @@ static int gdth_fill_cache_cmd(gdth_ha_str *ha, struct scsi_cmnd *scp, if (scsi_bufflen(scp)) { cmndinfo->dma_dir = (read_write == 1 ? - PCI_DMA_TODEVICE : PCI_DMA_FROMDEVICE); - sgcnt = pci_map_sg(ha->pdev, scsi_sglist(scp), scsi_sg_count(scp), - cmndinfo->dma_dir); + DMA_TO_DEVICE : DMA_FROM_DEVICE); + sgcnt = dma_map_sg(&ha->pdev->dev, scsi_sglist(scp), + scsi_sg_count(scp), cmndinfo->dma_dir); if (mode64) { struct scatterlist *sl; @@ -2528,12 +2087,6 @@ static int gdth_fill_cache_cmd(gdth_ha_str *ha, struct scsi_cmnd *scp, cmdp->u.cache64.sg_canz = sgcnt; scsi_for_each_sg(scp, sl, sgcnt, i) { cmdp->u.cache64.sg_lst[i].sg_ptr = sg_dma_address(sl); -#ifdef GDTH_DMA_STATISTICS - if (cmdp->u.cache64.sg_lst[i].sg_ptr > (u64)0xffffffff) - ha->dma64_cnt++; - else - ha->dma32_cnt++; -#endif cmdp->u.cache64.sg_lst[i].sg_len = sg_dma_len(sl); } } else { @@ -2543,9 +2096,6 @@ static int gdth_fill_cache_cmd(gdth_ha_str *ha, struct scsi_cmnd *scp, cmdp->u.cache.sg_canz = sgcnt; scsi_for_each_sg(scp, sl, sgcnt, i) { cmdp->u.cache.sg_lst[i].sg_ptr = sg_dma_address(sl); -#ifdef GDTH_DMA_STATISTICS - ha->dma32_cnt++; -#endif cmdp->u.cache.sg_lst[i].sg_len = sg_dma_len(sl); } } @@ -2603,8 +2153,6 @@ static int gdth_fill_raw_cmd(gdth_ha_str *ha, struct scsi_cmnd *scp, u8 b) dma_addr_t sense_paddr; int cmd_index, sgcnt, mode64; u8 t,l; - struct page *page; - unsigned long offset; struct gdth_cmndinfo *cmndinfo; t = scp->device->id; @@ -2613,9 +2161,6 @@ static int gdth_fill_raw_cmd(gdth_ha_str *ha, struct scsi_cmnd *scp, u8 b) TRACE(("gdth_fill_raw_cmd() cmd 0x%x bus %d ID %d LUN %d\n", scp->cmnd[0],b,t,l)); - if (ha->type==GDT_EISA && ha->cmd_cnt>0) - return 0; - mode64 = (ha->raw_feat & GDT_64BIT) ? TRUE : FALSE; cmdp->Service = SCSIRAWSERVICE; @@ -2649,10 +2194,8 @@ static int gdth_fill_raw_cmd(gdth_ha_str *ha, struct scsi_cmnd *scp, u8 b) } } else { - page = virt_to_page(scp->sense_buffer); - offset = (unsigned long)scp->sense_buffer & ~PAGE_MASK; - sense_paddr = pci_map_page(ha->pdev,page,offset, - 16,PCI_DMA_FROMDEVICE); + sense_paddr = dma_map_single(&ha->pdev->dev, scp->sense_buffer, 16, + DMA_FROM_DEVICE); cmndinfo->sense_paddr = sense_paddr; cmdp->OpCode = GDT_WRITE; /* always */ @@ -2693,9 +2236,9 @@ static int gdth_fill_raw_cmd(gdth_ha_str *ha, struct scsi_cmnd *scp, u8 b) } if (scsi_bufflen(scp)) { - cmndinfo->dma_dir = PCI_DMA_BIDIRECTIONAL; - sgcnt = pci_map_sg(ha->pdev, scsi_sglist(scp), scsi_sg_count(scp), - cmndinfo->dma_dir); + cmndinfo->dma_dir = DMA_BIDIRECTIONAL; + sgcnt = dma_map_sg(&ha->pdev->dev, scsi_sglist(scp), + scsi_sg_count(scp), cmndinfo->dma_dir); if (mode64) { struct scatterlist *sl; @@ -2703,12 +2246,6 @@ static int gdth_fill_raw_cmd(gdth_ha_str *ha, struct scsi_cmnd *scp, u8 b) cmdp->u.raw64.sg_ranz = sgcnt; scsi_for_each_sg(scp, sl, sgcnt, i) { cmdp->u.raw64.sg_lst[i].sg_ptr = sg_dma_address(sl); -#ifdef GDTH_DMA_STATISTICS - if (cmdp->u.raw64.sg_lst[i].sg_ptr > (u64)0xffffffff) - ha->dma64_cnt++; - else - ha->dma32_cnt++; -#endif cmdp->u.raw64.sg_lst[i].sg_len = sg_dma_len(sl); } } else { @@ -2718,9 +2255,6 @@ static int gdth_fill_raw_cmd(gdth_ha_str *ha, struct scsi_cmnd *scp, u8 b) cmdp->u.raw.sg_ranz = sgcnt; scsi_for_each_sg(scp, sl, sgcnt, i) { cmdp->u.raw.sg_lst[i].sg_ptr = sg_dma_address(sl); -#ifdef GDTH_DMA_STATISTICS - ha->dma32_cnt++; -#endif cmdp->u.raw.sg_lst[i].sg_len = sg_dma_len(sl); } } @@ -2778,9 +2312,6 @@ static int gdth_special_cmd(gdth_ha_str *ha, struct scsi_cmnd *scp) cmdp= ha->pccb; TRACE2(("gdth_special_cmd(): ")); - if (ha->type==GDT_EISA && ha->cmd_cnt>0) - return 0; - *cmdp = *cmndinfo->internal_cmd_str; cmdp->RequestBuffer = scp; @@ -2959,18 +2490,11 @@ static irqreturn_t __gdth_interrupt(gdth_ha_str *ha, { gdt6m_dpram_str __iomem *dp6m_ptr = NULL; gdt6_dpram_str __iomem *dp6_ptr; - gdt2_dpram_str __iomem *dp2_ptr; struct scsi_cmnd *scp; int rval, i; u8 IStatus; u16 Service; unsigned long flags = 0; -#ifdef INT_COAL - int coalesced = FALSE; - int next = FALSE; - gdth_coal_status *pcs = NULL; - int act_int_coal = 0; -#endif TRACE(("gdth_interrupt() IRQ %d\n", ha->irq)); @@ -2997,53 +2521,7 @@ static irqreturn_t __gdth_interrupt(gdth_ha_str *ha, ++act_ints; #endif -#ifdef INT_COAL - /* See if the fw is returning coalesced status */ - if (IStatus == COALINDEX) { - /* Coalesced status. Setup the initial status - buffer pointer and flags */ - pcs = ha->coal_stat; - coalesced = TRUE; - next = TRUE; - } - - do { - if (coalesced) { - /* For coalesced requests all status - information is found in the status buffer */ - IStatus = (u8)(pcs->status & 0xff); - } -#endif - - if (ha->type == GDT_EISA) { - if (IStatus & 0x80) { /* error flag */ - IStatus &= ~0x80; - ha->status = inw(ha->bmic + MAILBOXREG+8); - TRACE2(("gdth_interrupt() error %d/%d\n",IStatus,ha->status)); - } else /* no error */ - ha->status = S_OK; - ha->info = inl(ha->bmic + MAILBOXREG+12); - ha->service = inw(ha->bmic + MAILBOXREG+10); - ha->info2 = inl(ha->bmic + MAILBOXREG+4); - - outb(0xff, ha->bmic + EDOORREG); /* acknowledge interrupt */ - outb(0x00, ha->bmic + SEMA1REG); /* reset status semaphore */ - } else if (ha->type == GDT_ISA) { - dp2_ptr = ha->brd; - if (IStatus & 0x80) { /* error flag */ - IStatus &= ~0x80; - ha->status = readw(&dp2_ptr->u.ic.Status); - TRACE2(("gdth_interrupt() error %d/%d\n",IStatus,ha->status)); - } else /* no error */ - ha->status = S_OK; - ha->info = readl(&dp2_ptr->u.ic.Info[0]); - ha->service = readw(&dp2_ptr->u.ic.Service); - ha->info2 = readl(&dp2_ptr->u.ic.Info[1]); - - writeb(0xff, &dp2_ptr->io.irqdel); /* acknowledge interrupt */ - writeb(0, &dp2_ptr->u.ic.Cmd_Index);/* reset command index */ - writeb(0, &dp2_ptr->io.Sema1); /* reset status semaphore */ - } else if (ha->type == GDT_PCI) { + if (ha->type == GDT_PCI) { dp6_ptr = ha->brd; if (IStatus & 0x80) { /* error flag */ IStatus &= ~0x80; @@ -3075,28 +2553,15 @@ static irqreturn_t __gdth_interrupt(gdth_ha_str *ha, dp6m_ptr = ha->brd; if (IStatus & 0x80) { /* error flag */ IStatus &= ~0x80; -#ifdef INT_COAL - if (coalesced) - ha->status = pcs->ext_status & 0xffff; - else -#endif - ha->status = readw(&dp6m_ptr->i960r.status); + ha->status = readw(&dp6m_ptr->i960r.status); TRACE2(("gdth_interrupt() error %d/%d\n",IStatus,ha->status)); } else /* no error */ ha->status = S_OK; -#ifdef INT_COAL - /* get information */ - if (coalesced) { - ha->info = pcs->info0; - ha->info2 = pcs->info1; - ha->service = (pcs->ext_status >> 16) & 0xffff; - } else -#endif - { - ha->info = readl(&dp6m_ptr->i960r.info[0]); - ha->service = readw(&dp6m_ptr->i960r.service); - ha->info2 = readl(&dp6m_ptr->i960r.info[1]); - } + + ha->info = readl(&dp6m_ptr->i960r.info[0]); + ha->service = readw(&dp6m_ptr->i960r.service); + ha->info2 = readl(&dp6m_ptr->i960r.info[1]); + /* event string */ if (IStatus == ASYNCINDEX) { if (ha->service != SCREENSERVICE && @@ -3111,15 +2576,8 @@ static irqreturn_t __gdth_interrupt(gdth_ha_str *ha, } } } -#ifdef INT_COAL - /* Make sure that non coalesced interrupts get cleared - before being handled by gdth_async_event/gdth_sync_event */ - if (!coalesced) -#endif - { - writeb(0xff, &dp6m_ptr->i960r.edoor_reg); - writeb(0, &dp6m_ptr->i960r.sema1_reg); - } + writeb(0xff, &dp6m_ptr->i960r.edoor_reg); + writeb(0, &dp6m_ptr->i960r.sema1_reg); } else { TRACE2(("gdth_interrupt() unknown controller type\n")); if (!gdth_polling) @@ -3182,31 +2640,6 @@ static irqreturn_t __gdth_interrupt(gdth_ha_str *ha, gdth_scsi_done(scp); } -#ifdef INT_COAL - if (coalesced) { - /* go to the next status in the status buffer */ - ++pcs; -#ifdef GDTH_STATISTICS - ++act_int_coal; - if (act_int_coal > max_int_coal) { - max_int_coal = act_int_coal; - printk("GDT: max_int_coal = %d\n",(u16)max_int_coal); - } -#endif - /* see if there is another status */ - if (pcs->status == 0) - /* Stop the coalesce loop */ - next = FALSE; - } - } while (next); - - /* coalescing only for new GDT_PCIMPR controllers available */ - if (ha->type == GDT_PCIMPR && coalesced) { - writeb(0xff, &dp6m_ptr->i960r.edoor_reg); - writeb(0, &dp6m_ptr->i960r.sema1_reg); - } -#endif - gdth_next(ha); return IRQ_HANDLED; } @@ -3313,12 +2746,12 @@ static int gdth_sync_event(gdth_ha_str *ha, int service, u8 index, return 2; } if (scsi_bufflen(scp)) - pci_unmap_sg(ha->pdev, scsi_sglist(scp), scsi_sg_count(scp), + dma_unmap_sg(&ha->pdev->dev, scsi_sglist(scp), scsi_sg_count(scp), cmndinfo->dma_dir); if (cmndinfo->sense_paddr) - pci_unmap_page(ha->pdev, cmndinfo->sense_paddr, 16, - PCI_DMA_FROMDEVICE); + dma_unmap_page(&ha->pdev->dev, cmndinfo->sense_paddr, 16, + DMA_FROM_DEVICE); if (ha->status == S_OK) { cmndinfo->status = S_OK; @@ -3610,12 +3043,7 @@ static int gdth_async_event(gdth_ha_str *ha) + sizeof(u64); ha->cmd_cnt = 0; gdth_copy_command(ha); - if (ha->type == GDT_EISA) - printk("[EISA slot %d] ",(u16)ha->brd_phys); - else if (ha->type == GDT_ISA) - printk("[DPMEM 0x%4X] ",(u16)ha->brd_phys); - else - printk("[PCI %d/%d] ",(u16)(ha->brd_phys>>8), + printk("[PCI %d/%d] ",(u16)(ha->brd_phys>>8), (u16)((ha->brd_phys>>3)&0x1f)); gdth_release_event(ha); } @@ -3756,23 +3184,12 @@ static inline void gdth_timer_init(void) static void __init internal_setup(char *str,int *ints) { - int i, argc; + int i; char *cur_str, *argv; TRACE2(("internal_setup() str %s ints[0] %d\n", str ? str:"NULL", ints ? ints[0]:0)); - /* read irq[] from ints[] */ - if (ints) { - argc = ints[0]; - if (argc > 0) { - if (argc > MAXHA) - argc = MAXHA; - for (i = 0; i < argc; ++i) - irq[i] = ints[i+1]; - } - } - /* analyse string */ argv = str; while (argv && (cur_str = strchr(argv, ':'))) { @@ -3799,8 +3216,6 @@ static void __init internal_setup(char *str,int *ints) rescan = val; else if (!strncmp(argv, "shared_access:", 14)) shared_access = val; - else if (!strncmp(argv, "probe_eisa_isa:", 15)) - probe_eisa_isa = val; else if (!strncmp(argv, "reserve_list:", 13)) { reserve_list[0] = val; for (i = 1; i < MAX_RES_ARGS; i++) { @@ -3847,18 +3262,7 @@ static const char *gdth_ctr_name(gdth_ha_str *ha) { TRACE2(("gdth_ctr_name()\n")); - if (ha->type == GDT_EISA) { - switch (ha->stype) { - case GDT3_ID: - return("GDT3000/3020"); - case GDT3A_ID: - return("GDT3000A/3020A/3050A"); - case GDT3B_ID: - return("GDT3000B/3010A"); - } - } else if (ha->type == GDT_ISA) { - return("GDT2000/2020"); - } else if (ha->type == GDT_PCI) { + if (ha->type == GDT_PCI) { switch (ha->pdev->device) { case PCI_DEVICE_ID_VORTEX_GDT60x0: return("GDT6000/6020/6050"); @@ -4155,131 +3559,147 @@ static int ioc_resetdrv(void __user *arg, char *cmnd) return 0; } -static int ioc_general(void __user *arg, char *cmnd) +static void gdth_ioc_cacheservice(gdth_ha_str *ha, gdth_ioctl_general *gen, + u64 paddr) { - gdth_ioctl_general gen; - char *buf = NULL; - u64 paddr; - gdth_ha_str *ha; - int rval; - - if (copy_from_user(&gen, arg, sizeof(gdth_ioctl_general))) - return -EFAULT; - ha = gdth_find_ha(gen.ionode); - if (!ha) - return -EFAULT; - - if (gen.data_len > INT_MAX) - return -EINVAL; - if (gen.sense_len > INT_MAX) - return -EINVAL; - if (gen.data_len + gen.sense_len > INT_MAX) - return -EINVAL; - - if (gen.data_len + gen.sense_len != 0) { - if (!(buf = gdth_ioctl_alloc(ha, gen.data_len + gen.sense_len, - FALSE, &paddr))) - return -EFAULT; - if (copy_from_user(buf, arg + sizeof(gdth_ioctl_general), - gen.data_len + gen.sense_len)) { - gdth_ioctl_free(ha, gen.data_len+gen.sense_len, buf, paddr); - return -EFAULT; - } + if (ha->cache_feat & GDT_64BIT) { + /* copy elements from 32-bit IOCTL structure */ + gen->command.u.cache64.BlockCnt = gen->command.u.cache.BlockCnt; + gen->command.u.cache64.BlockNo = gen->command.u.cache.BlockNo; + gen->command.u.cache64.DeviceNo = gen->command.u.cache.DeviceNo; + + if (ha->cache_feat & SCATTER_GATHER) { + gen->command.u.cache64.DestAddr = (u64)-1; + gen->command.u.cache64.sg_canz = 1; + gen->command.u.cache64.sg_lst[0].sg_ptr = paddr; + gen->command.u.cache64.sg_lst[0].sg_len = gen->data_len; + gen->command.u.cache64.sg_lst[1].sg_len = 0; + } else { + gen->command.u.cache64.DestAddr = paddr; + gen->command.u.cache64.sg_canz = 0; + } + } else { + if (ha->cache_feat & SCATTER_GATHER) { + gen->command.u.cache.DestAddr = 0xffffffff; + gen->command.u.cache.sg_canz = 1; + gen->command.u.cache.sg_lst[0].sg_ptr = (u32)paddr; + gen->command.u.cache.sg_lst[0].sg_len = gen->data_len; + gen->command.u.cache.sg_lst[1].sg_len = 0; + } else { + gen->command.u.cache.DestAddr = paddr; + gen->command.u.cache.sg_canz = 0; + } + } +} - if (gen.command.OpCode == GDT_IOCTL) { - gen.command.u.ioctl.p_param = paddr; - } else if (gen.command.Service == CACHESERVICE) { - if (ha->cache_feat & GDT_64BIT) { - /* copy elements from 32-bit IOCTL structure */ - gen.command.u.cache64.BlockCnt = gen.command.u.cache.BlockCnt; - gen.command.u.cache64.BlockNo = gen.command.u.cache.BlockNo; - gen.command.u.cache64.DeviceNo = gen.command.u.cache.DeviceNo; - /* addresses */ - if (ha->cache_feat & SCATTER_GATHER) { - gen.command.u.cache64.DestAddr = (u64)-1; - gen.command.u.cache64.sg_canz = 1; - gen.command.u.cache64.sg_lst[0].sg_ptr = paddr; - gen.command.u.cache64.sg_lst[0].sg_len = gen.data_len; - gen.command.u.cache64.sg_lst[1].sg_len = 0; - } else { - gen.command.u.cache64.DestAddr = paddr; - gen.command.u.cache64.sg_canz = 0; - } - } else { - if (ha->cache_feat & SCATTER_GATHER) { - gen.command.u.cache.DestAddr = 0xffffffff; - gen.command.u.cache.sg_canz = 1; - gen.command.u.cache.sg_lst[0].sg_ptr = (u32)paddr; - gen.command.u.cache.sg_lst[0].sg_len = gen.data_len; - gen.command.u.cache.sg_lst[1].sg_len = 0; - } else { - gen.command.u.cache.DestAddr = paddr; - gen.command.u.cache.sg_canz = 0; - } - } - } else if (gen.command.Service == SCSIRAWSERVICE) { - if (ha->raw_feat & GDT_64BIT) { - /* copy elements from 32-bit IOCTL structure */ - char cmd[16]; - gen.command.u.raw64.sense_len = gen.command.u.raw.sense_len; - gen.command.u.raw64.bus = gen.command.u.raw.bus; - gen.command.u.raw64.lun = gen.command.u.raw.lun; - gen.command.u.raw64.target = gen.command.u.raw.target; - memcpy(cmd, gen.command.u.raw.cmd, 16); - memcpy(gen.command.u.raw64.cmd, cmd, 16); - gen.command.u.raw64.clen = gen.command.u.raw.clen; - gen.command.u.raw64.sdlen = gen.command.u.raw.sdlen; - gen.command.u.raw64.direction = gen.command.u.raw.direction; - /* addresses */ - if (ha->raw_feat & SCATTER_GATHER) { - gen.command.u.raw64.sdata = (u64)-1; - gen.command.u.raw64.sg_ranz = 1; - gen.command.u.raw64.sg_lst[0].sg_ptr = paddr; - gen.command.u.raw64.sg_lst[0].sg_len = gen.data_len; - gen.command.u.raw64.sg_lst[1].sg_len = 0; - } else { - gen.command.u.raw64.sdata = paddr; - gen.command.u.raw64.sg_ranz = 0; +static void gdth_ioc_scsiraw(gdth_ha_str *ha, gdth_ioctl_general *gen, + u64 paddr) +{ + if (ha->raw_feat & GDT_64BIT) { + /* copy elements from 32-bit IOCTL structure */ + char cmd[16]; + + gen->command.u.raw64.sense_len = gen->command.u.raw.sense_len; + gen->command.u.raw64.bus = gen->command.u.raw.bus; + gen->command.u.raw64.lun = gen->command.u.raw.lun; + gen->command.u.raw64.target = gen->command.u.raw.target; + memcpy(cmd, gen->command.u.raw.cmd, 16); + memcpy(gen->command.u.raw64.cmd, cmd, 16); + gen->command.u.raw64.clen = gen->command.u.raw.clen; + gen->command.u.raw64.sdlen = gen->command.u.raw.sdlen; + gen->command.u.raw64.direction = gen->command.u.raw.direction; + + /* addresses */ + if (ha->raw_feat & SCATTER_GATHER) { + gen->command.u.raw64.sdata = (u64)-1; + gen->command.u.raw64.sg_ranz = 1; + gen->command.u.raw64.sg_lst[0].sg_ptr = paddr; + gen->command.u.raw64.sg_lst[0].sg_len = gen->data_len; + gen->command.u.raw64.sg_lst[1].sg_len = 0; + } else { + gen->command.u.raw64.sdata = paddr; + gen->command.u.raw64.sg_ranz = 0; } - gen.command.u.raw64.sense_data = paddr + gen.data_len; - } else { - if (ha->raw_feat & SCATTER_GATHER) { - gen.command.u.raw.sdata = 0xffffffff; - gen.command.u.raw.sg_ranz = 1; - gen.command.u.raw.sg_lst[0].sg_ptr = (u32)paddr; - gen.command.u.raw.sg_lst[0].sg_len = gen.data_len; - gen.command.u.raw.sg_lst[1].sg_len = 0; - } else { - gen.command.u.raw.sdata = paddr; - gen.command.u.raw.sg_ranz = 0; + + gen->command.u.raw64.sense_data = paddr + gen->data_len; + } else { + if (ha->raw_feat & SCATTER_GATHER) { + gen->command.u.raw.sdata = 0xffffffff; + gen->command.u.raw.sg_ranz = 1; + gen->command.u.raw.sg_lst[0].sg_ptr = (u32)paddr; + gen->command.u.raw.sg_lst[0].sg_len = gen->data_len; + gen->command.u.raw.sg_lst[1].sg_len = 0; + } else { + gen->command.u.raw.sdata = paddr; + gen->command.u.raw.sg_ranz = 0; } - gen.command.u.raw.sense_data = (u32)paddr + gen.data_len; - } - } else { - gdth_ioctl_free(ha, gen.data_len+gen.sense_len, buf, paddr); - return -EFAULT; - } - } - rval = __gdth_execute(ha->sdev, &gen.command, cmnd, gen.timeout, &gen.info); - if (rval < 0) { - gdth_ioctl_free(ha, gen.data_len+gen.sense_len, buf, paddr); - return rval; - } - gen.status = rval; + gen->command.u.raw.sense_data = (u32)paddr + gen->data_len; + } +} - if (copy_to_user(arg + sizeof(gdth_ioctl_general), buf, - gen.data_len + gen.sense_len)) { - gdth_ioctl_free(ha, gen.data_len+gen.sense_len, buf, paddr); - return -EFAULT; - } - if (copy_to_user(arg, &gen, - sizeof(gdth_ioctl_general) - sizeof(gdth_cmd_str))) { - gdth_ioctl_free(ha, gen.data_len+gen.sense_len, buf, paddr); - return -EFAULT; - } - gdth_ioctl_free(ha, gen.data_len+gen.sense_len, buf, paddr); - return 0; +static int ioc_general(void __user *arg, char *cmnd) +{ + gdth_ioctl_general gen; + gdth_ha_str *ha; + char *buf = NULL; + dma_addr_t paddr; + int rval; + + if (copy_from_user(&gen, arg, sizeof(gdth_ioctl_general))) + return -EFAULT; + ha = gdth_find_ha(gen.ionode); + if (!ha) + return -EFAULT; + + if (gen.data_len > INT_MAX) + return -EINVAL; + if (gen.sense_len > INT_MAX) + return -EINVAL; + if (gen.data_len + gen.sense_len > INT_MAX) + return -EINVAL; + + if (gen.data_len + gen.sense_len > 0) { + buf = dma_alloc_coherent(&ha->pdev->dev, + gen.data_len + gen.sense_len, &paddr, + GFP_KERNEL); + if (!buf) + return -EFAULT; + + rval = -EFAULT; + if (copy_from_user(buf, arg + sizeof(gdth_ioctl_general), + gen.data_len + gen.sense_len)) + goto out_free_buf; + + if (gen.command.OpCode == GDT_IOCTL) + gen.command.u.ioctl.p_param = paddr; + else if (gen.command.Service == CACHESERVICE) + gdth_ioc_cacheservice(ha, &gen, paddr); + else if (gen.command.Service == SCSIRAWSERVICE) + gdth_ioc_scsiraw(ha, &gen, paddr); + else + goto out_free_buf; + } + + rval = __gdth_execute(ha->sdev, &gen.command, cmnd, gen.timeout, + &gen.info); + if (rval < 0) + goto out_free_buf; + gen.status = rval; + + rval = -EFAULT; + if (copy_to_user(arg + sizeof(gdth_ioctl_general), buf, + gen.data_len + gen.sense_len)) + goto out_free_buf; + if (copy_to_user(arg, &gen, + sizeof(gdth_ioctl_general) - sizeof(gdth_cmd_str))) + goto out_free_buf; + + rval = 0; +out_free_buf: + dma_free_coherent(&ha->pdev->dev, gen.data_len + gen.sense_len, buf, + paddr); + return rval; } static int ioc_hdrlist(void __user *arg, char *cmnd) @@ -4514,22 +3934,17 @@ static int gdth_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) (NULL == (ha = gdth_find_ha(ctrt.ionode)))) return -EFAULT; - if (ha->type == GDT_ISA || ha->type == GDT_EISA) { - ctrt.type = (u8)((ha->stype>>20) - 0x10); + if (ha->type != GDT_PCIMPR) { + ctrt.type = (u8)((ha->stype<<4) + 6); } else { - if (ha->type != GDT_PCIMPR) { - ctrt.type = (u8)((ha->stype<<4) + 6); - } else { - ctrt.type = - (ha->oem_id == OEM_ID_INTEL ? 0xfd : 0xfe); - if (ha->stype >= 0x300) - ctrt.ext_type = 0x6000 | ha->pdev->subsystem_device; - else - ctrt.ext_type = 0x6000 | ha->stype; - } - ctrt.device_id = ha->pdev->device; - ctrt.sub_device_id = ha->pdev->subsystem_device; + ctrt.type = (ha->oem_id == OEM_ID_INTEL ? 0xfd : 0xfe); + if (ha->stype >= 0x300) + ctrt.ext_type = 0x6000 | ha->pdev->subsystem_device; + else + ctrt.ext_type = 0x6000 | ha->stype; } + ctrt.device_id = ha->pdev->device; + ctrt.sub_device_id = ha->pdev->subsystem_device; ctrt.info = ha->brd_phys; ctrt.oem_id = ha->oem_id; if (copy_to_user(argp, &ctrt, sizeof(gdth_ioctl_ctrtype))) @@ -4683,272 +4098,6 @@ static struct scsi_host_template gdth_template = { .no_write_same = 1, }; -#ifdef CONFIG_ISA -static int __init gdth_isa_probe_one(u32 isa_bios) -{ - struct Scsi_Host *shp; - gdth_ha_str *ha; - dma_addr_t scratch_dma_handle = 0; - int error, i; - - if (!gdth_search_isa(isa_bios)) - return -ENXIO; - - shp = scsi_host_alloc(&gdth_template, sizeof(gdth_ha_str)); - if (!shp) - return -ENOMEM; - ha = shost_priv(shp); - - error = -ENODEV; - if (!gdth_init_isa(isa_bios,ha)) - goto out_host_put; - - /* controller found and initialized */ - printk("Configuring GDT-ISA HA at BIOS 0x%05X IRQ %u DRQ %u\n", - isa_bios, ha->irq, ha->drq); - - error = request_irq(ha->irq, gdth_interrupt, 0, "gdth", ha); - if (error) { - printk("GDT-ISA: Unable to allocate IRQ\n"); - goto out_host_put; - } - - error = request_dma(ha->drq, "gdth"); - if (error) { - printk("GDT-ISA: Unable to allocate DMA channel\n"); - goto out_free_irq; - } - - set_dma_mode(ha->drq,DMA_MODE_CASCADE); - enable_dma(ha->drq); - shp->unchecked_isa_dma = 1; - shp->irq = ha->irq; - shp->dma_channel = ha->drq; - - ha->hanum = gdth_ctr_count++; - ha->shost = shp; - - ha->pccb = &ha->cmdext; - ha->ccb_phys = 0L; - ha->pdev = NULL; - - error = -ENOMEM; - - ha->pscratch = pci_alloc_consistent(ha->pdev, GDTH_SCRATCH, - &scratch_dma_handle); - if (!ha->pscratch) - goto out_dec_counters; - ha->scratch_phys = scratch_dma_handle; - - ha->pmsg = pci_alloc_consistent(ha->pdev, sizeof(gdth_msg_str), - &scratch_dma_handle); - if (!ha->pmsg) - goto out_free_pscratch; - ha->msg_phys = scratch_dma_handle; - -#ifdef INT_COAL - ha->coal_stat = pci_alloc_consistent(ha->pdev, - sizeof(gdth_coal_status) * MAXOFFSETS, - &scratch_dma_handle); - if (!ha->coal_stat) - goto out_free_pmsg; - ha->coal_stat_phys = scratch_dma_handle; -#endif - - ha->scratch_busy = FALSE; - ha->req_first = NULL; - ha->tid_cnt = MAX_HDRIVES; - if (max_ids > 0 && max_ids < ha->tid_cnt) - ha->tid_cnt = max_ids; - for (i = 0; i < GDTH_MAXCMDS; ++i) - ha->cmd_tab[i].cmnd = UNUSED_CMND; - ha->scan_mode = rescan ? 0x10 : 0; - - error = -ENODEV; - if (!gdth_search_drives(ha)) { - printk("GDT-ISA: Error during device scan\n"); - goto out_free_coal_stat; - } - - if (hdr_channel < 0 || hdr_channel > ha->bus_cnt) - hdr_channel = ha->bus_cnt; - ha->virt_bus = hdr_channel; - - if (ha->cache_feat & ha->raw_feat & ha->screen_feat & GDT_64BIT) - shp->max_cmd_len = 16; - - shp->max_id = ha->tid_cnt; - shp->max_lun = MAXLUN; - shp->max_channel = ha->bus_cnt; - - spin_lock_init(&ha->smp_lock); - gdth_enable_int(ha); - - error = scsi_add_host(shp, NULL); - if (error) - goto out_free_coal_stat; - list_add_tail(&ha->list, &gdth_instances); - gdth_timer_init(); - - scsi_scan_host(shp); - - return 0; - - out_free_coal_stat: -#ifdef INT_COAL - pci_free_consistent(ha->pdev, sizeof(gdth_coal_status) * MAXOFFSETS, - ha->coal_stat, ha->coal_stat_phys); - out_free_pmsg: -#endif - pci_free_consistent(ha->pdev, sizeof(gdth_msg_str), - ha->pmsg, ha->msg_phys); - out_free_pscratch: - pci_free_consistent(ha->pdev, GDTH_SCRATCH, - ha->pscratch, ha->scratch_phys); - out_dec_counters: - gdth_ctr_count--; - out_free_irq: - free_irq(ha->irq, ha); - out_host_put: - scsi_host_put(shp); - return error; -} -#endif /* CONFIG_ISA */ - -#ifdef CONFIG_EISA -static int __init gdth_eisa_probe_one(u16 eisa_slot) -{ - struct Scsi_Host *shp; - gdth_ha_str *ha; - dma_addr_t scratch_dma_handle = 0; - int error, i; - - if (!gdth_search_eisa(eisa_slot)) - return -ENXIO; - - shp = scsi_host_alloc(&gdth_template, sizeof(gdth_ha_str)); - if (!shp) - return -ENOMEM; - ha = shost_priv(shp); - - error = -ENODEV; - if (!gdth_init_eisa(eisa_slot,ha)) - goto out_host_put; - - /* controller found and initialized */ - printk("Configuring GDT-EISA HA at Slot %d IRQ %u\n", - eisa_slot >> 12, ha->irq); - - error = request_irq(ha->irq, gdth_interrupt, 0, "gdth", ha); - if (error) { - printk("GDT-EISA: Unable to allocate IRQ\n"); - goto out_host_put; - } - - shp->unchecked_isa_dma = 0; - shp->irq = ha->irq; - shp->dma_channel = 0xff; - - ha->hanum = gdth_ctr_count++; - ha->shost = shp; - - TRACE2(("EISA detect Bus 0: hanum %d\n", ha->hanum)); - - ha->pccb = &ha->cmdext; - ha->ccb_phys = 0L; - - error = -ENOMEM; - - ha->pdev = NULL; - ha->pscratch = pci_alloc_consistent(ha->pdev, GDTH_SCRATCH, - &scratch_dma_handle); - if (!ha->pscratch) - goto out_free_irq; - ha->scratch_phys = scratch_dma_handle; - - ha->pmsg = pci_alloc_consistent(ha->pdev, sizeof(gdth_msg_str), - &scratch_dma_handle); - if (!ha->pmsg) - goto out_free_pscratch; - ha->msg_phys = scratch_dma_handle; - -#ifdef INT_COAL - ha->coal_stat = pci_alloc_consistent(ha->pdev, - sizeof(gdth_coal_status) * MAXOFFSETS, - &scratch_dma_handle); - if (!ha->coal_stat) - goto out_free_pmsg; - ha->coal_stat_phys = scratch_dma_handle; -#endif - - ha->ccb_phys = pci_map_single(ha->pdev,ha->pccb, - sizeof(gdth_cmd_str), PCI_DMA_BIDIRECTIONAL); - if (!ha->ccb_phys) - goto out_free_coal_stat; - - ha->scratch_busy = FALSE; - ha->req_first = NULL; - ha->tid_cnt = MAX_HDRIVES; - if (max_ids > 0 && max_ids < ha->tid_cnt) - ha->tid_cnt = max_ids; - for (i = 0; i < GDTH_MAXCMDS; ++i) - ha->cmd_tab[i].cmnd = UNUSED_CMND; - ha->scan_mode = rescan ? 0x10 : 0; - - if (!gdth_search_drives(ha)) { - printk("GDT-EISA: Error during device scan\n"); - error = -ENODEV; - goto out_free_ccb_phys; - } - - if (hdr_channel < 0 || hdr_channel > ha->bus_cnt) - hdr_channel = ha->bus_cnt; - ha->virt_bus = hdr_channel; - - if (ha->cache_feat & ha->raw_feat & ha->screen_feat & GDT_64BIT) - shp->max_cmd_len = 16; - - shp->max_id = ha->tid_cnt; - shp->max_lun = MAXLUN; - shp->max_channel = ha->bus_cnt; - - spin_lock_init(&ha->smp_lock); - gdth_enable_int(ha); - - error = scsi_add_host(shp, NULL); - if (error) - goto out_free_ccb_phys; - list_add_tail(&ha->list, &gdth_instances); - gdth_timer_init(); - - scsi_scan_host(shp); - - return 0; - - out_free_ccb_phys: - pci_unmap_single(ha->pdev,ha->ccb_phys, sizeof(gdth_cmd_str), - PCI_DMA_BIDIRECTIONAL); - out_free_coal_stat: -#ifdef INT_COAL - pci_free_consistent(ha->pdev, sizeof(gdth_coal_status) * MAXOFFSETS, - ha->coal_stat, ha->coal_stat_phys); - out_free_pmsg: -#endif - pci_free_consistent(ha->pdev, sizeof(gdth_msg_str), - ha->pmsg, ha->msg_phys); - out_free_pscratch: - pci_free_consistent(ha->pdev, GDTH_SCRATCH, - ha->pscratch, ha->scratch_phys); - out_free_irq: - free_irq(ha->irq, ha); - gdth_ctr_count--; - out_host_put: - scsi_host_put(shp); - return error; -} -#endif /* CONFIG_EISA */ - -#ifdef CONFIG_PCI static int gdth_pci_probe_one(gdth_pci_str *pcistr, gdth_ha_str **ha_out) { struct Scsi_Host *shp; @@ -4993,27 +4142,18 @@ static int gdth_pci_probe_one(gdth_pci_str *pcistr, gdth_ha_str **ha_out) error = -ENOMEM; - ha->pscratch = pci_alloc_consistent(ha->pdev, GDTH_SCRATCH, - &scratch_dma_handle); + ha->pscratch = dma_alloc_coherent(&ha->pdev->dev, GDTH_SCRATCH, + &scratch_dma_handle, GFP_KERNEL); if (!ha->pscratch) goto out_free_irq; ha->scratch_phys = scratch_dma_handle; - ha->pmsg = pci_alloc_consistent(ha->pdev, sizeof(gdth_msg_str), - &scratch_dma_handle); + ha->pmsg = dma_alloc_coherent(&ha->pdev->dev, sizeof(gdth_msg_str), + &scratch_dma_handle, GFP_KERNEL); if (!ha->pmsg) goto out_free_pscratch; ha->msg_phys = scratch_dma_handle; -#ifdef INT_COAL - ha->coal_stat = pci_alloc_consistent(ha->pdev, - sizeof(gdth_coal_status) * MAXOFFSETS, - &scratch_dma_handle); - if (!ha->coal_stat) - goto out_free_pmsg; - ha->coal_stat_phys = scratch_dma_handle; -#endif - ha->scratch_busy = FALSE; ha->req_first = NULL; ha->tid_cnt = pdev->device >= 0x200 ? MAXID : MAX_HDRIVES; @@ -5026,7 +4166,7 @@ static int gdth_pci_probe_one(gdth_pci_str *pcistr, gdth_ha_str **ha_out) error = -ENODEV; if (!gdth_search_drives(ha)) { printk("GDT-PCI %d: Error during device scan\n", ha->hanum); - goto out_free_coal_stat; + goto out_free_pmsg; } if (hdr_channel < 0 || hdr_channel > ha->bus_cnt) @@ -5036,19 +4176,19 @@ static int gdth_pci_probe_one(gdth_pci_str *pcistr, gdth_ha_str **ha_out) /* 64-bit DMA only supported from FW >= x.43 */ if (!(ha->cache_feat & ha->raw_feat & ha->screen_feat & GDT_64BIT) || !ha->dma64_support) { - if (pci_set_dma_mask(pdev, DMA_BIT_MASK(32))) { + if (dma_set_mask(&pdev->dev, DMA_BIT_MASK(32))) { printk(KERN_WARNING "GDT-PCI %d: " "Unable to set 32-bit DMA\n", ha->hanum); - goto out_free_coal_stat; + goto out_free_pmsg; } } else { shp->max_cmd_len = 16; - if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) { + if (!dma_set_mask(&pdev->dev, DMA_BIT_MASK(64))) { printk("GDT-PCI %d: 64-bit DMA enabled\n", ha->hanum); - } else if (pci_set_dma_mask(pdev, DMA_BIT_MASK(32))) { + } else if (dma_set_mask(&pdev->dev, DMA_BIT_MASK(32))) { printk(KERN_WARNING "GDT-PCI %d: " "Unable to set 64/32-bit DMA\n", ha->hanum); - goto out_free_coal_stat; + goto out_free_pmsg; } } @@ -5061,7 +4201,7 @@ static int gdth_pci_probe_one(gdth_pci_str *pcistr, gdth_ha_str **ha_out) error = scsi_add_host(shp, &pdev->dev); if (error) - goto out_free_coal_stat; + goto out_free_pmsg; list_add_tail(&ha->list, &gdth_instances); pci_set_drvdata(ha->pdev, ha); @@ -5073,16 +4213,11 @@ static int gdth_pci_probe_one(gdth_pci_str *pcistr, gdth_ha_str **ha_out) return 0; - out_free_coal_stat: -#ifdef INT_COAL - pci_free_consistent(ha->pdev, sizeof(gdth_coal_status) * MAXOFFSETS, - ha->coal_stat, ha->coal_stat_phys); out_free_pmsg: -#endif - pci_free_consistent(ha->pdev, sizeof(gdth_msg_str), + dma_free_coherent(&ha->pdev->dev, sizeof(gdth_msg_str), ha->pmsg, ha->msg_phys); out_free_pscratch: - pci_free_consistent(ha->pdev, GDTH_SCRATCH, + dma_free_coherent(&ha->pdev->dev, GDTH_SCRATCH, ha->pscratch, ha->scratch_phys); out_free_irq: free_irq(ha->irq, ha); @@ -5091,7 +4226,6 @@ static int gdth_pci_probe_one(gdth_pci_str *pcistr, gdth_ha_str **ha_out) scsi_host_put(shp); return error; } -#endif /* CONFIG_PCI */ static void gdth_remove_one(gdth_ha_str *ha) { @@ -5111,24 +4245,15 @@ static void gdth_remove_one(gdth_ha_str *ha) if (shp->irq) free_irq(shp->irq,ha); -#ifdef CONFIG_ISA - if (shp->dma_channel != 0xff) - free_dma(shp->dma_channel); -#endif -#ifdef INT_COAL - if (ha->coal_stat) - pci_free_consistent(ha->pdev, sizeof(gdth_coal_status) * - MAXOFFSETS, ha->coal_stat, ha->coal_stat_phys); -#endif if (ha->pscratch) - pci_free_consistent(ha->pdev, GDTH_SCRATCH, + dma_free_coherent(&ha->pdev->dev, GDTH_SCRATCH, ha->pscratch, ha->scratch_phys); if (ha->pmsg) - pci_free_consistent(ha->pdev, sizeof(gdth_msg_str), + dma_free_coherent(&ha->pdev->dev, sizeof(gdth_msg_str), ha->pmsg, ha->msg_phys); if (ha->ccb_phys) - pci_unmap_single(ha->pdev,ha->ccb_phys, - sizeof(gdth_cmd_str),PCI_DMA_BIDIRECTIONAL); + dma_unmap_single(&ha->pdev->dev, ha->ccb_phys, + sizeof(gdth_cmd_str), DMA_BIDIRECTIONAL); scsi_host_put(shp); } @@ -5167,26 +4292,6 @@ static int __init gdth_init(void) gdth_clear_events(); timer_setup(&gdth_timer, gdth_timeout, 0); - /* As default we do not probe for EISA or ISA controllers */ - if (probe_eisa_isa) { - /* scanning for controllers, at first: ISA controller */ -#ifdef CONFIG_ISA - u32 isa_bios; - for (isa_bios = 0xc8000UL; isa_bios <= 0xd8000UL; - isa_bios += 0x8000UL) - gdth_isa_probe_one(isa_bios); -#endif -#ifdef CONFIG_EISA - { - u16 eisa_slot; - for (eisa_slot = 0x1000; eisa_slot <= 0x8000; - eisa_slot += 0x1000) - gdth_eisa_probe_one(eisa_slot); - } -#endif - } - -#ifdef CONFIG_PCI /* scanning for PCI controllers */ if (pci_register_driver(&gdth_pci_driver)) { gdth_ha_str *ha; @@ -5195,7 +4300,6 @@ static int __init gdth_init(void) gdth_remove_one(ha); return -ENODEV; } -#endif /* CONFIG_PCI */ TRACE2(("gdth_detect() %d controller detected\n", gdth_ctr_count)); @@ -5216,9 +4320,7 @@ static void __exit gdth_exit(void) del_timer_sync(&gdth_timer); #endif -#ifdef CONFIG_PCI pci_unregister_driver(&gdth_pci_driver); -#endif list_for_each_entry(ha, &gdth_instances, list) gdth_remove_one(ha); diff --git a/drivers/scsi/gdth.h b/drivers/scsi/gdth.h index ee6ffcf388e8..5a13d406d40e 100644 --- a/drivers/scsi/gdth.h +++ b/drivers/scsi/gdth.h @@ -38,17 +38,9 @@ #define OEM_ID_INTEL 0x8000 /* controller classes */ -#define GDT_ISA 0x01 /* ISA controller */ -#define GDT_EISA 0x02 /* EISA controller */ #define GDT_PCI 0x03 /* PCI controller */ #define GDT_PCINEW 0x04 /* new PCI controller */ #define GDT_PCIMPR 0x05 /* PCI MPR controller */ -/* GDT_EISA, controller subtypes EISA */ -#define GDT3_ID 0x0130941c /* GDT3000/3020 */ -#define GDT3A_ID 0x0230941c /* GDT3000A/3020A/3050A */ -#define GDT3B_ID 0x0330941c /* GDT3000B/3010A */ -/* GDT_ISA */ -#define GDT2_ID 0x0120941c /* GDT2000/2020 */ #ifndef PCI_DEVICE_ID_VORTEX_GDT60x0 /* GDT_PCI */ @@ -281,17 +273,6 @@ #define GDTH_DATA_IN 0x01000000L /* data from target */ #define GDTH_DATA_OUT 0x00000000L /* data to target */ -/* BMIC registers (EISA controllers) */ -#define ID0REG 0x0c80 /* board ID */ -#define EINTENABREG 0x0c89 /* interrupt enable */ -#define SEMA0REG 0x0c8a /* command semaphore */ -#define SEMA1REG 0x0c8b /* status semaphore */ -#define LDOORREG 0x0c8d /* local doorbell */ -#define EDENABREG 0x0c8e /* EISA system doorbell enab. */ -#define EDOORREG 0x0c8f /* EISA system doorbell */ -#define MAILBOXREG 0x0c90 /* mailbox reg. (16 bytes) */ -#define EISAREG 0x0cc0 /* EISA configuration */ - /* other defines */ #define LINUX_OS 8 /* used for cache optim. */ #define SECS32 0x1f /* round capacity */ @@ -706,21 +687,11 @@ typedef struct { u8 fw_magic; /* contr. ID from firmware */ } __attribute__((packed)) gdt_pci_sram; -/* SRAM structure EISA controllers (but NOT GDT3000/3020) */ -typedef struct { - u8 os_used[16]; /* OS code per service */ - u16 need_deinit; /* switch betw. BIOS/driver */ - u8 switch_support; /* see need_deinit */ - u8 padding; -} __attribute__((packed)) gdt_eisa_sram; - - /* DPRAM ISA controllers */ typedef struct { union { struct { u8 bios_used[0x3c00-32]; /* 15KB - 32Bytes BIOS */ - u32 magic; /* controller (EISA) ID */ u16 need_deinit; /* switch betw. BIOS/driver */ u8 switch_support; /* see need_deinit */ u8 padding[9]; @@ -843,7 +814,6 @@ typedef struct { u16 cache_feat; /* feat. cache serv. (s/g,..)*/ u16 raw_feat; /* feat. raw service (s/g,..)*/ u16 screen_feat; /* feat. raw service (s/g,..)*/ - u16 bmic; /* BMIC address (EISA) */ void __iomem *brd; /* DPRAM address */ u32 brd_phys; /* slot number/BIOS address */ gdt6c_plx_regs *plx; /* PLX regs (new PCI contr.) */ diff --git a/drivers/scsi/gdth_ioctl.h b/drivers/scsi/gdth_ioctl.h index 4c91894ac244..ee4c9bf1022a 100644 --- a/drivers/scsi/gdth_ioctl.h +++ b/drivers/scsi/gdth_ioctl.h @@ -27,11 +27,7 @@ #define GDTH_MAXSG 32 /* max. s/g elements */ #define MAX_LDRIVES 255 /* max. log. drive count */ -#ifdef GDTH_IOCTL_PROC -#define MAX_HDRIVES 100 /* max. host drive count */ -#else #define MAX_HDRIVES MAX_LDRIVES /* max. host drive count */ -#endif /* scatter/gather element */ typedef struct { @@ -178,91 +174,6 @@ typedef struct { gdth_evt_data event_data; } __attribute__((packed)) gdth_evt_str; - -#ifdef GDTH_IOCTL_PROC -/* IOCTL structure (write) */ -typedef struct { - u32 magic; /* IOCTL magic */ - u16 ioctl; /* IOCTL */ - u16 ionode; /* controller number */ - u16 service; /* controller service */ - u16 timeout; /* timeout */ - union { - struct { - u8 command[512]; /* controller command */ - u8 data[1]; /* add. data */ - } general; - struct { - u8 lock; /* lock/unlock */ - u8 drive_cnt; /* drive count */ - u16 drives[MAX_HDRIVES];/* drives */ - } lockdrv; - struct { - u8 lock; /* lock/unlock */ - u8 channel; /* channel */ - } lockchn; - struct { - int erase; /* erase event ? */ - int handle; - u8 evt[EVENT_SIZE]; /* event structure */ - } event; - struct { - u8 bus; /* SCSI bus */ - u8 target; /* target ID */ - u8 lun; /* LUN */ - u8 cmd_len; /* command length */ - u8 cmd[12]; /* SCSI command */ - } scsi; - struct { - u16 hdr_no; /* host drive number */ - u8 flag; /* old meth./add/remove */ - } rescan; - } iu; -} gdth_iowr_str; - -/* IOCTL structure (read) */ -typedef struct { - u32 size; /* buffer size */ - u32 status; /* IOCTL error code */ - union { - struct { - u8 data[1]; /* data */ - } general; - struct { - u16 version; /* driver version */ - } drvers; - struct { - u8 type; /* controller type */ - u16 info; /* slot etc. */ - u16 oem_id; /* OEM ID */ - u16 bios_ver; /* not used */ - u16 access; /* not used */ - u16 ext_type; /* extended type */ - u16 device_id; /* device ID */ - u16 sub_device_id; /* sub device ID */ - } ctrtype; - struct { - u8 version; /* OS version */ - u8 subversion; /* OS subversion */ - u16 revision; /* revision */ - } osvers; - struct { - u16 count; /* controller count */ - } ctrcnt; - struct { - int handle; - u8 evt[EVENT_SIZE]; /* event structure */ - } event; - struct { - u8 bus; /* SCSI bus, 0xff: invalid */ - u8 target; /* target ID */ - u8 lun; /* LUN */ - u8 cluster_type; /* cluster properties */ - } hdr_list[MAX_HDRIVES]; /* index is host drive number */ - } iu; -} gdth_iord_str; -#endif - /* GDTIOCTL_GENERAL */ typedef struct { u16 ionode; /* controller number */ diff --git a/drivers/scsi/gdth_proc.c b/drivers/scsi/gdth_proc.c index 3a9751a80225..381d849726ac 100644 --- a/drivers/scsi/gdth_proc.c +++ b/drivers/scsi/gdth_proc.c @@ -31,7 +31,6 @@ static int gdth_set_asc_info(struct Scsi_Host *host, char *buffer, int i, found; gdth_cmd_str gdtcmd; gdth_cpar_str *pcpar; - u64 paddr; char cmnd[MAX_COMMAND_SIZE]; memset(cmnd, 0xff, 12); @@ -113,13 +112,23 @@ static int gdth_set_asc_info(struct Scsi_Host *host, char *buffer, } if (wb_mode) { - if (!gdth_ioctl_alloc(ha, sizeof(gdth_cpar_str), TRUE, &paddr)) - return(-EBUSY); + unsigned long flags; + + BUILD_BUG_ON(sizeof(gdth_cpar_str) > GDTH_SCRATCH); + + spin_lock_irqsave(&ha->smp_lock, flags); + if (ha->scratch_busy) { + spin_unlock_irqrestore(&ha->smp_lock, flags); + return -EBUSY; + } + ha->scratch_busy = TRUE; + spin_unlock_irqrestore(&ha->smp_lock, flags); + pcpar = (gdth_cpar_str *)ha->pscratch; memcpy( pcpar, &ha->cpar, sizeof(gdth_cpar_str) ); gdtcmd.Service = CACHESERVICE; gdtcmd.OpCode = GDT_IOCTL; - gdtcmd.u.ioctl.p_param = paddr; + gdtcmd.u.ioctl.p_param = ha->scratch_phys; gdtcmd.u.ioctl.param_size = sizeof(gdth_cpar_str); gdtcmd.u.ioctl.subfunc = CACHE_CONFIG; gdtcmd.u.ioctl.channel = INVALID_CHANNEL; @@ -127,7 +136,10 @@ static int gdth_set_asc_info(struct Scsi_Host *host, char *buffer, gdth_execute(host, &gdtcmd, cmnd, 30, NULL); - gdth_ioctl_free(ha, GDTH_SCRATCH, ha->pscratch, paddr); + spin_lock_irqsave(&ha->smp_lock, flags); + ha->scratch_busy = FALSE; + spin_unlock_irqrestore(&ha->smp_lock, flags); + printk("Done.\n"); return(orig_length); } @@ -143,7 +155,7 @@ int gdth_show_info(struct seq_file *m, struct Scsi_Host *host) int id, i, j, k, sec, flag; int no_mdrv = 0, drv_no, is_mirr; u32 cnt; - u64 paddr; + dma_addr_t paddr; int rc = -ENOMEM; gdth_cmd_str *gdtcmd; @@ -217,20 +229,14 @@ int gdth_show_info(struct seq_file *m, struct Scsi_Host *host) " Serial No.: \t0x%8X\tCache RAM size:\t%d KB\n", ha->binfo.ser_no, ha->binfo.memsize / 1024); -#ifdef GDTH_DMA_STATISTICS - /* controller statistics */ - seq_puts(m, "\nController Statistics:\n"); - seq_printf(m, - " 32-bit DMA buffer:\t%lu\t64-bit DMA buffer:\t%lu\n", - ha->dma32_cnt, ha->dma64_cnt); -#endif - if (ha->more_proc) { + size_t size = max_t(size_t, GDTH_SCRATCH, sizeof(gdth_hget_str)); + /* more information: 2. about physical devices */ seq_puts(m, "\nPhysical Devices:"); flag = FALSE; - buf = gdth_ioctl_alloc(ha, GDTH_SCRATCH, FALSE, &paddr); + buf = dma_alloc_coherent(&ha->pdev->dev, size, &paddr, GFP_KERNEL); if (!buf) goto stop_output; for (i = 0; i < ha->bus_cnt; ++i) { @@ -323,7 +329,6 @@ int gdth_show_info(struct seq_file *m, struct Scsi_Host *host) } } } - gdth_ioctl_free(ha, GDTH_SCRATCH, buf, paddr); if (!flag) seq_puts(m, "\n --\n"); @@ -332,9 +337,6 @@ int gdth_show_info(struct seq_file *m, struct Scsi_Host *host) seq_puts(m, "\nLogical Drives:"); flag = FALSE; - buf = gdth_ioctl_alloc(ha, GDTH_SCRATCH, FALSE, &paddr); - if (!buf) - goto stop_output; for (i = 0; i < MAX_LDRIVES; ++i) { if (!ha->hdr[i].is_logdrv) continue; @@ -408,8 +410,7 @@ int gdth_show_info(struct seq_file *m, struct Scsi_Host *host) seq_printf(m, " To Array Drv.:\t%s\n", hrec); } - gdth_ioctl_free(ha, GDTH_SCRATCH, buf, paddr); - + if (!flag) seq_puts(m, "\n --\n"); @@ -417,9 +418,6 @@ int gdth_show_info(struct seq_file *m, struct Scsi_Host *host) seq_puts(m, "\nArray Drives:"); flag = FALSE; - buf = gdth_ioctl_alloc(ha, GDTH_SCRATCH, FALSE, &paddr); - if (!buf) - goto stop_output; for (i = 0; i < MAX_LDRIVES; ++i) { if (!(ha->hdr[i].is_arraydrv && ha->hdr[i].is_master)) continue; @@ -468,8 +466,7 @@ int gdth_show_info(struct seq_file *m, struct Scsi_Host *host) hrec); } } - gdth_ioctl_free(ha, GDTH_SCRATCH, buf, paddr); - + if (!flag) seq_puts(m, "\n --\n"); @@ -477,9 +474,6 @@ int gdth_show_info(struct seq_file *m, struct Scsi_Host *host) seq_puts(m, "\nHost Drives:"); flag = FALSE; - buf = gdth_ioctl_alloc(ha, sizeof(gdth_hget_str), FALSE, &paddr); - if (!buf) - goto stop_output; for (i = 0; i < MAX_LDRIVES; ++i) { if (!ha->hdr[i].is_logdrv || (ha->hdr[i].is_arraydrv && !ha->hdr[i].is_master)) @@ -510,7 +504,7 @@ int gdth_show_info(struct seq_file *m, struct Scsi_Host *host) } } } - gdth_ioctl_free(ha, sizeof(gdth_hget_str), buf, paddr); + dma_free_coherent(&ha->pdev->dev, size, buf, paddr); for (i = 0; i < MAX_HDRIVES; ++i) { if (!(ha->hdr[i].present)) @@ -563,65 +557,6 @@ free_fail: return rc; } -static char *gdth_ioctl_alloc(gdth_ha_str *ha, int size, int scratch, - u64 *paddr) -{ - unsigned long flags; - char *ret_val; - - if (size == 0) - return NULL; - - spin_lock_irqsave(&ha->smp_lock, flags); - - if (!ha->scratch_busy && size <= GDTH_SCRATCH) { - ha->scratch_busy = TRUE; - ret_val = ha->pscratch; - *paddr = ha->scratch_phys; - } else if (scratch) { - ret_val = NULL; - } else { - dma_addr_t dma_addr; - - ret_val = pci_alloc_consistent(ha->pdev, size, &dma_addr); - *paddr = dma_addr; - } - - spin_unlock_irqrestore(&ha->smp_lock, flags); - return ret_val; -} - -static void gdth_ioctl_free(gdth_ha_str *ha, int size, char *buf, u64 paddr) -{ - unsigned long flags; - - if (buf == ha->pscratch) { - spin_lock_irqsave(&ha->smp_lock, flags); - ha->scratch_busy = FALSE; - spin_unlock_irqrestore(&ha->smp_lock, flags); - } else { - pci_free_consistent(ha->pdev, size, buf, paddr); - } -} - -#ifdef GDTH_IOCTL_PROC -static int gdth_ioctl_check_bin(gdth_ha_str *ha, u16 size) -{ - unsigned long flags; - int ret_val; - - spin_lock_irqsave(&ha->smp_lock, flags); - - ret_val = FALSE; - if (ha->scratch_busy) { - if (((gdth_iord_str *)ha->pscratch)->size == (u32)size) - ret_val = TRUE; - } - spin_unlock_irqrestore(&ha->smp_lock, flags); - return ret_val; -} -#endif - static void gdth_wait_completion(gdth_ha_str *ha, int busnum, int id) { unsigned long flags; diff --git a/drivers/scsi/gdth_proc.h b/drivers/scsi/gdth_proc.h index d7d0aa283695..4cc5377cb92e 100644 --- a/drivers/scsi/gdth_proc.h +++ b/drivers/scsi/gdth_proc.h @@ -12,9 +12,6 @@ int gdth_execute(struct Scsi_Host *shost, gdth_cmd_str *gdtcmd, char *cmnd, static int gdth_set_asc_info(struct Scsi_Host *host, char *buffer, int length, gdth_ha_str *ha); -static char *gdth_ioctl_alloc(gdth_ha_str *ha, int size, int scratch, - u64 *paddr); -static void gdth_ioctl_free(gdth_ha_str *ha, int size, char *buf, u64 paddr); static void gdth_wait_completion(gdth_ha_str *ha, int busnum, int id); #endif diff --git a/drivers/scsi/hisi_sas/hisi_sas.h b/drivers/scsi/hisi_sas/hisi_sas.h index af291947a54d..6c87bd34509a 100644 --- a/drivers/scsi/hisi_sas/hisi_sas.h +++ b/drivers/scsi/hisi_sas/hisi_sas.h @@ -14,6 +14,7 @@ #include <linux/acpi.h> #include <linux/clk.h> +#include <linux/debugfs.h> #include <linux/dmapool.h> #include <linux/iopoll.h> #include <linux/lcm.h> @@ -29,7 +30,7 @@ #define HISI_SAS_MAX_PHYS 9 #define HISI_SAS_MAX_QUEUES 32 -#define HISI_SAS_QUEUE_SLOTS 512 +#define HISI_SAS_QUEUE_SLOTS 4096 #define HISI_SAS_MAX_ITCT_ENTRIES 1024 #define HISI_SAS_MAX_DEVICES HISI_SAS_MAX_ITCT_ENTRIES #define HISI_SAS_RESET_BIT 0 @@ -40,20 +41,25 @@ #define HISI_SAS_COMMAND_TABLE_SZ (sizeof(union hisi_sas_command_table)) #define hisi_sas_status_buf_addr(buf) \ - (buf + offsetof(struct hisi_sas_slot_buf_table, status_buffer)) -#define hisi_sas_status_buf_addr_mem(slot) hisi_sas_status_buf_addr(slot->buf) + ((buf) + offsetof(struct hisi_sas_slot_buf_table, status_buffer)) +#define hisi_sas_status_buf_addr_mem(slot) hisi_sas_status_buf_addr((slot)->buf) #define hisi_sas_status_buf_addr_dma(slot) \ - hisi_sas_status_buf_addr(slot->buf_dma) + hisi_sas_status_buf_addr((slot)->buf_dma) #define hisi_sas_cmd_hdr_addr(buf) \ - (buf + offsetof(struct hisi_sas_slot_buf_table, command_header)) -#define hisi_sas_cmd_hdr_addr_mem(slot) hisi_sas_cmd_hdr_addr(slot->buf) -#define hisi_sas_cmd_hdr_addr_dma(slot) hisi_sas_cmd_hdr_addr(slot->buf_dma) + ((buf) + offsetof(struct hisi_sas_slot_buf_table, command_header)) +#define hisi_sas_cmd_hdr_addr_mem(slot) hisi_sas_cmd_hdr_addr((slot)->buf) +#define hisi_sas_cmd_hdr_addr_dma(slot) hisi_sas_cmd_hdr_addr((slot)->buf_dma) #define hisi_sas_sge_addr(buf) \ - (buf + offsetof(struct hisi_sas_slot_buf_table, sge_page)) -#define hisi_sas_sge_addr_mem(slot) hisi_sas_sge_addr(slot->buf) -#define hisi_sas_sge_addr_dma(slot) hisi_sas_sge_addr(slot->buf_dma) + ((buf) + offsetof(struct hisi_sas_slot_buf_table, sge_page)) +#define hisi_sas_sge_addr_mem(slot) hisi_sas_sge_addr((slot)->buf) +#define hisi_sas_sge_addr_dma(slot) hisi_sas_sge_addr((slot)->buf_dma) + +#define hisi_sas_sge_dif_addr(buf) \ + ((buf) + offsetof(struct hisi_sas_slot_dif_buf_table, sge_dif_page)) +#define hisi_sas_sge_dif_addr_mem(slot) hisi_sas_sge_dif_addr((slot)->buf) +#define hisi_sas_sge_dif_addr_dma(slot) hisi_sas_sge_dif_addr((slot)->buf_dma) #define HISI_SAS_MAX_SSP_RESP_SZ (sizeof(struct ssp_frame_hdr) + 1024) #define HISI_SAS_MAX_SMP_RESP_SZ 1028 @@ -73,7 +79,13 @@ SHOST_DIF_TYPE2_PROTECTION | \ SHOST_DIF_TYPE3_PROTECTION) -#define HISI_SAS_PROT_MASK (HISI_SAS_DIF_PROT_MASK) +#define HISI_SAS_DIX_PROT_MASK (SHOST_DIX_TYPE1_PROTECTION | \ + SHOST_DIX_TYPE2_PROTECTION | \ + SHOST_DIX_TYPE3_PROTECTION) + +#define HISI_SAS_PROT_MASK (HISI_SAS_DIF_PROT_MASK | HISI_SAS_DIX_PROT_MASK) + +#define HISI_SAS_WAIT_PHYUP_TIMEOUT 20 struct hisi_hba; @@ -82,11 +94,6 @@ enum { PORT_TYPE_SATA = (1U << 0), }; -enum dev_status { - HISI_SAS_DEV_NORMAL, - HISI_SAS_DEV_EH, -}; - enum { HISI_SAS_INT_ABT_CMD = 0, HISI_SAS_INT_ABT_DEV = 1, @@ -145,6 +152,7 @@ struct hisi_sas_phy { struct asd_sas_phy sas_phy; struct sas_identify identify; struct completion *reset_completion; + struct timer_list timer; spinlock_t lock; u64 port_id; /* from hw */ u64 frame_rcvd_size; @@ -165,6 +173,7 @@ struct hisi_sas_port { struct hisi_sas_cq { struct hisi_hba *hisi_hba; + const struct cpumask *pci_irq_mask; struct tasklet_struct tasklet; int rd_point; int id; @@ -187,7 +196,7 @@ struct hisi_sas_device { enum sas_device_type dev_type; int device_id; int sata_idx; - u8 dev_status; + spinlock_t lock; /* For protecting slots */ }; struct hisi_sas_tmf_task { @@ -203,12 +212,14 @@ struct hisi_sas_slot { struct sas_task *task; struct hisi_sas_port *port; u64 n_elem; + u64 n_elem_dif; int dlvry_queue; int dlvry_queue_slot; int cmplt_queue; int cmplt_queue_slot; int abort; int ready; + int device_id; void *cmd_hdr; dma_addr_t cmd_hdr_dma; struct timer_list internal_abort_timer; @@ -220,6 +231,24 @@ struct hisi_sas_slot { u16 idx; }; +#define HISI_SAS_DEBUGFS_REG(x) {#x, x} + +struct hisi_sas_debugfs_reg_lu { + char *name; + int off; +}; + +struct hisi_sas_debugfs_reg { + const struct hisi_sas_debugfs_reg_lu *lu; + int count; + int base_off; + union { + u32 (*read_global_reg)(struct hisi_hba *hisi_hba, u32 off); + u32 (*read_port_reg)(struct hisi_hba *hisi_hba, int port, + u32 off); + }; +}; + struct hisi_sas_hw { int (*hw_init)(struct hisi_hba *hisi_hba); void (*setup_itct)(struct hisi_hba *hisi_hba, @@ -227,7 +256,7 @@ struct hisi_sas_hw { int (*slot_index_alloc)(struct hisi_hba *hisi_hba, struct domain_device *device); struct hisi_sas_device *(*alloc_dev)(struct domain_device *device); - void (*sl_notify)(struct hisi_hba *hisi_hba, int phy_no); + void (*sl_notify_ssp)(struct hisi_hba *hisi_hba, int phy_no); int (*get_free_slot)(struct hisi_hba *hisi_hba, struct hisi_sas_dq *dq); void (*start_delivery)(struct hisi_sas_dq *dq); void (*prep_ssp)(struct hisi_hba *hisi_hba, @@ -259,11 +288,16 @@ struct hisi_sas_hw { u32 (*get_phys_state)(struct hisi_hba *hisi_hba); int (*write_gpio)(struct hisi_hba *hisi_hba, u8 reg_type, u8 reg_index, u8 reg_count, u8 *write_data); - void (*wait_cmds_complete_timeout)(struct hisi_hba *hisi_hba, - int delay_ms, int timeout_ms); + int (*wait_cmds_complete_timeout)(struct hisi_hba *hisi_hba, + int delay_ms, int timeout_ms); + void (*snapshot_prepare)(struct hisi_hba *hisi_hba); + void (*snapshot_restore)(struct hisi_hba *hisi_hba); int max_command_entries; int complete_hdr_size; struct scsi_host_template *sht; + + const struct hisi_sas_debugfs_reg *debugfs_reg_global; + const struct hisi_sas_debugfs_reg *debugfs_reg_port; }; struct hisi_hba { @@ -329,9 +363,25 @@ struct hisi_hba { const struct hisi_sas_hw *hw; /* Low level hw interface */ unsigned long sata_dev_bitmap[BITS_TO_LONGS(HISI_SAS_MAX_DEVICES)]; struct work_struct rst_work; + struct work_struct debugfs_work; u32 phy_state; u32 intr_coal_ticks; /* Time of interrupt coalesce in us */ u32 intr_coal_count; /* Interrupt count to coalesce */ + + int cq_nvecs; + unsigned int *reply_map; + + /* debugfs memories */ + u32 *debugfs_global_reg; + u32 *debugfs_port_reg[HISI_SAS_MAX_PHYS]; + void *debugfs_complete_hdr[HISI_SAS_MAX_QUEUES]; + struct hisi_sas_cmd_hdr *debugfs_cmd_hdr[HISI_SAS_MAX_QUEUES]; + struct hisi_sas_iost *debugfs_iost; + struct hisi_sas_itct *debugfs_itct; + + struct dentry *debugfs_dir; + struct dentry *debugfs_dump_dentry; + bool debugfs_snapshot; }; /* Generic HW DMA host memory structures */ @@ -430,6 +480,11 @@ struct hisi_sas_sge_page { struct hisi_sas_sge sge[HISI_SAS_SGE_PAGE_CNT]; } __aligned(16); +#define HISI_SAS_SGE_DIF_PAGE_CNT SG_CHUNK_SIZE +struct hisi_sas_sge_dif_page { + struct hisi_sas_sge sge[HISI_SAS_SGE_DIF_PAGE_CNT]; +} __aligned(16); + struct hisi_sas_command_table_ssp { struct ssp_frame_hdr hdr; union { @@ -460,9 +515,18 @@ struct hisi_sas_slot_buf_table { struct hisi_sas_sge_page sge_page; }; +struct hisi_sas_slot_dif_buf_table { + struct hisi_sas_slot_buf_table slot_buf; + struct hisi_sas_sge_dif_page sge_dif_page; +}; + extern struct scsi_transport_template *hisi_sas_stt; + +extern bool hisi_sas_debugfs_enable; +extern struct dentry *hisi_sas_debugfs_dir; + extern void hisi_sas_stop_phys(struct hisi_hba *hisi_hba); -extern int hisi_sas_alloc(struct hisi_hba *hisi_hba, struct Scsi_Host *shost); +extern int hisi_sas_alloc(struct hisi_hba *hisi_hba); extern void hisi_sas_free(struct hisi_hba *hisi_hba); extern u8 hisi_sas_get_ata_protocol(struct host_to_dev_fis *fis, int direction); @@ -487,10 +551,14 @@ extern void hisi_sas_init_mem(struct hisi_hba *hisi_hba); extern void hisi_sas_rst_work_handler(struct work_struct *work); extern void hisi_sas_sync_rst_work_handler(struct work_struct *work); extern void hisi_sas_kill_tasklets(struct hisi_hba *hisi_hba); +extern void hisi_sas_phy_oob_ready(struct hisi_hba *hisi_hba, int phy_no); extern bool hisi_sas_notify_phy_event(struct hisi_sas_phy *phy, enum hisi_sas_phy_event event); extern void hisi_sas_release_tasks(struct hisi_hba *hisi_hba); extern u8 hisi_sas_get_prog_phy_linkrate_mask(enum sas_linkrate max); extern void hisi_sas_controller_reset_prepare(struct hisi_hba *hisi_hba); extern void hisi_sas_controller_reset_done(struct hisi_hba *hisi_hba); +extern void hisi_sas_debugfs_init(struct hisi_hba *hisi_hba); +extern void hisi_sas_debugfs_exit(struct hisi_hba *hisi_hba); +extern void hisi_sas_debugfs_work_handler(struct work_struct *work); #endif diff --git a/drivers/scsi/hisi_sas/hisi_sas_main.c b/drivers/scsi/hisi_sas/hisi_sas_main.c index bc17fa0d8375..13ca5a0bdf6b 100644 --- a/drivers/scsi/hisi_sas/hisi_sas_main.c +++ b/drivers/scsi/hisi_sas/hisi_sas_main.c @@ -144,7 +144,7 @@ EXPORT_SYMBOL_GPL(hisi_sas_get_ncq_tag); */ u8 hisi_sas_get_prog_phy_linkrate_mask(enum sas_linkrate max) { - u16 rate = 0; + u8 rate = 0; int i; max -= SAS_LINK_RATE_1_5_GBPS; @@ -241,8 +241,9 @@ static void hisi_sas_slot_index_init(struct hisi_hba *hisi_hba) void hisi_sas_slot_task_free(struct hisi_hba *hisi_hba, struct sas_task *task, struct hisi_sas_slot *slot) { - struct hisi_sas_dq *dq = &hisi_hba->dq[slot->dlvry_queue]; unsigned long flags; + int device_id = slot->device_id; + struct hisi_sas_device *sas_dev = &hisi_hba->devices[device_id]; if (task) { struct device *dev = hisi_hba->dev; @@ -252,17 +253,24 @@ void hisi_sas_slot_task_free(struct hisi_hba *hisi_hba, struct sas_task *task, task->lldd_task = NULL; - if (!sas_protocol_ata(task->task_proto)) + if (!sas_protocol_ata(task->task_proto)) { + struct sas_ssp_task *ssp_task = &task->ssp_task; + struct scsi_cmnd *scsi_cmnd = ssp_task->cmd; + if (slot->n_elem) dma_unmap_sg(dev, task->scatter, task->num_scatter, task->data_dir); + if (slot->n_elem_dif) + dma_unmap_sg(dev, scsi_prot_sglist(scsi_cmnd), + scsi_prot_sg_count(scsi_cmnd), + task->data_dir); + } } - - spin_lock_irqsave(&dq->lock, flags); + spin_lock_irqsave(&sas_dev->lock, flags); list_del_init(&slot->entry); - spin_unlock_irqrestore(&dq->lock, flags); + spin_unlock_irqrestore(&sas_dev->lock, flags); memset(slot, 0, offsetof(struct hisi_sas_slot, buf)); @@ -380,6 +388,59 @@ prep_out: return rc; } +static void hisi_sas_dif_dma_unmap(struct hisi_hba *hisi_hba, + struct sas_task *task, int n_elem_dif) +{ + struct device *dev = hisi_hba->dev; + + if (n_elem_dif) { + struct sas_ssp_task *ssp_task = &task->ssp_task; + struct scsi_cmnd *scsi_cmnd = ssp_task->cmd; + + dma_unmap_sg(dev, scsi_prot_sglist(scsi_cmnd), + scsi_prot_sg_count(scsi_cmnd), + task->data_dir); + } +} + +static int hisi_sas_dif_dma_map(struct hisi_hba *hisi_hba, + int *n_elem_dif, struct sas_task *task) +{ + struct device *dev = hisi_hba->dev; + struct sas_ssp_task *ssp_task; + struct scsi_cmnd *scsi_cmnd; + int rc; + + if (task->num_scatter) { + ssp_task = &task->ssp_task; + scsi_cmnd = ssp_task->cmd; + + if (scsi_prot_sg_count(scsi_cmnd)) { + *n_elem_dif = dma_map_sg(dev, + scsi_prot_sglist(scsi_cmnd), + scsi_prot_sg_count(scsi_cmnd), + task->data_dir); + + if (!*n_elem_dif) + return -ENOMEM; + + if (*n_elem_dif > HISI_SAS_SGE_DIF_PAGE_CNT) { + dev_err(dev, "task prep: n_elem_dif(%d) too large\n", + *n_elem_dif); + rc = -EINVAL; + goto err_out_dif_dma_unmap; + } + } + } + + return 0; + +err_out_dif_dma_unmap: + dma_unmap_sg(dev, scsi_prot_sglist(scsi_cmnd), + scsi_prot_sg_count(scsi_cmnd), task->data_dir); + return rc; +} + static int hisi_sas_task_prep(struct sas_task *task, struct hisi_sas_dq **dq_pointer, bool is_tmf, struct hisi_sas_tmf_task *tmf, @@ -394,7 +455,7 @@ static int hisi_sas_task_prep(struct sas_task *task, struct asd_sas_port *sas_port = device->port; struct device *dev = hisi_hba->dev; int dlvry_queue_slot, dlvry_queue, rc, slot_idx; - int n_elem = 0, n_elem_req = 0, n_elem_resp = 0; + int n_elem = 0, n_elem_dif = 0, n_elem_req = 0, n_elem_resp = 0; struct hisi_sas_dq *dq; unsigned long flags; int wr_q_index; @@ -410,7 +471,14 @@ static int hisi_sas_task_prep(struct sas_task *task, return -ECOMM; } - *dq_pointer = dq = sas_dev->dq; + if (hisi_hba->reply_map) { + int cpu = raw_smp_processor_id(); + unsigned int dq_index = hisi_hba->reply_map[cpu]; + + *dq_pointer = dq = &hisi_hba->dq[dq_index]; + } else { + *dq_pointer = dq = sas_dev->dq; + } port = to_hisi_sas_port(sas_port); if (port && !port->port_attached) { @@ -427,6 +495,12 @@ static int hisi_sas_task_prep(struct sas_task *task, if (rc < 0) goto prep_out; + if (!sas_protocol_ata(task->task_proto)) { + rc = hisi_sas_dif_dma_map(hisi_hba, &n_elem_dif, task); + if (rc < 0) + goto err_out_dma_unmap; + } + if (hisi_hba->hw->slot_index_alloc) rc = hisi_hba->hw->slot_index_alloc(hisi_hba, device); else { @@ -445,7 +519,7 @@ static int hisi_sas_task_prep(struct sas_task *task, rc = hisi_sas_slot_index_alloc(hisi_hba, scsi_cmnd); } if (rc < 0) - goto err_out_dma_unmap; + goto err_out_dif_dma_unmap; slot_idx = rc; slot = &hisi_hba->slot_info[slot_idx]; @@ -459,13 +533,17 @@ static int hisi_sas_task_prep(struct sas_task *task, } list_add_tail(&slot->delivery, &dq->list); - list_add_tail(&slot->entry, &sas_dev->list); spin_unlock_irqrestore(&dq->lock, flags); + spin_lock_irqsave(&sas_dev->lock, flags); + list_add_tail(&slot->entry, &sas_dev->list); + spin_unlock_irqrestore(&sas_dev->lock, flags); dlvry_queue = dq->id; dlvry_queue_slot = wr_q_index; + slot->device_id = sas_dev->device_id; slot->n_elem = n_elem; + slot->n_elem_dif = n_elem_dif; slot->dlvry_queue = dlvry_queue; slot->dlvry_queue_slot = dlvry_queue_slot; cmd_hdr_base = hisi_hba->cmd_hdr[dlvry_queue]; @@ -509,6 +587,9 @@ static int hisi_sas_task_prep(struct sas_task *task, err_out_tag: hisi_sas_slot_index_free(hisi_hba, slot_idx); +err_out_dif_dma_unmap: + if (!sas_protocol_ata(task->task_proto)) + hisi_sas_dif_dma_unmap(hisi_hba, task, n_elem_dif); err_out_dma_unmap: hisi_sas_dma_unmap(hisi_hba, task, n_elem, n_elem_req, n_elem_resp); @@ -626,11 +707,11 @@ static struct hisi_sas_device *hisi_sas_alloc_dev(struct domain_device *device) hisi_hba->devices[i].device_id = i; sas_dev = &hisi_hba->devices[i]; - sas_dev->dev_status = HISI_SAS_DEV_NORMAL; sas_dev->dev_type = device->dev_type; sas_dev->hisi_hba = hisi_hba; sas_dev->sas_device = device; sas_dev->dq = dq; + spin_lock_init(&sas_dev->lock); INIT_LIST_HEAD(&hisi_hba->devices[i].list); break; } @@ -778,7 +859,8 @@ static void hisi_sas_phyup_work(struct work_struct *work) struct asd_sas_phy *sas_phy = &phy->sas_phy; int phy_no = sas_phy->id; - hisi_hba->hw->sl_notify(hisi_hba, phy_no); /* This requires a sleep */ + if (phy->identify.target_port_protocols == SAS_PROTOCOL_SSP) + hisi_hba->hw->sl_notify_ssp(hisi_hba, phy_no); hisi_sas_bytes_dmaed(hisi_hba, phy_no); } @@ -808,6 +890,30 @@ bool hisi_sas_notify_phy_event(struct hisi_sas_phy *phy, } EXPORT_SYMBOL_GPL(hisi_sas_notify_phy_event); +static void hisi_sas_wait_phyup_timedout(struct timer_list *t) +{ + struct hisi_sas_phy *phy = from_timer(phy, t, timer); + struct hisi_hba *hisi_hba = phy->hisi_hba; + struct device *dev = hisi_hba->dev; + int phy_no = phy->sas_phy.id; + + dev_warn(dev, "phy%d wait phyup timeout, issuing link reset\n", phy_no); + hisi_sas_notify_phy_event(phy, HISI_PHYE_LINK_RESET); +} + +void hisi_sas_phy_oob_ready(struct hisi_hba *hisi_hba, int phy_no) +{ + struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no]; + struct device *dev = hisi_hba->dev; + + if (!timer_pending(&phy->timer)) { + dev_dbg(dev, "phy%d OOB ready\n", phy_no); + phy->timer.expires = jiffies + HISI_SAS_WAIT_PHYUP_TIMEOUT * HZ; + add_timer(&phy->timer); + } +} +EXPORT_SYMBOL_GPL(hisi_sas_phy_oob_ready); + static void hisi_sas_phy_init(struct hisi_hba *hisi_hba, int phy_no) { struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no]; @@ -836,6 +942,8 @@ static void hisi_sas_phy_init(struct hisi_hba *hisi_hba, int phy_no) INIT_WORK(&phy->works[i], hisi_sas_phye_fns[i]); spin_lock_init(&phy->lock); + + timer_setup(&phy->timer, hisi_sas_wait_phyup_timedout, 0); } static void hisi_sas_port_notify_formed(struct asd_sas_phy *sas_phy) @@ -926,7 +1034,7 @@ static void hisi_sas_dev_gone(struct domain_device *device) if (!test_bit(HISI_SAS_RESET_BIT, &hisi_hba->flags)) { hisi_sas_internal_task_abort(hisi_hba, device, - HISI_SAS_INT_ABT_DEV, 0); + HISI_SAS_INT_ABT_DEV, 0); hisi_sas_dereg_device(hisi_hba, device); @@ -946,7 +1054,7 @@ static int hisi_sas_queue_command(struct sas_task *task, gfp_t gfp_flags) return hisi_sas_task_exec(task, gfp_flags, 0, NULL); } -static void hisi_sas_phy_set_linkrate(struct hisi_hba *hisi_hba, int phy_no, +static int hisi_sas_phy_set_linkrate(struct hisi_hba *hisi_hba, int phy_no, struct sas_phy_linkrates *r) { struct sas_phy_linkrates _r; @@ -955,6 +1063,9 @@ static void hisi_sas_phy_set_linkrate(struct hisi_hba *hisi_hba, int phy_no, struct asd_sas_phy *sas_phy = &phy->sas_phy; enum sas_linkrate min, max; + if (r->minimum_linkrate > SAS_LINK_RATE_1_5_GBPS) + return -EINVAL; + if (r->maximum_linkrate == SAS_LINK_RATE_UNKNOWN) { max = sas_phy->phy->maximum_linkrate; min = r->minimum_linkrate; @@ -962,7 +1073,7 @@ static void hisi_sas_phy_set_linkrate(struct hisi_hba *hisi_hba, int phy_no, max = r->maximum_linkrate; min = sas_phy->phy->minimum_linkrate; } else - return; + return -EINVAL; _r.maximum_linkrate = max; _r.minimum_linkrate = min; @@ -974,6 +1085,8 @@ static void hisi_sas_phy_set_linkrate(struct hisi_hba *hisi_hba, int phy_no, msleep(100); hisi_hba->hw->phy_set_linkrate(hisi_hba, phy_no, &_r); hisi_hba->hw->phy_start(hisi_hba, phy_no); + + return 0; } static int hisi_sas_control_phy(struct asd_sas_phy *sas_phy, enum phy_func func, @@ -999,8 +1112,7 @@ static int hisi_sas_control_phy(struct asd_sas_phy *sas_phy, enum phy_func func, break; case PHY_FUNC_SET_LINK_RATE: - hisi_sas_phy_set_linkrate(hisi_hba, phy_no, funcdata); - break; + return hisi_sas_phy_set_linkrate(hisi_hba, phy_no, funcdata); case PHY_FUNC_GET_EVENTS: if (hisi_hba->hw->get_events) { hisi_hba->hw->get_events(hisi_hba, phy_no); @@ -1068,7 +1180,7 @@ static int hisi_sas_exec_internal_tmf_task(struct domain_device *device, task->task_done = hisi_sas_task_done; task->slow_task->timer.function = hisi_sas_tmf_timedout; - task->slow_task->timer.expires = jiffies + TASK_TIMEOUT*HZ; + task->slow_task->timer.expires = jiffies + TASK_TIMEOUT * HZ; add_timer(&task->slow_task->timer); res = hisi_sas_task_exec(task, GFP_KERNEL, 1, tmf); @@ -1429,6 +1541,9 @@ static int hisi_sas_controller_reset(struct hisi_hba *hisi_hba) struct Scsi_Host *shost = hisi_hba->shost; int rc; + if (hisi_sas_debugfs_enable && hisi_hba->debugfs_itct) + queue_work(hisi_hba->wq, &hisi_hba->debugfs_work); + if (!hisi_hba->hw->soft_reset) return -1; @@ -1491,7 +1606,6 @@ static int hisi_sas_abort_task(struct sas_task *task) task->task_state_flags |= SAS_TASK_STATE_ABORTED; spin_unlock_irqrestore(&task->task_state_lock, flags); - sas_dev->dev_status = HISI_SAS_DEV_EH; if (task->lldd_task && task->task_proto & SAS_PROTOCOL_SSP) { struct scsi_cmnd *cmnd = task->uldd_task; struct hisi_sas_slot *slot = task->lldd_task; @@ -1527,7 +1641,8 @@ static int hisi_sas_abort_task(struct sas_task *task) task->task_proto & SAS_PROTOCOL_STP) { if (task->dev->dev_type == SAS_SATA_DEV) { rc = hisi_sas_internal_task_abort(hisi_hba, device, - HISI_SAS_INT_ABT_DEV, 0); + HISI_SAS_INT_ABT_DEV, + 0); if (rc < 0) { dev_err(dev, "abort task: internal abort failed\n"); goto out; @@ -1542,7 +1657,7 @@ static int hisi_sas_abort_task(struct sas_task *task) struct hisi_sas_cq *cq = &hisi_hba->cq[slot->dlvry_queue]; rc = hisi_sas_internal_task_abort(hisi_hba, device, - HISI_SAS_INT_ABT_CMD, tag); + HISI_SAS_INT_ABT_CMD, tag); if (((rc < 0) || (rc == TMF_RESP_FUNC_FAILED)) && task->lldd_task) { /* @@ -1568,7 +1683,7 @@ static int hisi_sas_abort_task_set(struct domain_device *device, u8 *lun) int rc = TMF_RESP_FUNC_FAILED; rc = hisi_sas_internal_task_abort(hisi_hba, device, - HISI_SAS_INT_ABT_DEV, 0); + HISI_SAS_INT_ABT_DEV, 0); if (rc < 0) { dev_err(dev, "abort task set: internal abort rc=%d\n", rc); return TMF_RESP_FUNC_FAILED; @@ -1586,8 +1701,8 @@ static int hisi_sas_abort_task_set(struct domain_device *device, u8 *lun) static int hisi_sas_clear_aca(struct domain_device *device, u8 *lun) { - int rc = TMF_RESP_FUNC_FAILED; struct hisi_sas_tmf_task tmf_task; + int rc; tmf_task.tmf = TMF_CLEAR_ACA; rc = hisi_sas_debug_issue_ssp_tmf(device, lun, &tmf_task); @@ -1635,17 +1750,12 @@ static int hisi_sas_debug_I_T_nexus_reset(struct domain_device *device) static int hisi_sas_I_T_nexus_reset(struct domain_device *device) { - struct hisi_sas_device *sas_dev = device->lldd_dev; struct hisi_hba *hisi_hba = dev_to_hisi_hba(device); struct device *dev = hisi_hba->dev; - int rc = TMF_RESP_FUNC_FAILED; - - if (sas_dev->dev_status != HISI_SAS_DEV_EH) - return TMF_RESP_FUNC_FAILED; - sas_dev->dev_status = HISI_SAS_DEV_NORMAL; + int rc; rc = hisi_sas_internal_task_abort(hisi_hba, device, - HISI_SAS_INT_ABT_DEV, 0); + HISI_SAS_INT_ABT_DEV, 0); if (rc < 0) { dev_err(dev, "I_T nexus reset: internal abort (%d)\n", rc); return TMF_RESP_FUNC_FAILED; @@ -1667,7 +1777,6 @@ static int hisi_sas_lu_reset(struct domain_device *device, u8 *lun) struct device *dev = hisi_hba->dev; int rc = TMF_RESP_FUNC_FAILED; - sas_dev->dev_status = HISI_SAS_DEV_EH; if (dev_is_sata(device)) { struct sas_phy *phy; @@ -1691,7 +1800,7 @@ static int hisi_sas_lu_reset(struct domain_device *device, u8 *lun) struct hisi_sas_tmf_task tmf_task = { .tmf = TMF_LU_RESET }; rc = hisi_sas_internal_task_abort(hisi_hba, device, - HISI_SAS_INT_ABT_DEV, 0); + HISI_SAS_INT_ABT_DEV, 0); if (rc < 0) { dev_err(dev, "lu_reset: internal abort failed\n"); goto out; @@ -1777,7 +1886,7 @@ static int hisi_sas_query_task(struct sas_task *task) static int hisi_sas_internal_abort_task_exec(struct hisi_hba *hisi_hba, int device_id, struct sas_task *task, int abort_flag, - int task_tag) + int task_tag, struct hisi_sas_dq *dq) { struct domain_device *device = task->dev; struct hisi_sas_device *sas_dev = device->lldd_dev; @@ -1786,7 +1895,6 @@ hisi_sas_internal_abort_task_exec(struct hisi_hba *hisi_hba, int device_id, struct hisi_sas_slot *slot; struct asd_sas_port *sas_port = device->port; struct hisi_sas_cmd_hdr *cmd_hdr_base; - struct hisi_sas_dq *dq = sas_dev->dq; int dlvry_queue_slot, dlvry_queue, n_elem = 0, rc, slot_idx; unsigned long flags, flags_dq = 0; int wr_q_index; @@ -1816,10 +1924,14 @@ hisi_sas_internal_abort_task_exec(struct hisi_hba *hisi_hba, int device_id, } list_add_tail(&slot->delivery, &dq->list); spin_unlock_irqrestore(&dq->lock, flags_dq); + spin_lock_irqsave(&sas_dev->lock, flags); + list_add_tail(&slot->entry, &sas_dev->list); + spin_unlock_irqrestore(&sas_dev->lock, flags); dlvry_queue = dq->id; dlvry_queue_slot = wr_q_index; + slot->device_id = sas_dev->device_id; slot->n_elem = n_elem; slot->dlvry_queue = dlvry_queue; slot->dlvry_queue_slot = dlvry_queue_slot; @@ -1843,7 +1955,6 @@ hisi_sas_internal_abort_task_exec(struct hisi_hba *hisi_hba, int device_id, WRITE_ONCE(slot->ready, 1); /* send abort command to the chip */ spin_lock_irqsave(&dq->lock, flags); - list_add_tail(&slot->entry, &sas_dev->list); hisi_hba->hw->start_delivery(dq); spin_unlock_irqrestore(&dq->lock, flags); @@ -1858,18 +1969,19 @@ err_out: } /** - * hisi_sas_internal_task_abort -- execute an internal + * _hisi_sas_internal_task_abort -- execute an internal * abort command for single IO command or a device * @hisi_hba: host controller struct * @device: domain device * @abort_flag: mode of operation, device or single IO * @tag: tag of IO to be aborted (only relevant to single * IO mode) + * @dq: delivery queue for this internal abort command */ static int -hisi_sas_internal_task_abort(struct hisi_hba *hisi_hba, - struct domain_device *device, - int abort_flag, int tag) +_hisi_sas_internal_task_abort(struct hisi_hba *hisi_hba, + struct domain_device *device, int abort_flag, + int tag, struct hisi_sas_dq *dq) { struct sas_task *task; struct hisi_sas_device *sas_dev = device->lldd_dev; @@ -1893,11 +2005,11 @@ hisi_sas_internal_task_abort(struct hisi_hba *hisi_hba, task->task_proto = device->tproto; task->task_done = hisi_sas_task_done; task->slow_task->timer.function = hisi_sas_tmf_timedout; - task->slow_task->timer.expires = jiffies + INTERNAL_ABORT_TIMEOUT*HZ; + task->slow_task->timer.expires = jiffies + INTERNAL_ABORT_TIMEOUT * HZ; add_timer(&task->slow_task->timer); res = hisi_sas_internal_abort_task_exec(hisi_hba, sas_dev->device_id, - task, abort_flag, tag); + task, abort_flag, tag, dq); if (res) { del_timer(&task->slow_task->timer); dev_err(dev, "internal task abort: executing internal task failed: %d\n", @@ -1923,6 +2035,7 @@ hisi_sas_internal_task_abort(struct hisi_hba *hisi_hba, slot->task = NULL; } dev_err(dev, "internal task abort: timeout and not done.\n"); + res = -EIO; goto exit; } else @@ -1953,6 +2066,46 @@ exit: return res; } +static int +hisi_sas_internal_task_abort(struct hisi_hba *hisi_hba, + struct domain_device *device, + int abort_flag, int tag) +{ + struct hisi_sas_slot *slot; + struct device *dev = hisi_hba->dev; + struct hisi_sas_dq *dq; + int i, rc; + + switch (abort_flag) { + case HISI_SAS_INT_ABT_CMD: + slot = &hisi_hba->slot_info[tag]; + dq = &hisi_hba->dq[slot->dlvry_queue]; + return _hisi_sas_internal_task_abort(hisi_hba, device, + abort_flag, tag, dq); + case HISI_SAS_INT_ABT_DEV: + for (i = 0; i < hisi_hba->cq_nvecs; i++) { + struct hisi_sas_cq *cq = &hisi_hba->cq[i]; + const struct cpumask *mask = cq->pci_irq_mask; + + if (mask && !cpumask_intersects(cpu_online_mask, mask)) + continue; + dq = &hisi_hba->dq[i]; + rc = _hisi_sas_internal_task_abort(hisi_hba, device, + abort_flag, tag, + dq); + if (rc) + return rc; + } + break; + default: + dev_err(dev, "Unrecognised internal abort flag (%d)\n", + abort_flag); + return -EINVAL; + } + + return 0; +} + static void hisi_sas_port_formed(struct asd_sas_phy *sas_phy) { hisi_sas_port_notify_formed(sas_phy); @@ -2019,7 +2172,7 @@ void hisi_sas_kill_tasklets(struct hisi_hba *hisi_hba) { int i; - for (i = 0; i < hisi_hba->queue_count; i++) { + for (i = 0; i < hisi_hba->cq_nvecs; i++) { struct hisi_sas_cq *cq = &hisi_hba->cq[i]; tasklet_kill(&cq->tasklet); @@ -2048,14 +2201,18 @@ static struct sas_domain_function_template hisi_sas_transport_ops = { void hisi_sas_init_mem(struct hisi_hba *hisi_hba) { - int i, s, max_command_entries = hisi_hba->hw->max_command_entries; + int i, s, j, max_command_entries = hisi_hba->hw->max_command_entries; + struct hisi_sas_breakpoint *sata_breakpoint = hisi_hba->sata_breakpoint; for (i = 0; i < hisi_hba->queue_count; i++) { struct hisi_sas_cq *cq = &hisi_hba->cq[i]; struct hisi_sas_dq *dq = &hisi_hba->dq[i]; + struct hisi_sas_cmd_hdr *cmd_hdr = hisi_hba->cmd_hdr[i]; + + s = sizeof(struct hisi_sas_cmd_hdr); + for (j = 0; j < HISI_SAS_QUEUE_SLOTS; j++) + memset(&cmd_hdr[j], 0, s); - s = sizeof(struct hisi_sas_cmd_hdr) * HISI_SAS_QUEUE_SLOTS; - memset(hisi_hba->cmd_hdr[i], 0, s); dq->wr_point = 0; s = hisi_hba->hw->complete_hdr_size * HISI_SAS_QUEUE_SLOTS; @@ -2072,12 +2229,13 @@ void hisi_sas_init_mem(struct hisi_hba *hisi_hba) s = max_command_entries * sizeof(struct hisi_sas_breakpoint); memset(hisi_hba->breakpoint, 0, s); - s = HISI_SAS_MAX_ITCT_ENTRIES * sizeof(struct hisi_sas_sata_breakpoint); - memset(hisi_hba->sata_breakpoint, 0, s); + s = sizeof(struct hisi_sas_sata_breakpoint); + for (j = 0; j < HISI_SAS_MAX_ITCT_ENTRIES; j++) + memset(&sata_breakpoint[j], 0, s); } EXPORT_SYMBOL_GPL(hisi_sas_init_mem); -int hisi_sas_alloc(struct hisi_hba *hisi_hba, struct Scsi_Host *shost) +int hisi_sas_alloc(struct hisi_hba *hisi_hba) { struct device *dev = hisi_hba->dev; int i, j, s, max_command_entries = hisi_hba->hw->max_command_entries; @@ -2095,7 +2253,6 @@ int hisi_sas_alloc(struct hisi_hba *hisi_hba, struct Scsi_Host *shost) for (i = 0; i < HISI_SAS_MAX_DEVICES; i++) { hisi_hba->devices[i].dev_type = SAS_PHY_UNUSED; hisi_hba->devices[i].device_id = i; - hisi_hba->devices[i].dev_status = HISI_SAS_DEV_NORMAL; } for (i = 0; i < hisi_hba->queue_count; i++) { @@ -2131,10 +2288,9 @@ int hisi_sas_alloc(struct hisi_hba *hisi_hba, struct Scsi_Host *shost) s = HISI_SAS_MAX_ITCT_ENTRIES * sizeof(struct hisi_sas_itct); hisi_hba->itct = dmam_alloc_coherent(dev, s, &hisi_hba->itct_dma, - GFP_KERNEL); + GFP_KERNEL | __GFP_ZERO); if (!hisi_hba->itct) goto err_out; - memset(hisi_hba->itct, 0, s); hisi_hba->slot_info = devm_kcalloc(dev, max_command_entries, sizeof(struct hisi_sas_slot), @@ -2144,19 +2300,24 @@ int hisi_sas_alloc(struct hisi_hba *hisi_hba, struct Scsi_Host *shost) /* roundup to avoid overly large block size */ max_command_entries_ru = roundup(max_command_entries, 64); - sz_slot_buf_ru = roundup(sizeof(struct hisi_sas_slot_buf_table), 64); + if (hisi_hba->prot_mask & HISI_SAS_DIX_PROT_MASK) + sz_slot_buf_ru = sizeof(struct hisi_sas_slot_dif_buf_table); + else + sz_slot_buf_ru = sizeof(struct hisi_sas_slot_buf_table); + sz_slot_buf_ru = roundup(sz_slot_buf_ru, 64); s = lcm(max_command_entries_ru, sz_slot_buf_ru); blk_cnt = (max_command_entries_ru * sz_slot_buf_ru) / s; slots_per_blk = s / sz_slot_buf_ru; + for (i = 0; i < blk_cnt; i++) { - struct hisi_sas_slot_buf_table *buf; - dma_addr_t buf_dma; int slot_index = i * slots_per_blk; + dma_addr_t buf_dma; + void *buf; - buf = dmam_alloc_coherent(dev, s, &buf_dma, GFP_KERNEL); + buf = dmam_alloc_coherent(dev, s, &buf_dma, + GFP_KERNEL | __GFP_ZERO); if (!buf) goto err_out; - memset(buf, 0, s); for (j = 0; j < slots_per_blk; j++, slot_index++) { struct hisi_sas_slot *slot; @@ -2166,8 +2327,8 @@ int hisi_sas_alloc(struct hisi_hba *hisi_hba, struct Scsi_Host *shost) slot->buf_dma = buf_dma; slot->idx = slot_index; - buf++; - buf_dma += sizeof(*buf); + buf += sz_slot_buf_ru; + buf_dma += sz_slot_buf_ru; } } @@ -2365,7 +2526,7 @@ static struct Scsi_Host *hisi_sas_shost_alloc(struct platform_device *pdev, goto err_out; } - if (hisi_sas_alloc(hisi_hba, shost)) { + if (hisi_sas_alloc(hisi_hba)) { hisi_sas_free(hisi_hba); goto err_out; } @@ -2461,6 +2622,555 @@ err_out_ha: } EXPORT_SYMBOL_GPL(hisi_sas_probe); +struct dentry *hisi_sas_debugfs_dir; + +static void hisi_sas_debugfs_snapshot_cq_reg(struct hisi_hba *hisi_hba) +{ + int queue_entry_size = hisi_hba->hw->complete_hdr_size; + int i; + + for (i = 0; i < hisi_hba->queue_count; i++) + memcpy(hisi_hba->debugfs_complete_hdr[i], + hisi_hba->complete_hdr[i], + HISI_SAS_QUEUE_SLOTS * queue_entry_size); +} + +static void hisi_sas_debugfs_snapshot_dq_reg(struct hisi_hba *hisi_hba) +{ + int queue_entry_size = sizeof(struct hisi_sas_cmd_hdr); + int i; + + for (i = 0; i < hisi_hba->queue_count; i++) { + struct hisi_sas_cmd_hdr *debugfs_cmd_hdr, *cmd_hdr; + int j; + + debugfs_cmd_hdr = hisi_hba->debugfs_cmd_hdr[i]; + cmd_hdr = hisi_hba->cmd_hdr[i]; + + for (j = 0; j < HISI_SAS_QUEUE_SLOTS; j++) + memcpy(&debugfs_cmd_hdr[j], &cmd_hdr[j], + queue_entry_size); + } +} + +static void hisi_sas_debugfs_snapshot_port_reg(struct hisi_hba *hisi_hba) +{ + const struct hisi_sas_debugfs_reg *port = + hisi_hba->hw->debugfs_reg_port; + int i, phy_cnt; + u32 offset; + u32 *databuf; + + for (phy_cnt = 0; phy_cnt < hisi_hba->n_phy; phy_cnt++) { + databuf = (u32 *)hisi_hba->debugfs_port_reg[phy_cnt]; + for (i = 0; i < port->count; i++, databuf++) { + offset = port->base_off + 4 * i; + *databuf = port->read_port_reg(hisi_hba, phy_cnt, + offset); + } + } +} + +static void hisi_sas_debugfs_snapshot_global_reg(struct hisi_hba *hisi_hba) +{ + u32 *databuf = (u32 *)hisi_hba->debugfs_global_reg; + const struct hisi_sas_debugfs_reg *global = + hisi_hba->hw->debugfs_reg_global; + int i; + + for (i = 0; i < global->count; i++, databuf++) + *databuf = global->read_global_reg(hisi_hba, 4 * i); +} + +static void hisi_sas_debugfs_snapshot_itct_reg(struct hisi_hba *hisi_hba) +{ + void *databuf = hisi_hba->debugfs_itct; + struct hisi_sas_itct *itct; + int i; + + itct = hisi_hba->itct; + + for (i = 0; i < HISI_SAS_MAX_ITCT_ENTRIES; i++, itct++) { + memcpy(databuf, itct, sizeof(struct hisi_sas_itct)); + databuf += sizeof(struct hisi_sas_itct); + } +} + +static void hisi_sas_debugfs_snapshot_iost_reg(struct hisi_hba *hisi_hba) +{ + int max_command_entries = hisi_hba->hw->max_command_entries; + void *databuf = hisi_hba->debugfs_iost; + struct hisi_sas_iost *iost; + int i; + + iost = hisi_hba->iost; + + for (i = 0; i < max_command_entries; i++, iost++) { + memcpy(databuf, iost, sizeof(struct hisi_sas_iost)); + databuf += sizeof(struct hisi_sas_iost); + } +} + +static const char * +hisi_sas_debugfs_to_reg_name(int off, int base_off, + const struct hisi_sas_debugfs_reg_lu *lu) +{ + for (; lu->name; lu++) { + if (off == lu->off - base_off) + return lu->name; + } + + return NULL; +} + +static void hisi_sas_debugfs_print_reg(u32 *regs_val, const void *ptr, + struct seq_file *s) +{ + const struct hisi_sas_debugfs_reg *reg = ptr; + int i; + + for (i = 0; i < reg->count; i++) { + int off = i * 4; + const char *name; + + name = hisi_sas_debugfs_to_reg_name(off, reg->base_off, + reg->lu); + + if (name) + seq_printf(s, "0x%08x 0x%08x %s\n", off, + regs_val[i], name); + else + seq_printf(s, "0x%08x 0x%08x\n", off, + regs_val[i]); + } +} + +static int hisi_sas_debugfs_global_show(struct seq_file *s, void *p) +{ + struct hisi_hba *hisi_hba = s->private; + const struct hisi_sas_hw *hw = hisi_hba->hw; + const struct hisi_sas_debugfs_reg *reg_global = hw->debugfs_reg_global; + + hisi_sas_debugfs_print_reg(hisi_hba->debugfs_global_reg, + reg_global, s); + + return 0; +} + +static int hisi_sas_debugfs_global_open(struct inode *inode, struct file *filp) +{ + return single_open(filp, hisi_sas_debugfs_global_show, + inode->i_private); +} + +static const struct file_operations hisi_sas_debugfs_global_fops = { + .open = hisi_sas_debugfs_global_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, + .owner = THIS_MODULE, +}; + +static int hisi_sas_debugfs_port_show(struct seq_file *s, void *p) +{ + struct hisi_sas_phy *phy = s->private; + struct hisi_hba *hisi_hba = phy->hisi_hba; + const struct hisi_sas_hw *hw = hisi_hba->hw; + const struct hisi_sas_debugfs_reg *reg_port = hw->debugfs_reg_port; + u32 *databuf = hisi_hba->debugfs_port_reg[phy->sas_phy.id]; + + hisi_sas_debugfs_print_reg(databuf, reg_port, s); + + return 0; +} + +static int hisi_sas_debugfs_port_open(struct inode *inode, struct file *filp) +{ + return single_open(filp, hisi_sas_debugfs_port_show, inode->i_private); +} + +static const struct file_operations hisi_sas_debugfs_port_fops = { + .open = hisi_sas_debugfs_port_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, + .owner = THIS_MODULE, +}; + +static int hisi_sas_show_row_64(struct seq_file *s, int index, + int sz, __le64 *ptr) +{ + int i; + + /* completion header size not fixed per HW version */ + seq_printf(s, "index %04d:\n\t", index); + for (i = 1; i <= sz / 8; i++, ptr++) { + seq_printf(s, " 0x%016llx", le64_to_cpu(*ptr)); + if (!(i % 2)) + seq_puts(s, "\n\t"); + } + + seq_puts(s, "\n"); + + return 0; +} + +static int hisi_sas_show_row_32(struct seq_file *s, int index, + int sz, __le32 *ptr) +{ + int i; + + /* completion header size not fixed per HW version */ + seq_printf(s, "index %04d:\n\t", index); + for (i = 1; i <= sz / 4; i++, ptr++) { + seq_printf(s, " 0x%08x", le32_to_cpu(*ptr)); + if (!(i % 4)) + seq_puts(s, "\n\t"); + } + seq_puts(s, "\n"); + + return 0; +} + +static int hisi_sas_cq_show_slot(struct seq_file *s, int slot, void *cq_ptr) +{ + struct hisi_sas_cq *cq = cq_ptr; + struct hisi_hba *hisi_hba = cq->hisi_hba; + void *complete_queue = hisi_hba->debugfs_complete_hdr[cq->id]; + __le32 *complete_hdr = complete_queue + + (hisi_hba->hw->complete_hdr_size * slot); + + return hisi_sas_show_row_32(s, slot, + hisi_hba->hw->complete_hdr_size, + complete_hdr); +} + +static int hisi_sas_debugfs_cq_show(struct seq_file *s, void *p) +{ + struct hisi_sas_cq *cq = s->private; + int slot, ret; + + for (slot = 0; slot < HISI_SAS_QUEUE_SLOTS; slot++) { + ret = hisi_sas_cq_show_slot(s, slot, cq); + if (ret) + return ret; + } + return 0; +} + +static int hisi_sas_debugfs_cq_open(struct inode *inode, struct file *filp) +{ + return single_open(filp, hisi_sas_debugfs_cq_show, inode->i_private); +} + +static const struct file_operations hisi_sas_debugfs_cq_fops = { + .open = hisi_sas_debugfs_cq_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, + .owner = THIS_MODULE, +}; + +static int hisi_sas_dq_show_slot(struct seq_file *s, int slot, void *dq_ptr) +{ + struct hisi_sas_dq *dq = dq_ptr; + struct hisi_hba *hisi_hba = dq->hisi_hba; + void *cmd_queue = hisi_hba->debugfs_cmd_hdr[dq->id]; + __le32 *cmd_hdr = cmd_queue + + sizeof(struct hisi_sas_cmd_hdr) * slot; + + return hisi_sas_show_row_32(s, slot, sizeof(struct hisi_sas_cmd_hdr), + cmd_hdr); +} + +static int hisi_sas_debugfs_dq_show(struct seq_file *s, void *p) +{ + int slot, ret; + + for (slot = 0; slot < HISI_SAS_QUEUE_SLOTS; slot++) { + ret = hisi_sas_dq_show_slot(s, slot, s->private); + if (ret) + return ret; + } + return 0; +} + +static int hisi_sas_debugfs_dq_open(struct inode *inode, struct file *filp) +{ + return single_open(filp, hisi_sas_debugfs_dq_show, inode->i_private); +} + +static const struct file_operations hisi_sas_debugfs_dq_fops = { + .open = hisi_sas_debugfs_dq_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, + .owner = THIS_MODULE, +}; + +static int hisi_sas_debugfs_iost_show(struct seq_file *s, void *p) +{ + struct hisi_hba *hisi_hba = s->private; + struct hisi_sas_iost *debugfs_iost = hisi_hba->debugfs_iost; + int i, ret, max_command_entries = hisi_hba->hw->max_command_entries; + __le64 *iost = &debugfs_iost->qw0; + + for (i = 0; i < max_command_entries; i++, debugfs_iost++) { + ret = hisi_sas_show_row_64(s, i, sizeof(*debugfs_iost), + iost); + if (ret) + return ret; + } + + return 0; +} + +static int hisi_sas_debugfs_iost_open(struct inode *inode, struct file *filp) +{ + return single_open(filp, hisi_sas_debugfs_iost_show, inode->i_private); +} + +static const struct file_operations hisi_sas_debugfs_iost_fops = { + .open = hisi_sas_debugfs_iost_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, + .owner = THIS_MODULE, +}; + +static int hisi_sas_debugfs_itct_show(struct seq_file *s, void *p) +{ + int i, ret; + struct hisi_hba *hisi_hba = s->private; + struct hisi_sas_itct *debugfs_itct = hisi_hba->debugfs_itct; + __le64 *itct = &debugfs_itct->qw0; + + for (i = 0; i < HISI_SAS_MAX_ITCT_ENTRIES; i++, debugfs_itct++) { + ret = hisi_sas_show_row_64(s, i, sizeof(*debugfs_itct), + itct); + if (ret) + return ret; + } + + return 0; +} + +static int hisi_sas_debugfs_itct_open(struct inode *inode, struct file *filp) +{ + return single_open(filp, hisi_sas_debugfs_itct_show, inode->i_private); +} + +static const struct file_operations hisi_sas_debugfs_itct_fops = { + .open = hisi_sas_debugfs_itct_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, + .owner = THIS_MODULE, +}; + +static void hisi_sas_debugfs_create_files(struct hisi_hba *hisi_hba) +{ + struct dentry *dump_dentry; + struct dentry *dentry; + char name[256]; + int p; + int c; + int d; + + /* Create dump dir inside device dir */ + dump_dentry = debugfs_create_dir("dump", hisi_hba->debugfs_dir); + hisi_hba->debugfs_dump_dentry = dump_dentry; + + debugfs_create_file("global", 0400, dump_dentry, hisi_hba, + &hisi_sas_debugfs_global_fops); + + /* Create port dir and files */ + dentry = debugfs_create_dir("port", dump_dentry); + for (p = 0; p < hisi_hba->n_phy; p++) { + snprintf(name, 256, "%d", p); + + debugfs_create_file(name, 0400, dentry, &hisi_hba->phy[p], + &hisi_sas_debugfs_port_fops); + } + + /* Create CQ dir and files */ + dentry = debugfs_create_dir("cq", dump_dentry); + for (c = 0; c < hisi_hba->queue_count; c++) { + snprintf(name, 256, "%d", c); + + debugfs_create_file(name, 0400, dentry, &hisi_hba->cq[c], + &hisi_sas_debugfs_cq_fops); + } + + /* Create DQ dir and files */ + dentry = debugfs_create_dir("dq", dump_dentry); + for (d = 0; d < hisi_hba->queue_count; d++) { + snprintf(name, 256, "%d", d); + + debugfs_create_file(name, 0400, dentry, &hisi_hba->dq[d], + &hisi_sas_debugfs_dq_fops); + } + + debugfs_create_file("iost", 0400, dump_dentry, hisi_hba, + &hisi_sas_debugfs_iost_fops); + + debugfs_create_file("itct", 0400, dump_dentry, hisi_hba, + &hisi_sas_debugfs_itct_fops); + + return; +} + +static void hisi_sas_debugfs_snapshot_regs(struct hisi_hba *hisi_hba) +{ + hisi_hba->hw->snapshot_prepare(hisi_hba); + + hisi_sas_debugfs_snapshot_global_reg(hisi_hba); + hisi_sas_debugfs_snapshot_port_reg(hisi_hba); + hisi_sas_debugfs_snapshot_cq_reg(hisi_hba); + hisi_sas_debugfs_snapshot_dq_reg(hisi_hba); + hisi_sas_debugfs_snapshot_itct_reg(hisi_hba); + hisi_sas_debugfs_snapshot_iost_reg(hisi_hba); + + hisi_sas_debugfs_create_files(hisi_hba); + + hisi_hba->hw->snapshot_restore(hisi_hba); +} + +static ssize_t hisi_sas_debugfs_trigger_dump_write(struct file *file, + const char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct hisi_hba *hisi_hba = file->f_inode->i_private; + char buf[8]; + + /* A bit racy, but don't care too much since it's only debugfs */ + if (hisi_hba->debugfs_snapshot) + return -EFAULT; + + if (count > 8) + return -EFAULT; + + if (copy_from_user(buf, user_buf, count)) + return -EFAULT; + + if (buf[0] != '1') + return -EFAULT; + + queue_work(hisi_hba->wq, &hisi_hba->debugfs_work); + + return count; +} + +static const struct file_operations hisi_sas_debugfs_trigger_dump_fops = { + .write = &hisi_sas_debugfs_trigger_dump_write, + .owner = THIS_MODULE, +}; + +void hisi_sas_debugfs_work_handler(struct work_struct *work) +{ + struct hisi_hba *hisi_hba = + container_of(work, struct hisi_hba, debugfs_work); + + if (hisi_hba->debugfs_snapshot) + return; + hisi_hba->debugfs_snapshot = true; + + hisi_sas_debugfs_snapshot_regs(hisi_hba); +} +EXPORT_SYMBOL_GPL(hisi_sas_debugfs_work_handler); + +void hisi_sas_debugfs_init(struct hisi_hba *hisi_hba) +{ + int max_command_entries = hisi_hba->hw->max_command_entries; + struct device *dev = hisi_hba->dev; + int p, i, c, d; + size_t sz; + + hisi_hba->debugfs_dir = debugfs_create_dir(dev_name(dev), + hisi_sas_debugfs_dir); + debugfs_create_file("trigger_dump", 0600, + hisi_hba->debugfs_dir, + hisi_hba, + &hisi_sas_debugfs_trigger_dump_fops); + + /* Alloc buffer for global */ + sz = hisi_hba->hw->debugfs_reg_global->count * 4; + hisi_hba->debugfs_global_reg = + devm_kmalloc(dev, sz, GFP_KERNEL); + + if (!hisi_hba->debugfs_global_reg) + goto fail_global; + + /* Alloc buffer for port */ + sz = hisi_hba->hw->debugfs_reg_port->count * 4; + for (p = 0; p < hisi_hba->n_phy; p++) { + hisi_hba->debugfs_port_reg[p] = + devm_kmalloc(dev, sz, GFP_KERNEL); + + if (!hisi_hba->debugfs_port_reg[p]) + goto fail_port; + } + + /* Alloc buffer for cq */ + sz = hisi_hba->hw->complete_hdr_size * HISI_SAS_QUEUE_SLOTS; + for (c = 0; c < hisi_hba->queue_count; c++) { + hisi_hba->debugfs_complete_hdr[c] = + devm_kmalloc(dev, sz, GFP_KERNEL); + + if (!hisi_hba->debugfs_complete_hdr[c]) + goto fail_cq; + } + + /* Alloc buffer for dq */ + sz = sizeof(struct hisi_sas_cmd_hdr) * HISI_SAS_QUEUE_SLOTS; + for (d = 0; d < hisi_hba->queue_count; d++) { + hisi_hba->debugfs_cmd_hdr[d] = + devm_kmalloc(dev, sz, GFP_KERNEL); + + if (!hisi_hba->debugfs_cmd_hdr[d]) + goto fail_iost_dq; + } + + /* Alloc buffer for iost */ + sz = max_command_entries * sizeof(struct hisi_sas_iost); + + hisi_hba->debugfs_iost = devm_kmalloc(dev, sz, GFP_KERNEL); + if (!hisi_hba->debugfs_iost) + goto fail_iost_dq; + + /* Alloc buffer for itct */ + /* New memory allocation must be locate before itct */ + sz = HISI_SAS_MAX_ITCT_ENTRIES * sizeof(struct hisi_sas_itct); + + hisi_hba->debugfs_itct = devm_kmalloc(dev, sz, GFP_KERNEL); + if (!hisi_hba->debugfs_itct) + goto fail_itct; + + return; +fail_itct: + devm_kfree(dev, hisi_hba->debugfs_iost); +fail_iost_dq: + for (i = 0; i < d; i++) + devm_kfree(dev, hisi_hba->debugfs_cmd_hdr[i]); +fail_cq: + for (i = 0; i < c; i++) + devm_kfree(dev, hisi_hba->debugfs_complete_hdr[i]); +fail_port: + for (i = 0; i < p; i++) + devm_kfree(dev, hisi_hba->debugfs_port_reg[i]); + devm_kfree(dev, hisi_hba->debugfs_global_reg); +fail_global: + debugfs_remove_recursive(hisi_hba->debugfs_dir); + dev_dbg(dev, "failed to init debugfs!\n"); +} +EXPORT_SYMBOL_GPL(hisi_sas_debugfs_init); + +void hisi_sas_debugfs_exit(struct hisi_hba *hisi_hba) +{ + debugfs_remove_recursive(hisi_hba->debugfs_dir); +} +EXPORT_SYMBOL_GPL(hisi_sas_debugfs_exit); + int hisi_sas_remove(struct platform_device *pdev) { struct sas_ha_struct *sha = platform_get_drvdata(pdev); @@ -2479,18 +3189,28 @@ int hisi_sas_remove(struct platform_device *pdev) } EXPORT_SYMBOL_GPL(hisi_sas_remove); +bool hisi_sas_debugfs_enable; +EXPORT_SYMBOL_GPL(hisi_sas_debugfs_enable); +module_param_named(debugfs_enable, hisi_sas_debugfs_enable, bool, 0444); +MODULE_PARM_DESC(hisi_sas_debugfs_enable, "Enable driver debugfs (default disabled)"); + static __init int hisi_sas_init(void) { hisi_sas_stt = sas_domain_attach_transport(&hisi_sas_transport_ops); if (!hisi_sas_stt) return -ENOMEM; + if (hisi_sas_debugfs_enable) + hisi_sas_debugfs_dir = debugfs_create_dir("hisi_sas", NULL); + return 0; } static __exit void hisi_sas_exit(void) { sas_release_transport(hisi_sas_stt); + + debugfs_remove(hisi_sas_debugfs_dir); } module_init(hisi_sas_init); diff --git a/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c b/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c index 28ab52a021cf..293807443480 100644 --- a/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c +++ b/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c @@ -835,7 +835,7 @@ static void phys_init_v1_hw(struct hisi_hba *hisi_hba) mod_timer(timer, jiffies + HZ); } -static void sl_notify_v1_hw(struct hisi_hba *hisi_hba, int phy_no) +static void sl_notify_ssp_v1_hw(struct hisi_hba *hisi_hba, int phy_no) { u32 sl_control; @@ -1749,6 +1749,8 @@ static int interrupt_init_v1_hw(struct hisi_hba *hisi_hba) } } + hisi_hba->cq_nvecs = hisi_hba->queue_count; + return 0; } @@ -1826,7 +1828,7 @@ static struct scsi_host_template sht_v1_hw = { static const struct hisi_sas_hw hisi_sas_v1_hw = { .hw_init = hisi_sas_v1_init, .setup_itct = setup_itct_v1_hw, - .sl_notify = sl_notify_v1_hw, + .sl_notify_ssp = sl_notify_ssp_v1_hw, .clear_itct = clear_itct_v1_hw, .prep_smp = prep_smp_v1_hw, .prep_ssp = prep_ssp_v1_hw, diff --git a/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c b/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c index c8ebff3ba559..e40cc6b3b67b 100644 --- a/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c +++ b/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c @@ -868,12 +868,12 @@ hisi_sas_device *alloc_dev_quirk_v2_hw(struct domain_device *device) hisi_hba->devices[i].device_id = i; sas_dev = &hisi_hba->devices[i]; - sas_dev->dev_status = HISI_SAS_DEV_NORMAL; sas_dev->dev_type = device->dev_type; sas_dev->hisi_hba = hisi_hba; sas_dev->sas_device = device; sas_dev->sata_idx = sata_idx; sas_dev->dq = dq; + spin_lock_init(&sas_dev->lock); INIT_LIST_HEAD(&hisi_hba->devices[i].list); break; } @@ -1589,7 +1589,7 @@ static void phys_init_v2_hw(struct hisi_hba *hisi_hba) } } -static void sl_notify_v2_hw(struct hisi_hba *hisi_hba, int phy_no) +static void sl_notify_ssp_v2_hw(struct hisi_hba *hisi_hba, int phy_no) { u32 sl_control; @@ -2677,6 +2677,8 @@ static int phy_up_v2_hw(int phy_no, struct hisi_hba *hisi_hba) if (is_sata_phy_v2_hw(hisi_hba, phy_no)) goto end; + del_timer(&phy->timer); + if (phy_no == 8) { u32 port_state = hisi_sas_read32(hisi_hba, PORT_STATE); @@ -2756,6 +2758,7 @@ static int phy_down_v2_hw(int phy_no, struct hisi_hba *hisi_hba) struct hisi_sas_port *port = phy->port; struct device *dev = hisi_hba->dev; + del_timer(&phy->timer); hisi_sas_phy_write32(hisi_hba, phy_no, PHYCTRL_NOT_RDY_MSK, 1); phy_state = hisi_sas_read32(hisi_hba, PHY_STATE); @@ -2944,6 +2947,9 @@ static irqreturn_t int_chnl_int_v2_hw(int irq_no, void *p) if (irq_value0 & CHL_INT0_SL_RX_BCST_ACK_MSK) phy_bcast_v2_hw(phy_no, hisi_hba); + if (irq_value0 & CHL_INT0_PHY_RDY_MSK) + hisi_sas_phy_oob_ready(hisi_hba, phy_no); + hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT0, irq_value0 & (~CHL_INT0_HOTPLUG_TOUT_MSK) @@ -3227,6 +3233,8 @@ static irqreturn_t sata_int_v2_hw(int irq_no, void *p) unsigned long flags; int phy_no, offset; + del_timer(&phy->timer); + phy_no = sas_phy->id; initial_fis = &hisi_hba->initial_fis[phy_no]; fis = &initial_fis->fis; @@ -3393,6 +3401,8 @@ static int interrupt_init_v2_hw(struct hisi_hba *hisi_hba) tasklet_init(t, cq_tasklet_v2_hw, (unsigned long)cq); } + hisi_hba->cq_nvecs = hisi_hba->queue_count; + return 0; free_cq_int_irqs: @@ -3542,8 +3552,8 @@ static int write_gpio_v2_hw(struct hisi_hba *hisi_hba, u8 reg_type, return 0; } -static void wait_cmds_complete_timeout_v2_hw(struct hisi_hba *hisi_hba, - int delay_ms, int timeout_ms) +static int wait_cmds_complete_timeout_v2_hw(struct hisi_hba *hisi_hba, + int delay_ms, int timeout_ms) { struct device *dev = hisi_hba->dev; int entries, entries_old = 0, time; @@ -3557,7 +3567,12 @@ static void wait_cmds_complete_timeout_v2_hw(struct hisi_hba *hisi_hba, msleep(delay_ms); } + if (time >= timeout_ms) + return -ETIMEDOUT; + dev_dbg(dev, "wait commands complete %dms\n", time); + + return 0; } static struct device_attribute *host_attrs_v2_hw[] = { @@ -3590,7 +3605,7 @@ static const struct hisi_sas_hw hisi_sas_v2_hw = { .setup_itct = setup_itct_v2_hw, .slot_index_alloc = slot_index_alloc_quirk_v2_hw, .alloc_dev = alloc_dev_quirk_v2_hw, - .sl_notify = sl_notify_v2_hw, + .sl_notify_ssp = sl_notify_ssp_v2_hw, .get_wideport_bitmap = get_wideport_bitmap_v2_hw, .clear_itct = clear_itct_v2_hw, .free_device = free_device_v2_hw, diff --git a/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c b/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c index e0570fd8466e..9ec8848ec541 100644 --- a/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c +++ b/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c @@ -11,7 +11,7 @@ #include "hisi_sas.h" #define DRV_NAME "hisi_sas_v3_hw" -/* global registers need init*/ +/* global registers need init */ #define DLVRY_QUEUE_ENABLE 0x0 #define IOST_BASE_ADDR_LO 0x8 #define IOST_BASE_ADDR_HI 0xc @@ -186,6 +186,7 @@ #define CHL_INT0_MSK (PORT_BASE + 0x1c0) #define CHL_INT1_MSK (PORT_BASE + 0x1c4) #define CHL_INT2_MSK (PORT_BASE + 0x1c8) +#define SAS_EC_INT_COAL_TIME (PORT_BASE + 0x1cc) #define CHL_INT_COAL_EN (PORT_BASE + 0x1d0) #define SAS_RX_TRAIN_TIMER (PORT_BASE + 0x2a4) #define PHY_CTRL_RDY_MSK (PORT_BASE + 0x2b0) @@ -205,6 +206,7 @@ #define ERR_CNT_DWS_LOST (PORT_BASE + 0x380) #define ERR_CNT_RESET_PROB (PORT_BASE + 0x384) #define ERR_CNT_INVLD_DW (PORT_BASE + 0x390) +#define ERR_CNT_CODE_ERR (PORT_BASE + 0x394) #define ERR_CNT_DISP_ERR (PORT_BASE + 0x398) #define DEFAULT_ITCT_HW 2048 /* reset value, not reprogrammed */ @@ -397,6 +399,11 @@ struct hisi_sas_err_record_v3 { #define USR_DATA_BLOCK_SZ_OFF 20 #define USR_DATA_BLOCK_SZ_MSK (0x3 << USR_DATA_BLOCK_SZ_OFF) #define T10_CHK_MSK_OFF 16 +#define T10_CHK_REF_TAG_MSK (0xf0 << T10_CHK_MSK_OFF) +#define T10_CHK_APP_TAG_MSK (0xc << T10_CHK_MSK_OFF) + +#define BASE_VECTORS_V3_HW 16 +#define MIN_AFFINE_VECTORS_V3_HW (BASE_VECTORS_V3_HW + 1) static bool hisi_sas_intr_conv; MODULE_PARM_DESC(intr_conv, "interrupt converge enable (0-1)"); @@ -406,6 +413,11 @@ static int prot_mask; module_param(prot_mask, int, 0); MODULE_PARM_DESC(prot_mask, " host protection capabilities mask, def=0x0 "); +static bool auto_affine_msi_experimental; +module_param(auto_affine_msi_experimental, bool, 0444); +MODULE_PARM_DESC(auto_affine_msi_experimental, "Enable auto-affinity of MSI IRQs as experimental:\n" + "default is off"); + static u32 hisi_sas_read32(struct hisi_hba *hisi_hba, u32 off) { void __iomem *regs = hisi_hba->regs + off; @@ -716,7 +728,7 @@ static void clear_itct_v3_hw(struct hisi_hba *hisi_hba, hisi_sas_write32(hisi_hba, ENT_INT_SRC3, ENT_INT_SRC3_ITC_INT_MSK); - /* clear the itct table*/ + /* clear the itct table */ reg_val = ITCT_CLR_EN_MSK | (dev_id & ITCT_DEV_MSK); hisi_sas_write32(hisi_hba, ITCT_CLR, reg_val); @@ -868,7 +880,7 @@ static void phys_init_v3_hw(struct hisi_hba *hisi_hba) } } -static void sl_notify_v3_hw(struct hisi_hba *hisi_hba, int phy_no) +static void sl_notify_ssp_v3_hw(struct hisi_hba *hisi_hba, int phy_no) { u32 sl_control; @@ -967,19 +979,44 @@ static void prep_prd_sge_v3_hw(struct hisi_hba *hisi_hba, hdr->prd_table_addr = cpu_to_le64(hisi_sas_sge_addr_dma(slot)); - hdr->sg_len = cpu_to_le32(n_elem << CMD_HDR_DATA_SGL_LEN_OFF); + hdr->sg_len |= cpu_to_le32(n_elem << CMD_HDR_DATA_SGL_LEN_OFF); +} + +static void prep_prd_sge_dif_v3_hw(struct hisi_hba *hisi_hba, + struct hisi_sas_slot *slot, + struct hisi_sas_cmd_hdr *hdr, + struct scatterlist *scatter, + int n_elem) +{ + struct hisi_sas_sge_dif_page *sge_dif_page; + struct scatterlist *sg; + int i; + + sge_dif_page = hisi_sas_sge_dif_addr_mem(slot); + + for_each_sg(scatter, sg, n_elem, i) { + struct hisi_sas_sge *entry = &sge_dif_page->sge[i]; + + entry->addr = cpu_to_le64(sg_dma_address(sg)); + entry->page_ctrl_0 = 0; + entry->page_ctrl_1 = 0; + entry->data_len = cpu_to_le32(sg_dma_len(sg)); + entry->data_off = 0; + } + + hdr->dif_prd_table_addr = + cpu_to_le64(hisi_sas_sge_dif_addr_dma(slot)); + + hdr->sg_len |= cpu_to_le32(n_elem << CMD_HDR_DIF_SGL_LEN_OFF); } static u32 get_prot_chk_msk_v3_hw(struct scsi_cmnd *scsi_cmnd) { unsigned char prot_flags = scsi_cmnd->prot_flags; - if (prot_flags & SCSI_PROT_TRANSFER_PI) { - if (prot_flags & SCSI_PROT_REF_CHECK) - return 0xc << 16; - return 0xfc << 16; - } - return 0; + if (prot_flags & SCSI_PROT_REF_CHECK) + return T10_CHK_APP_TAG_MSK; + return T10_CHK_REF_TAG_MSK | T10_CHK_APP_TAG_MSK; } static void fill_prot_v3_hw(struct scsi_cmnd *scsi_cmnd, @@ -990,15 +1027,33 @@ static void fill_prot_v3_hw(struct scsi_cmnd *scsi_cmnd, u32 lbrt_chk_val = t10_pi_ref_tag(scsi_cmnd->request); switch (prot_op) { + case SCSI_PROT_READ_INSERT: + prot->dw0 |= T10_INSRT_EN_MSK; + prot->lbrtgv = lbrt_chk_val; + break; case SCSI_PROT_READ_STRIP: prot->dw0 |= (T10_RMV_EN_MSK | T10_CHK_EN_MSK); prot->lbrtcv = lbrt_chk_val; prot->dw4 |= get_prot_chk_msk_v3_hw(scsi_cmnd); break; + case SCSI_PROT_READ_PASS: + prot->dw0 |= T10_CHK_EN_MSK; + prot->lbrtcv = lbrt_chk_val; + prot->dw4 |= get_prot_chk_msk_v3_hw(scsi_cmnd); + break; case SCSI_PROT_WRITE_INSERT: prot->dw0 |= T10_INSRT_EN_MSK; prot->lbrtgv = lbrt_chk_val; break; + case SCSI_PROT_WRITE_STRIP: + prot->dw0 |= (T10_RMV_EN_MSK | T10_CHK_EN_MSK); + prot->lbrtcv = lbrt_chk_val; + break; + case SCSI_PROT_WRITE_PASS: + prot->dw0 |= T10_CHK_EN_MSK; + prot->lbrtcv = lbrt_chk_val; + prot->dw4 |= get_prot_chk_msk_v3_hw(scsi_cmnd); + break; default: WARN(1, "prot_op(0x%x) is not valid\n", prot_op); break; @@ -1033,8 +1088,8 @@ static void prep_ssp_v3_hw(struct hisi_hba *hisi_hba, struct sas_ssp_task *ssp_task = &task->ssp_task; struct scsi_cmnd *scsi_cmnd = ssp_task->cmd; struct hisi_sas_tmf_task *tmf = slot->tmf; - unsigned char prot_op = scsi_get_prot_op(scsi_cmnd); int has_data = 0, priority = !!tmf; + unsigned char prot_op; u8 *buf_cmd; u32 dw1 = 0, dw2 = 0, len = 0; @@ -1049,6 +1104,7 @@ static void prep_ssp_v3_hw(struct hisi_hba *hisi_hba, dw1 |= 2 << CMD_HDR_FRAME_TYPE_OFF; dw1 |= DIR_NO_DATA << CMD_HDR_DIR_OFF; } else { + prot_op = scsi_get_prot_op(scsi_cmnd); dw1 |= 1 << CMD_HDR_FRAME_TYPE_OFF; switch (scsi_cmnd->sc_data_direction) { case DMA_TO_DEVICE: @@ -1074,9 +1130,15 @@ static void prep_ssp_v3_hw(struct hisi_hba *hisi_hba, hdr->dw2 = cpu_to_le32(dw2); hdr->transfer_tags = cpu_to_le32(slot->idx); - if (has_data) + if (has_data) { prep_prd_sge_v3_hw(hisi_hba, slot, hdr, task->scatter, - slot->n_elem); + slot->n_elem); + + if (scsi_prot_sg_count(scsi_cmnd)) + prep_prd_sge_dif_v3_hw(hisi_hba, slot, hdr, + scsi_prot_sglist(scsi_cmnd), + slot->n_elem_dif); + } hdr->cmd_table_addr = cpu_to_le64(hisi_sas_cmd_hdr_addr_dma(slot)); hdr->sts_buffer_addr = cpu_to_le64(hisi_sas_status_buf_addr_dma(slot)); @@ -1117,18 +1179,19 @@ static void prep_ssp_v3_hw(struct hisi_hba *hisi_hba, fill_prot_v3_hw(scsi_cmnd, &prot); memcpy(buf_cmd_prot, &prot, sizeof(struct hisi_sas_protect_iu_v3_hw)); - /* * For READ, we need length of info read to memory, while for * WRITE we need length of data written to the disk. */ - if (prot_op == SCSI_PROT_WRITE_INSERT) { + if (prot_op == SCSI_PROT_WRITE_INSERT || + prot_op == SCSI_PROT_READ_INSERT || + prot_op == SCSI_PROT_WRITE_PASS || + prot_op == SCSI_PROT_READ_PASS) { unsigned int interval = scsi_prot_interval(scsi_cmnd); unsigned int ilog2_interval = ilog2(interval); len = (task->total_xfer_len >> ilog2_interval) * 8; } - } hdr->dw1 = cpu_to_le32(dw1); @@ -1288,6 +1351,7 @@ static irqreturn_t phy_up_v3_hw(int phy_no, struct hisi_hba *hisi_hba) struct device *dev = hisi_hba->dev; unsigned long flags; + del_timer(&phy->timer); hisi_sas_phy_write32(hisi_hba, phy_no, PHYCTRL_PHY_ENA_MSK, 1); port_id = hisi_sas_read32(hisi_hba, PHY_PORT_NUM_MA); @@ -1381,9 +1445,11 @@ end: static irqreturn_t phy_down_v3_hw(int phy_no, struct hisi_hba *hisi_hba) { + struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no]; u32 phy_state, sl_ctrl, txid_auto; struct device *dev = hisi_hba->dev; + del_timer(&phy->timer); hisi_sas_phy_write32(hisi_hba, phy_no, PHYCTRL_NOT_RDY_MSK, 1); phy_state = hisi_sas_read32(hisi_hba, PHY_STATE); @@ -1552,6 +1618,19 @@ static void handle_chl_int2_v3_hw(struct hisi_hba *hisi_hba, int phy_no) hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT2, irq_value); } +static void handle_chl_int0_v3_hw(struct hisi_hba *hisi_hba, int phy_no) +{ + u32 irq_value0 = hisi_sas_phy_read32(hisi_hba, phy_no, CHL_INT0); + + if (irq_value0 & CHL_INT0_PHY_RDY_MSK) + hisi_sas_phy_oob_ready(hisi_hba, phy_no); + + hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT0, + irq_value0 & (~CHL_INT0_SL_RX_BCST_ACK_MSK) + & (~CHL_INT0_SL_PHY_ENABLE_MSK) + & (~CHL_INT0_NOT_RDY_MSK)); +} + static irqreturn_t int_chnl_int_v3_hw(int irq_no, void *p) { struct hisi_hba *hisi_hba = p; @@ -1562,8 +1641,8 @@ static irqreturn_t int_chnl_int_v3_hw(int irq_no, void *p) & 0xeeeeeeee; while (irq_msk) { - u32 irq_value0 = hisi_sas_phy_read32(hisi_hba, phy_no, - CHL_INT0); + if (irq_msk & (2 << (phy_no * 4))) + handle_chl_int0_v3_hw(hisi_hba, phy_no); if (irq_msk & (4 << (phy_no * 4))) handle_chl_int1_v3_hw(hisi_hba, phy_no); @@ -1571,13 +1650,6 @@ static irqreturn_t int_chnl_int_v3_hw(int irq_no, void *p) if (irq_msk & (8 << (phy_no * 4))) handle_chl_int2_v3_hw(hisi_hba, phy_no); - if (irq_msk & (2 << (phy_no * 4)) && irq_value0) { - hisi_sas_phy_write32(hisi_hba, phy_no, - CHL_INT0, irq_value0 - & (~CHL_INT0_SL_RX_BCST_ACK_MSK) - & (~CHL_INT0_SL_PHY_ENABLE_MSK) - & (~CHL_INT0_NOT_RDY_MSK)); - } irq_msk &= ~(0xe << (phy_no * 4)); phy_no++; } @@ -1644,6 +1716,7 @@ static irqreturn_t fatal_axi_int_v3_hw(int irq_no, void *p) u32 irq_value, irq_msk; struct hisi_hba *hisi_hba = p; struct device *dev = hisi_hba->dev; + struct pci_dev *pdev = hisi_hba->pci_dev; int i; irq_msk = hisi_sas_read32(hisi_hba, ENT_INT_SRC_MSK3); @@ -1675,6 +1748,17 @@ static irqreturn_t fatal_axi_int_v3_hw(int irq_no, void *p) error->msg, irq_value); queue_work(hisi_hba->wq, &hisi_hba->rst_work); } + + if (pdev->revision < 0x21) { + u32 reg_val; + + reg_val = hisi_sas_read32(hisi_hba, + AXI_MASTER_CFG_BASE + + AM_CTRL_GLOBAL); + reg_val |= AM_CTRL_SHUTDOWN_REQ_MSK; + hisi_sas_write32(hisi_hba, AXI_MASTER_CFG_BASE + + AM_CTRL_GLOBAL, reg_val); + } } if (irq_value & BIT(ENT_INT_SRC3_ITC_INT_OFF)) { @@ -1959,21 +2043,68 @@ static irqreturn_t cq_interrupt_v3_hw(int irq_no, void *p) return IRQ_HANDLED; } +static void setup_reply_map_v3_hw(struct hisi_hba *hisi_hba, int nvecs) +{ + const struct cpumask *mask; + int queue, cpu; + + for (queue = 0; queue < nvecs; queue++) { + struct hisi_sas_cq *cq = &hisi_hba->cq[queue]; + + mask = pci_irq_get_affinity(hisi_hba->pci_dev, queue + + BASE_VECTORS_V3_HW); + if (!mask) + goto fallback; + cq->pci_irq_mask = mask; + for_each_cpu(cpu, mask) + hisi_hba->reply_map[cpu] = queue; + } + return; + +fallback: + for_each_possible_cpu(cpu) + hisi_hba->reply_map[cpu] = cpu % hisi_hba->queue_count; + /* Don't clean all CQ masks */ +} + static int interrupt_init_v3_hw(struct hisi_hba *hisi_hba) { struct device *dev = hisi_hba->dev; struct pci_dev *pdev = hisi_hba->pci_dev; int vectors, rc; int i, k; - int max_msi = HISI_SAS_MSI_COUNT_V3_HW; - - vectors = pci_alloc_irq_vectors(hisi_hba->pci_dev, 1, - max_msi, PCI_IRQ_MSI); - if (vectors < max_msi) { - dev_err(dev, "could not allocate all msi (%d)\n", vectors); - return -ENOENT; + int max_msi = HISI_SAS_MSI_COUNT_V3_HW, min_msi; + + if (auto_affine_msi_experimental) { + struct irq_affinity desc = { + .pre_vectors = BASE_VECTORS_V3_HW, + }; + + min_msi = MIN_AFFINE_VECTORS_V3_HW; + + hisi_hba->reply_map = devm_kcalloc(dev, nr_cpu_ids, + sizeof(unsigned int), + GFP_KERNEL); + if (!hisi_hba->reply_map) + return -ENOMEM; + vectors = pci_alloc_irq_vectors_affinity(hisi_hba->pci_dev, + min_msi, max_msi, + PCI_IRQ_MSI | + PCI_IRQ_AFFINITY, + &desc); + if (vectors < 0) + return -ENOENT; + setup_reply_map_v3_hw(hisi_hba, vectors - BASE_VECTORS_V3_HW); + } else { + min_msi = max_msi; + vectors = pci_alloc_irq_vectors(hisi_hba->pci_dev, min_msi, + max_msi, PCI_IRQ_MSI); + if (vectors < 0) + return vectors; } + hisi_hba->cq_nvecs = vectors - BASE_VECTORS_V3_HW; + rc = devm_request_irq(dev, pci_irq_vector(pdev, 1), int_phy_up_down_bcast_v3_hw, 0, DRV_NAME " phy", hisi_hba); @@ -2002,7 +2133,7 @@ static int interrupt_init_v3_hw(struct hisi_hba *hisi_hba) } /* Init tasklets for cq only */ - for (i = 0; i < hisi_hba->queue_count; i++) { + for (i = 0; i < hisi_hba->cq_nvecs; i++) { struct hisi_sas_cq *cq = &hisi_hba->cq[i]; struct tasklet_struct *t = &cq->tasklet; int nr = hisi_sas_intr_conv ? 16 : 16 + i; @@ -2201,8 +2332,8 @@ static int write_gpio_v3_hw(struct hisi_hba *hisi_hba, u8 reg_type, return 0; } -static void wait_cmds_complete_timeout_v3_hw(struct hisi_hba *hisi_hba, - int delay_ms, int timeout_ms) +static int wait_cmds_complete_timeout_v3_hw(struct hisi_hba *hisi_hba, + int delay_ms, int timeout_ms) { struct device *dev = hisi_hba->dev; int entries, entries_old = 0, time; @@ -2216,7 +2347,12 @@ static void wait_cmds_complete_timeout_v3_hw(struct hisi_hba *hisi_hba, msleep(delay_ms); } + if (time >= timeout_ms) + return -ETIMEDOUT; + dev_dbg(dev, "wait commands complete %dms\n", time); + + return 0; } static ssize_t intr_conv_v3_hw_show(struct device *dev, @@ -2332,6 +2468,159 @@ static struct device_attribute *host_attrs_v3_hw[] = { NULL }; +static const struct hisi_sas_debugfs_reg_lu debugfs_port_reg_lu[] = { + HISI_SAS_DEBUGFS_REG(PHY_CFG), + HISI_SAS_DEBUGFS_REG(HARD_PHY_LINKRATE), + HISI_SAS_DEBUGFS_REG(PROG_PHY_LINK_RATE), + HISI_SAS_DEBUGFS_REG(PHY_CTRL), + HISI_SAS_DEBUGFS_REG(SL_CFG), + HISI_SAS_DEBUGFS_REG(AIP_LIMIT), + HISI_SAS_DEBUGFS_REG(SL_CONTROL), + HISI_SAS_DEBUGFS_REG(RX_PRIMS_STATUS), + HISI_SAS_DEBUGFS_REG(TX_ID_DWORD0), + HISI_SAS_DEBUGFS_REG(TX_ID_DWORD1), + HISI_SAS_DEBUGFS_REG(TX_ID_DWORD2), + HISI_SAS_DEBUGFS_REG(TX_ID_DWORD3), + HISI_SAS_DEBUGFS_REG(TX_ID_DWORD4), + HISI_SAS_DEBUGFS_REG(TX_ID_DWORD5), + HISI_SAS_DEBUGFS_REG(TX_ID_DWORD6), + HISI_SAS_DEBUGFS_REG(TXID_AUTO), + HISI_SAS_DEBUGFS_REG(RX_IDAF_DWORD0), + HISI_SAS_DEBUGFS_REG(RXOP_CHECK_CFG_H), + HISI_SAS_DEBUGFS_REG(STP_LINK_TIMER), + HISI_SAS_DEBUGFS_REG(STP_LINK_TIMEOUT_STATE), + HISI_SAS_DEBUGFS_REG(CON_CFG_DRIVER), + HISI_SAS_DEBUGFS_REG(SAS_SSP_CON_TIMER_CFG), + HISI_SAS_DEBUGFS_REG(SAS_SMP_CON_TIMER_CFG), + HISI_SAS_DEBUGFS_REG(SAS_STP_CON_TIMER_CFG), + HISI_SAS_DEBUGFS_REG(CHL_INT0), + HISI_SAS_DEBUGFS_REG(CHL_INT1), + HISI_SAS_DEBUGFS_REG(CHL_INT2), + HISI_SAS_DEBUGFS_REG(CHL_INT0_MSK), + HISI_SAS_DEBUGFS_REG(CHL_INT1_MSK), + HISI_SAS_DEBUGFS_REG(CHL_INT2_MSK), + HISI_SAS_DEBUGFS_REG(SAS_EC_INT_COAL_TIME), + HISI_SAS_DEBUGFS_REG(CHL_INT_COAL_EN), + HISI_SAS_DEBUGFS_REG(SAS_RX_TRAIN_TIMER), + HISI_SAS_DEBUGFS_REG(PHY_CTRL_RDY_MSK), + HISI_SAS_DEBUGFS_REG(PHYCTRL_NOT_RDY_MSK), + HISI_SAS_DEBUGFS_REG(PHYCTRL_DWS_RESET_MSK), + HISI_SAS_DEBUGFS_REG(PHYCTRL_PHY_ENA_MSK), + HISI_SAS_DEBUGFS_REG(SL_RX_BCAST_CHK_MSK), + HISI_SAS_DEBUGFS_REG(PHYCTRL_OOB_RESTART_MSK), + HISI_SAS_DEBUGFS_REG(DMA_TX_STATUS), + HISI_SAS_DEBUGFS_REG(DMA_RX_STATUS), + HISI_SAS_DEBUGFS_REG(COARSETUNE_TIME), + HISI_SAS_DEBUGFS_REG(ERR_CNT_DWS_LOST), + HISI_SAS_DEBUGFS_REG(ERR_CNT_RESET_PROB), + HISI_SAS_DEBUGFS_REG(ERR_CNT_INVLD_DW), + HISI_SAS_DEBUGFS_REG(ERR_CNT_CODE_ERR), + HISI_SAS_DEBUGFS_REG(ERR_CNT_DISP_ERR), + {} +}; + +static const struct hisi_sas_debugfs_reg debugfs_port_reg = { + .lu = debugfs_port_reg_lu, + .count = 0x100, + .base_off = PORT_BASE, + .read_port_reg = hisi_sas_phy_read32, +}; + +static const struct hisi_sas_debugfs_reg_lu debugfs_global_reg_lu[] = { + HISI_SAS_DEBUGFS_REG(DLVRY_QUEUE_ENABLE), + HISI_SAS_DEBUGFS_REG(PHY_CONTEXT), + HISI_SAS_DEBUGFS_REG(PHY_STATE), + HISI_SAS_DEBUGFS_REG(PHY_PORT_NUM_MA), + HISI_SAS_DEBUGFS_REG(PHY_CONN_RATE), + HISI_SAS_DEBUGFS_REG(ITCT_CLR), + HISI_SAS_DEBUGFS_REG(IO_SATA_BROKEN_MSG_ADDR_LO), + HISI_SAS_DEBUGFS_REG(IO_SATA_BROKEN_MSG_ADDR_HI), + HISI_SAS_DEBUGFS_REG(SATA_INITI_D2H_STORE_ADDR_LO), + HISI_SAS_DEBUGFS_REG(SATA_INITI_D2H_STORE_ADDR_HI), + HISI_SAS_DEBUGFS_REG(CFG_MAX_TAG), + HISI_SAS_DEBUGFS_REG(HGC_SAS_TX_OPEN_FAIL_RETRY_CTRL), + HISI_SAS_DEBUGFS_REG(HGC_SAS_TXFAIL_RETRY_CTRL), + HISI_SAS_DEBUGFS_REG(HGC_GET_ITV_TIME), + HISI_SAS_DEBUGFS_REG(DEVICE_MSG_WORK_MODE), + HISI_SAS_DEBUGFS_REG(OPENA_WT_CONTI_TIME), + HISI_SAS_DEBUGFS_REG(I_T_NEXUS_LOSS_TIME), + HISI_SAS_DEBUGFS_REG(MAX_CON_TIME_LIMIT_TIME), + HISI_SAS_DEBUGFS_REG(BUS_INACTIVE_LIMIT_TIME), + HISI_SAS_DEBUGFS_REG(REJECT_TO_OPEN_LIMIT_TIME), + HISI_SAS_DEBUGFS_REG(CQ_INT_CONVERGE_EN), + HISI_SAS_DEBUGFS_REG(CFG_AGING_TIME), + HISI_SAS_DEBUGFS_REG(HGC_DFX_CFG2), + HISI_SAS_DEBUGFS_REG(CFG_ABT_SET_QUERY_IPTT), + HISI_SAS_DEBUGFS_REG(CFG_ABT_SET_IPTT_DONE), + HISI_SAS_DEBUGFS_REG(HGC_IOMB_PROC1_STATUS), + HISI_SAS_DEBUGFS_REG(CHNL_INT_STATUS), + HISI_SAS_DEBUGFS_REG(HGC_AXI_FIFO_ERR_INFO), + HISI_SAS_DEBUGFS_REG(INT_COAL_EN), + HISI_SAS_DEBUGFS_REG(OQ_INT_COAL_TIME), + HISI_SAS_DEBUGFS_REG(OQ_INT_COAL_CNT), + HISI_SAS_DEBUGFS_REG(ENT_INT_COAL_TIME), + HISI_SAS_DEBUGFS_REG(ENT_INT_COAL_CNT), + HISI_SAS_DEBUGFS_REG(OQ_INT_SRC), + HISI_SAS_DEBUGFS_REG(OQ_INT_SRC_MSK), + HISI_SAS_DEBUGFS_REG(ENT_INT_SRC1), + HISI_SAS_DEBUGFS_REG(ENT_INT_SRC2), + HISI_SAS_DEBUGFS_REG(ENT_INT_SRC3), + HISI_SAS_DEBUGFS_REG(ENT_INT_SRC_MSK1), + HISI_SAS_DEBUGFS_REG(ENT_INT_SRC_MSK2), + HISI_SAS_DEBUGFS_REG(ENT_INT_SRC_MSK3), + HISI_SAS_DEBUGFS_REG(CHNL_PHYUPDOWN_INT_MSK), + HISI_SAS_DEBUGFS_REG(CHNL_ENT_INT_MSK), + HISI_SAS_DEBUGFS_REG(HGC_COM_INT_MSK), + HISI_SAS_DEBUGFS_REG(SAS_ECC_INTR), + HISI_SAS_DEBUGFS_REG(SAS_ECC_INTR_MSK), + HISI_SAS_DEBUGFS_REG(HGC_ERR_STAT_EN), + HISI_SAS_DEBUGFS_REG(CQE_SEND_CNT), + HISI_SAS_DEBUGFS_REG(DLVRY_Q_0_DEPTH), + HISI_SAS_DEBUGFS_REG(DLVRY_Q_0_WR_PTR), + HISI_SAS_DEBUGFS_REG(DLVRY_Q_0_RD_PTR), + HISI_SAS_DEBUGFS_REG(HYPER_STREAM_ID_EN_CFG), + HISI_SAS_DEBUGFS_REG(OQ0_INT_SRC_MSK), + HISI_SAS_DEBUGFS_REG(COMPL_Q_0_DEPTH), + HISI_SAS_DEBUGFS_REG(COMPL_Q_0_WR_PTR), + HISI_SAS_DEBUGFS_REG(COMPL_Q_0_RD_PTR), + HISI_SAS_DEBUGFS_REG(AWQOS_AWCACHE_CFG), + HISI_SAS_DEBUGFS_REG(ARQOS_ARCACHE_CFG), + HISI_SAS_DEBUGFS_REG(HILINK_ERR_DFX), + HISI_SAS_DEBUGFS_REG(SAS_GPIO_CFG_0), + HISI_SAS_DEBUGFS_REG(SAS_GPIO_CFG_1), + HISI_SAS_DEBUGFS_REG(SAS_GPIO_TX_0_1), + HISI_SAS_DEBUGFS_REG(SAS_CFG_DRIVE_VLD), + {} +}; + +static const struct hisi_sas_debugfs_reg debugfs_global_reg = { + .lu = debugfs_global_reg_lu, + .count = 0x800, + .read_global_reg = hisi_sas_read32, +}; + +static void debugfs_snapshot_prepare_v3_hw(struct hisi_hba *hisi_hba) +{ + struct device *dev = hisi_hba->dev; + + set_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags); + + hisi_sas_write32(hisi_hba, DLVRY_QUEUE_ENABLE, 0); + + if (wait_cmds_complete_timeout_v3_hw(hisi_hba, 100, 5000) == -ETIMEDOUT) + dev_dbg(dev, "Wait commands complete timeout!\n"); + + hisi_sas_kill_tasklets(hisi_hba); +} + +static void debugfs_snapshot_restore_v3_hw(struct hisi_hba *hisi_hba) +{ + hisi_sas_write32(hisi_hba, DLVRY_QUEUE_ENABLE, + (u32)((1ULL << hisi_hba->queue_count) - 1)); + + clear_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags); +} + static struct scsi_host_template sht_v3_hw = { .name = DRV_NAME, .module = THIS_MODULE, @@ -2344,6 +2633,7 @@ static struct scsi_host_template sht_v3_hw = { .bios_param = sas_bios_param, .this_id = -1, .sg_tablesize = HISI_SAS_SGE_PAGE_CNT, + .sg_prot_tablesize = HISI_SAS_SGE_PAGE_CNT, .max_sectors = SCSI_DEFAULT_MAX_SECTORS, .eh_device_reset_handler = sas_eh_device_reset_handler, .eh_target_reset_handler = sas_eh_target_reset_handler, @@ -2360,7 +2650,7 @@ static const struct hisi_sas_hw hisi_sas_v3_hw = { .get_wideport_bitmap = get_wideport_bitmap_v3_hw, .complete_hdr_size = sizeof(struct hisi_sas_complete_v3_hdr), .clear_itct = clear_itct_v3_hw, - .sl_notify = sl_notify_v3_hw, + .sl_notify_ssp = sl_notify_ssp_v3_hw, .prep_ssp = prep_ssp_v3_hw, .prep_smp = prep_smp_v3_hw, .prep_stp = prep_ata_v3_hw, @@ -2380,6 +2670,10 @@ static const struct hisi_sas_hw hisi_sas_v3_hw = { .get_events = phy_get_events_v3_hw, .write_gpio = write_gpio_v3_hw, .wait_cmds_complete_timeout = wait_cmds_complete_timeout_v3_hw, + .debugfs_reg_global = &debugfs_global_reg, + .debugfs_reg_port = &debugfs_port_reg, + .snapshot_prepare = debugfs_snapshot_prepare_v3_hw, + .snapshot_restore = debugfs_snapshot_restore_v3_hw, }; static struct Scsi_Host * @@ -2397,6 +2691,7 @@ hisi_sas_shost_alloc_pci(struct pci_dev *pdev) hisi_hba = shost_priv(shost); INIT_WORK(&hisi_hba->rst_work, hisi_sas_rst_work_handler); + INIT_WORK(&hisi_hba->debugfs_work, hisi_sas_debugfs_work_handler); hisi_hba->hw = &hisi_sas_v3_hw; hisi_hba->pci_dev = pdev; hisi_hba->dev = dev; @@ -2414,7 +2709,7 @@ hisi_sas_shost_alloc_pci(struct pci_dev *pdev) if (hisi_sas_get_fw_info(hisi_hba) < 0) goto err_out; - if (hisi_sas_alloc(hisi_hba, shost)) { + if (hisi_sas_alloc(hisi_hba)) { hisi_sas_free(hisi_hba); goto err_out; } @@ -2513,8 +2808,14 @@ hisi_sas_v3_probe(struct pci_dev *pdev, const struct pci_device_id *id) dev_info(dev, "Registering for DIF/DIX prot_mask=0x%x\n", prot_mask); scsi_host_set_prot(hisi_hba->shost, prot_mask); + if (hisi_hba->prot_mask & HISI_SAS_DIX_PROT_MASK) + scsi_host_set_guard(hisi_hba->shost, + SHOST_DIX_GUARD_CRC); } + if (hisi_sas_debugfs_enable) + hisi_sas_debugfs_init(hisi_hba); + rc = scsi_add_host(shost, dev); if (rc) goto err_out_ha; @@ -2551,7 +2852,7 @@ hisi_sas_v3_destroy_irqs(struct pci_dev *pdev, struct hisi_hba *hisi_hba) free_irq(pci_irq_vector(pdev, 1), hisi_hba); free_irq(pci_irq_vector(pdev, 2), hisi_hba); free_irq(pci_irq_vector(pdev, 11), hisi_hba); - for (i = 0; i < hisi_hba->queue_count; i++) { + for (i = 0; i < hisi_hba->cq_nvecs; i++) { struct hisi_sas_cq *cq = &hisi_hba->cq[i]; int nr = hisi_sas_intr_conv ? 16 : 16 + i; @@ -2567,6 +2868,8 @@ static void hisi_sas_v3_remove(struct pci_dev *pdev) struct hisi_hba *hisi_hba = sha->lldd_ha; struct Scsi_Host *shost = sha->core.shost; + hisi_sas_debugfs_exit(hisi_hba); + if (timer_pending(&hisi_hba->timer)) del_timer(&hisi_hba->timer); diff --git a/drivers/scsi/hpsa.c b/drivers/scsi/hpsa.c index ff67ef5d5347..f044e7d10d63 100644 --- a/drivers/scsi/hpsa.c +++ b/drivers/scsi/hpsa.c @@ -251,10 +251,11 @@ static int number_of_controllers; static irqreturn_t do_hpsa_intr_intx(int irq, void *dev_id); static irqreturn_t do_hpsa_intr_msi(int irq, void *dev_id); -static int hpsa_ioctl(struct scsi_device *dev, int cmd, void __user *arg); +static int hpsa_ioctl(struct scsi_device *dev, unsigned int cmd, + void __user *arg); #ifdef CONFIG_COMPAT -static int hpsa_compat_ioctl(struct scsi_device *dev, int cmd, +static int hpsa_compat_ioctl(struct scsi_device *dev, unsigned int cmd, void __user *arg); #endif @@ -1327,7 +1328,7 @@ static int hpsa_scsi_add_entry(struct ctlr_info *h, dev_warn(&h->pdev->dev, "physical device with no LUN=0," " suspect firmware bug or unsupported hardware " "configuration.\n"); - return -1; + return -1; } lun_assigned: @@ -4110,7 +4111,7 @@ static int hpsa_gather_lun_info(struct ctlr_info *h, "maximum logical LUNs (%d) exceeded. " "%d LUNs ignored.\n", HPSA_MAX_LUN, *nlogicals - HPSA_MAX_LUN); - *nlogicals = HPSA_MAX_LUN; + *nlogicals = HPSA_MAX_LUN; } if (*nlogicals + *nphysicals > HPSA_MAX_PHYS_LUN) { dev_warn(&h->pdev->dev, @@ -6127,7 +6128,7 @@ static void cmd_free(struct ctlr_info *h, struct CommandList *c) #ifdef CONFIG_COMPAT -static int hpsa_ioctl32_passthru(struct scsi_device *dev, int cmd, +static int hpsa_ioctl32_passthru(struct scsi_device *dev, unsigned int cmd, void __user *arg) { IOCTL32_Command_struct __user *arg32 = @@ -6164,7 +6165,7 @@ static int hpsa_ioctl32_passthru(struct scsi_device *dev, int cmd, } static int hpsa_ioctl32_big_passthru(struct scsi_device *dev, - int cmd, void __user *arg) + unsigned int cmd, void __user *arg) { BIG_IOCTL32_Command_struct __user *arg32 = (BIG_IOCTL32_Command_struct __user *) arg; @@ -6201,7 +6202,8 @@ static int hpsa_ioctl32_big_passthru(struct scsi_device *dev, return err; } -static int hpsa_compat_ioctl(struct scsi_device *dev, int cmd, void __user *arg) +static int hpsa_compat_ioctl(struct scsi_device *dev, unsigned int cmd, + void __user *arg) { switch (cmd) { case CCISS_GETPCIINFO: @@ -6521,7 +6523,8 @@ static void check_ioctl_unit_attention(struct ctlr_info *h, /* * ioctl */ -static int hpsa_ioctl(struct scsi_device *dev, int cmd, void __user *arg) +static int hpsa_ioctl(struct scsi_device *dev, unsigned int cmd, + void __user *arg) { struct ctlr_info *h; void __user *argp = (void __user *)arg; diff --git a/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c b/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c index cc9cae469c4b..7ca277e28d63 100644 --- a/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c +++ b/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c @@ -3788,11 +3788,6 @@ static int ibmvscsis_write_pending(struct se_cmd *se_cmd) return 0; } -static int ibmvscsis_write_pending_status(struct se_cmd *se_cmd) -{ - return 0; -} - static void ibmvscsis_set_default_node_attrs(struct se_node_acl *nacl) { } @@ -4053,7 +4048,6 @@ static const struct target_core_fabric_ops ibmvscsis_ops = { .release_cmd = ibmvscsis_release_cmd, .sess_get_index = ibmvscsis_sess_get_index, .write_pending = ibmvscsis_write_pending, - .write_pending_status = ibmvscsis_write_pending_status, .set_default_node_attributes = ibmvscsis_set_default_node_attrs, .get_cmd_state = ibmvscsis_get_cmd_state, .queue_data_in = ibmvscsis_queue_data_in, diff --git a/drivers/scsi/ipr.c b/drivers/scsi/ipr.c index d1b4025a4503..6d053e220153 100644 --- a/drivers/scsi/ipr.c +++ b/drivers/scsi/ipr.c @@ -6696,7 +6696,8 @@ err_nodev: * Return value: * 0 on success / other on failure **/ -static int ipr_ioctl(struct scsi_device *sdev, int cmd, void __user *arg) +static int ipr_ioctl(struct scsi_device *sdev, unsigned int cmd, + void __user *arg) { struct ipr_resource_entry *res; diff --git a/drivers/scsi/iscsi_tcp.c b/drivers/scsi/iscsi_tcp.c index cae6368ebb98..ed3debce2819 100644 --- a/drivers/scsi/iscsi_tcp.c +++ b/drivers/scsi/iscsi_tcp.c @@ -518,7 +518,7 @@ static int iscsi_sw_tcp_pdu_init(struct iscsi_task *task, if (!task->sc) iscsi_sw_tcp_send_linear_data_prep(conn, task->data, count); else { - struct scsi_data_buffer *sdb = scsi_out(task->sc); + struct scsi_data_buffer *sdb = &task->sc->sdb; err = iscsi_sw_tcp_send_data_prep(conn, sdb->table.sgl, sdb->table.nents, offset, @@ -952,12 +952,6 @@ static umode_t iscsi_sw_tcp_attr_is_visible(int param_type, int param) return 0; } -static int iscsi_sw_tcp_slave_alloc(struct scsi_device *sdev) -{ - blk_queue_flag_set(QUEUE_FLAG_BIDI, sdev->request_queue); - return 0; -} - static int iscsi_sw_tcp_slave_configure(struct scsi_device *sdev) { struct iscsi_sw_tcp_host *tcp_sw_host = iscsi_host_priv(sdev->host); @@ -985,7 +979,6 @@ static struct scsi_host_template iscsi_sw_tcp_sht = { .eh_device_reset_handler= iscsi_eh_device_reset, .eh_target_reset_handler = iscsi_eh_recover_target, .dma_boundary = PAGE_SIZE - 1, - .slave_alloc = iscsi_sw_tcp_slave_alloc, .slave_configure = iscsi_sw_tcp_slave_configure, .target_alloc = iscsi_target_alloc, .proc_name = "iscsi_tcp", diff --git a/drivers/scsi/libiscsi.c b/drivers/scsi/libiscsi.c index 120fc520f27a..21309d5b456d 100644 --- a/drivers/scsi/libiscsi.c +++ b/drivers/scsi/libiscsi.c @@ -228,32 +228,6 @@ static int iscsi_prep_ecdb_ahs(struct iscsi_task *task) return 0; } -static int iscsi_prep_bidi_ahs(struct iscsi_task *task) -{ - struct scsi_cmnd *sc = task->sc; - struct iscsi_rlength_ahdr *rlen_ahdr; - int rc; - - rlen_ahdr = iscsi_next_hdr(task); - rc = iscsi_add_hdr(task, sizeof(*rlen_ahdr)); - if (rc) - return rc; - - rlen_ahdr->ahslength = - cpu_to_be16(sizeof(rlen_ahdr->read_length) + - sizeof(rlen_ahdr->reserved)); - rlen_ahdr->ahstype = ISCSI_AHSTYPE_RLENGTH; - rlen_ahdr->reserved = 0; - rlen_ahdr->read_length = cpu_to_be32(scsi_in(sc)->length); - - ISCSI_DBG_SESSION(task->conn->session, - "bidi-in rlen_ahdr->read_length(%d) " - "rlen_ahdr->ahslength(%d)\n", - be32_to_cpu(rlen_ahdr->read_length), - be16_to_cpu(rlen_ahdr->ahslength)); - return 0; -} - /** * iscsi_check_tmf_restrictions - check if a task is affected by TMF * @task: iscsi task @@ -392,13 +366,6 @@ static int iscsi_prep_scsi_cmd_pdu(struct iscsi_task *task) memcpy(hdr->cdb, sc->cmnd, cmd_len); task->imm_count = 0; - if (scsi_bidi_cmnd(sc)) { - hdr->flags |= ISCSI_FLAG_CMD_READ; - rc = iscsi_prep_bidi_ahs(task); - if (rc) - return rc; - } - if (scsi_get_prot_op(sc) != SCSI_PROT_NORMAL) task->protected = true; @@ -473,12 +440,10 @@ static int iscsi_prep_scsi_cmd_pdu(struct iscsi_task *task) conn->scsicmd_pdus_cnt++; ISCSI_DBG_SESSION(session, "iscsi prep [%s cid %d sc %p cdb 0x%x " - "itt 0x%x len %d bidi_len %d cmdsn %d win %d]\n", - scsi_bidi_cmnd(sc) ? "bidirectional" : + "itt 0x%x len %d cmdsn %d win %d]\n", sc->sc_data_direction == DMA_TO_DEVICE ? "write" : "read", conn->id, sc, sc->cmnd[0], task->itt, transfer_length, - scsi_bidi_cmnd(sc) ? scsi_in(sc)->length : 0, session->cmdsn, session->max_cmdsn - session->exp_cmdsn + 1); return 0; @@ -647,12 +612,7 @@ static void fail_scsi_task(struct iscsi_task *task, int err) state = ISCSI_TASK_ABRT_TMF; sc->result = err << 16; - if (!scsi_bidi_cmnd(sc)) - scsi_set_resid(sc, scsi_bufflen(sc)); - else { - scsi_out(sc)->resid = scsi_out(sc)->length; - scsi_in(sc)->resid = scsi_in(sc)->length; - } + scsi_set_resid(sc, scsi_bufflen(sc)); /* regular RX path uses back_lock */ spin_lock_bh(&conn->session->back_lock); @@ -907,14 +867,7 @@ invalid_datalen: if (rhdr->flags & (ISCSI_FLAG_CMD_BIDI_UNDERFLOW | ISCSI_FLAG_CMD_BIDI_OVERFLOW)) { - int res_count = be32_to_cpu(rhdr->bi_residual_count); - - if (scsi_bidi_cmnd(sc) && res_count > 0 && - (rhdr->flags & ISCSI_FLAG_CMD_BIDI_OVERFLOW || - res_count <= scsi_in(sc)->length)) - scsi_in(sc)->resid = res_count; - else - sc->result = (DID_BAD_TARGET << 16) | rhdr->cmd_status; + sc->result = (DID_BAD_TARGET << 16) | rhdr->cmd_status; } if (rhdr->flags & (ISCSI_FLAG_CMD_UNDERFLOW | @@ -961,8 +914,8 @@ iscsi_data_in_rsp(struct iscsi_conn *conn, struct iscsi_hdr *hdr, if (res_count > 0 && (rhdr->flags & ISCSI_FLAG_CMD_OVERFLOW || - res_count <= scsi_in(sc)->length)) - scsi_in(sc)->resid = res_count; + res_count <= sc->sdb.length)) + scsi_set_resid(sc, res_count); else sc->result = (DID_BAD_TARGET << 16) | rhdr->cmd_status; } @@ -1810,12 +1763,7 @@ fault: spin_unlock_bh(&session->frwd_lock); ISCSI_DBG_SESSION(session, "iscsi: cmd 0x%x is not queued (%d)\n", sc->cmnd[0], reason); - if (!scsi_bidi_cmnd(sc)) - scsi_set_resid(sc, scsi_bufflen(sc)); - else { - scsi_out(sc)->resid = scsi_out(sc)->length; - scsi_in(sc)->resid = scsi_in(sc)->length; - } + scsi_set_resid(sc, scsi_bufflen(sc)); sc->scsi_done(sc); return 0; } diff --git a/drivers/scsi/libiscsi_tcp.c b/drivers/scsi/libiscsi_tcp.c index 8a6b1b3f8277..9923e9e3b884 100644 --- a/drivers/scsi/libiscsi_tcp.c +++ b/drivers/scsi/libiscsi_tcp.c @@ -495,7 +495,7 @@ static int iscsi_tcp_data_in(struct iscsi_conn *conn, struct iscsi_task *task) struct iscsi_tcp_task *tcp_task = task->dd_data; struct iscsi_data_rsp *rhdr = (struct iscsi_data_rsp *)tcp_conn->in.hdr; int datasn = be32_to_cpu(rhdr->datasn); - unsigned total_in_length = scsi_in(task->sc)->length; + unsigned total_in_length = task->sc->sdb.length; /* * lib iscsi will update this in the completion handling if there @@ -580,11 +580,11 @@ static int iscsi_tcp_r2t_rsp(struct iscsi_conn *conn, struct iscsi_task *task) data_length, session->max_burst); data_offset = be32_to_cpu(rhdr->data_offset); - if (data_offset + data_length > scsi_out(task->sc)->length) { + if (data_offset + data_length > task->sc->sdb.length) { iscsi_conn_printk(KERN_ERR, conn, "invalid R2T with data len %u at offset %u " "and total length %d\n", data_length, - data_offset, scsi_out(task->sc)->length); + data_offset, task->sc->sdb.length); return ISCSI_ERR_DATALEN; } @@ -696,7 +696,7 @@ iscsi_tcp_hdr_dissect(struct iscsi_conn *conn, struct iscsi_hdr *hdr) if (tcp_conn->in.datalen) { struct iscsi_tcp_task *tcp_task = task->dd_data; struct ahash_request *rx_hash = NULL; - struct scsi_data_buffer *sdb = scsi_in(task->sc); + struct scsi_data_buffer *sdb = &task->sc->sdb; /* * Setup copy of Data-In into the struct scsi_cmnd diff --git a/drivers/scsi/libsas/sas_expander.c b/drivers/scsi/libsas/sas_expander.c index f21c93bbb35c..17b45a0c7bc3 100644 --- a/drivers/scsi/libsas/sas_expander.c +++ b/drivers/scsi/libsas/sas_expander.c @@ -25,6 +25,7 @@ #include <linux/scatterlist.h> #include <linux/blkdev.h> #include <linux/slab.h> +#include <asm/unaligned.h> #include "sas_internal.h" @@ -614,7 +615,14 @@ int sas_smp_phy_control(struct domain_device *dev, int phy_id, } res = smp_execute_task(dev, pc_req, PC_REQ_SIZE, pc_resp,PC_RESP_SIZE); - + if (res) { + pr_err("ex %016llx phy%02d PHY control failed: %d\n", + SAS_ADDR(dev->sas_addr), phy_id, res); + } else if (pc_resp[2] != SMP_RESP_FUNC_ACC) { + pr_err("ex %016llx phy%02d PHY control failed: function result 0x%x\n", + SAS_ADDR(dev->sas_addr), phy_id, pc_resp[2]); + res = pc_resp[2]; + } kfree(pc_resp); kfree(pc_req); return res; @@ -689,10 +697,10 @@ int sas_smp_get_phy_events(struct sas_phy *phy) if (res) goto out; - phy->invalid_dword_count = scsi_to_u32(&resp[12]); - phy->running_disparity_error_count = scsi_to_u32(&resp[16]); - phy->loss_of_dword_sync_count = scsi_to_u32(&resp[20]); - phy->phy_reset_problem_count = scsi_to_u32(&resp[24]); + phy->invalid_dword_count = get_unaligned_be32(&resp[12]); + phy->running_disparity_error_count = get_unaligned_be32(&resp[16]); + phy->loss_of_dword_sync_count = get_unaligned_be32(&resp[20]); + phy->phy_reset_problem_count = get_unaligned_be32(&resp[24]); out: kfree(req); @@ -817,6 +825,26 @@ static struct domain_device *sas_ex_discover_end_dev( #ifdef CONFIG_SCSI_SAS_ATA if ((phy->attached_tproto & SAS_PROTOCOL_STP) || phy->attached_sata_dev) { + if (child->linkrate > parent->min_linkrate) { + struct sas_phy_linkrates rates = { + .maximum_linkrate = parent->min_linkrate, + .minimum_linkrate = parent->min_linkrate, + }; + int ret; + + pr_notice("ex %016llx phy%02d SATA device linkrate > min pathway connection rate, attempting to lower device linkrate\n", + SAS_ADDR(child->sas_addr), phy_id); + ret = sas_smp_phy_control(parent, phy_id, + PHY_FUNC_LINK_RESET, &rates); + if (ret) { + pr_err("ex %016llx phy%02d SATA device could not set linkrate (%d)\n", + SAS_ADDR(child->sas_addr), phy_id, ret); + goto out_free; + } + pr_notice("ex %016llx phy%02d SATA device set linkrate successfully\n", + SAS_ADDR(child->sas_addr), phy_id); + child->linkrate = child->min_linkrate; + } res = sas_get_ata_info(child, phy); if (res) goto out_free; diff --git a/drivers/scsi/libsas/sas_scsi_host.c b/drivers/scsi/libsas/sas_scsi_host.c index c43a00a9d819..b775445892af 100644 --- a/drivers/scsi/libsas/sas_scsi_host.c +++ b/drivers/scsi/libsas/sas_scsi_host.c @@ -799,7 +799,7 @@ out: shost->host_failed, tries); } -int sas_ioctl(struct scsi_device *sdev, int cmd, void __user *arg) +int sas_ioctl(struct scsi_device *sdev, unsigned int cmd, void __user *arg) { struct domain_device *dev = sdev_to_domain_dev(sdev); diff --git a/drivers/scsi/lpfc/lpfc.h b/drivers/scsi/lpfc/lpfc.h index ebdfe5b26937..41d849f283f6 100644 --- a/drivers/scsi/lpfc/lpfc.h +++ b/drivers/scsi/lpfc/lpfc.h @@ -1,7 +1,7 @@ /******************************************************************* * This file is part of the Emulex Linux Device Driver for * * Fibre Channel Host Bus Adapters. * - * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term * + * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term * * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * * Copyright (C) 2004-2016 Emulex. All rights reserved. * * EMULEX and SLI are trademarks of Emulex. * @@ -84,8 +84,6 @@ struct lpfc_sli2_slim; #define LPFC_HB_MBOX_INTERVAL 5 /* Heart beat interval in seconds. */ #define LPFC_HB_MBOX_TIMEOUT 30 /* Heart beat timeout in seconds. */ -#define LPFC_LOOK_AHEAD_OFF 0 /* Look ahead logic is turned off */ - /* Error Attention event polling interval */ #define LPFC_ERATT_POLL_INTERVAL 5 /* EATT poll interval in seconds */ @@ -146,6 +144,7 @@ struct lpfc_nvmet_ctxbuf { struct lpfc_nvmet_rcv_ctx *context; struct lpfc_iocbq *iocbq; struct lpfc_sglq *sglq; + struct work_struct defer_work; }; struct lpfc_dma_pool { @@ -235,8 +234,6 @@ typedef struct lpfc_vpd { } sli3Feat; } lpfc_vpd_t; -struct lpfc_scsi_buf; - /* * lpfc stat counters @@ -466,6 +463,7 @@ struct lpfc_vport { uint32_t cfg_use_adisc; uint32_t cfg_discovery_threads; uint32_t cfg_log_verbose; + uint32_t cfg_enable_fc4_type; uint32_t cfg_max_luns; uint32_t cfg_enable_da_id; uint32_t cfg_max_scsicmpl_time; @@ -479,6 +477,7 @@ struct lpfc_vport { struct dentry *debug_disc_trc; struct dentry *debug_nodelist; struct dentry *debug_nvmestat; + struct dentry *debug_scsistat; struct dentry *debug_nvmektime; struct dentry *debug_cpucheck; struct dentry *vport_debugfs_root; @@ -596,6 +595,13 @@ struct lpfc_mbox_ext_buf_ctx { struct list_head ext_dmabuf_list; }; +struct lpfc_epd_pool { + /* Expedite pool */ + struct list_head list; + u32 count; + spinlock_t lock; /* lock for expedite pool */ +}; + struct lpfc_ras_fwlog { uint8_t *fwlog_buff; uint32_t fw_buffcount; /* Buffer size posted to FW */ @@ -617,20 +623,19 @@ struct lpfc_ras_fwlog { struct lpfc_hba { /* SCSI interface function jump table entries */ - int (*lpfc_new_scsi_buf) - (struct lpfc_vport *, int); - struct lpfc_scsi_buf * (*lpfc_get_scsi_buf) - (struct lpfc_hba *, struct lpfc_nodelist *); + struct lpfc_io_buf * (*lpfc_get_scsi_buf) + (struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, + struct scsi_cmnd *cmnd); int (*lpfc_scsi_prep_dma_buf) - (struct lpfc_hba *, struct lpfc_scsi_buf *); + (struct lpfc_hba *, struct lpfc_io_buf *); void (*lpfc_scsi_unprep_dma_buf) - (struct lpfc_hba *, struct lpfc_scsi_buf *); + (struct lpfc_hba *, struct lpfc_io_buf *); void (*lpfc_release_scsi_buf) - (struct lpfc_hba *, struct lpfc_scsi_buf *); + (struct lpfc_hba *, struct lpfc_io_buf *); void (*lpfc_rampdown_queue_depth) (struct lpfc_hba *); void (*lpfc_scsi_prep_cmnd) - (struct lpfc_vport *, struct lpfc_scsi_buf *, + (struct lpfc_vport *, struct lpfc_io_buf *, struct lpfc_nodelist *); /* IOCB interface function jump table entries */ @@ -673,13 +678,17 @@ struct lpfc_hba { (struct lpfc_hba *); int (*lpfc_bg_scsi_prep_dma_buf) - (struct lpfc_hba *, struct lpfc_scsi_buf *); + (struct lpfc_hba *, struct lpfc_io_buf *); /* Add new entries here */ + /* expedite pool */ + struct lpfc_epd_pool epd_pool; + /* SLI4 specific HBA data structure */ struct lpfc_sli4_hba sli4_hba; struct workqueue_struct *wq; + struct delayed_work eq_delay_work; struct lpfc_sli sli; uint8_t pci_dev_grp; /* lpfc PCI dev group: 0x0, 0x1, 0x2,... */ @@ -713,7 +722,6 @@ struct lpfc_hba { #define HBA_FCOE_MODE 0x4 /* HBA function in FCoE Mode */ #define HBA_SP_QUEUE_EVT 0x8 /* Slow-path qevt posted to worker thread*/ #define HBA_POST_RECEIVE_BUFFER 0x10 /* Rcv buffers need to be posted */ -#define FCP_XRI_ABORT_EVENT 0x20 #define ELS_XRI_ABORT_EVENT 0x40 #define ASYNC_EVENT 0x80 #define LINK_DISABLED 0x100 /* Link disabled by user */ @@ -784,12 +792,12 @@ struct lpfc_hba { uint8_t nvmet_support; /* driver supports NVMET */ #define LPFC_NVMET_MAX_PORTS 32 uint8_t mds_diags_support; - uint32_t initial_imax; uint8_t bbcredit_support; uint8_t enab_exp_wqcq_pages; /* HBA Config Parameters */ uint32_t cfg_ack0; + uint32_t cfg_xri_rebalancing; uint32_t cfg_enable_npiv; uint32_t cfg_enable_rrq; uint32_t cfg_topology; @@ -811,12 +819,14 @@ struct lpfc_hba { uint32_t cfg_use_msi; uint32_t cfg_auto_imax; uint32_t cfg_fcp_imax; + uint32_t cfg_cq_poll_threshold; + uint32_t cfg_cq_max_proc_limit; uint32_t cfg_fcp_cpu_map; - uint32_t cfg_fcp_io_channel; + uint32_t cfg_hdw_queue; + uint32_t cfg_irq_chann; uint32_t cfg_suppress_rsp; uint32_t cfg_nvme_oas; uint32_t cfg_nvme_embed_cmd; - uint32_t cfg_nvme_io_channel; uint32_t cfg_nvmet_mrq_post; uint32_t cfg_nvmet_mrq; uint32_t cfg_enable_nvmet; @@ -852,6 +862,7 @@ struct lpfc_hba { uint32_t cfg_prot_guard; uint32_t cfg_hostmem_hgp; uint32_t cfg_log_verbose; + uint32_t cfg_enable_fc4_type; uint32_t cfg_aer_support; uint32_t cfg_sriov_nr_virtfn; uint32_t cfg_request_firmware_upgrade; @@ -872,15 +883,12 @@ struct lpfc_hba { uint32_t cfg_ras_fwlog_level; uint32_t cfg_ras_fwlog_buffsize; uint32_t cfg_ras_fwlog_func; - uint32_t cfg_enable_fc4_type; uint32_t cfg_enable_bbcr; /* Enable BB Credit Recovery */ uint32_t cfg_enable_dpp; /* Enable Direct Packet Push */ - uint32_t cfg_xri_split; #define LPFC_ENABLE_FCP 1 #define LPFC_ENABLE_NVME 2 #define LPFC_ENABLE_BOTH 3 uint32_t cfg_enable_pbde; - uint32_t io_channel_irqs; /* number of irqs for io channels */ struct nvmet_fc_target_port *targetport; lpfc_vpd_t vpd; /* vital product data */ @@ -952,14 +960,6 @@ struct lpfc_hba { struct timer_list eratt_poll; uint32_t eratt_poll_interval; - /* - * stat counters - */ - atomic_t fc4ScsiInputRequests; - atomic_t fc4ScsiOutputRequests; - atomic_t fc4ScsiControlRequests; - atomic_t fc4ScsiIoCmpls; - uint64_t bg_guard_err_cnt; uint64_t bg_apptag_err_cnt; uint64_t bg_reftag_err_cnt; @@ -970,13 +970,6 @@ struct lpfc_hba { struct list_head lpfc_scsi_buf_list_get; struct list_head lpfc_scsi_buf_list_put; uint32_t total_scsi_bufs; - spinlock_t nvme_buf_list_get_lock; /* NVME buf alloc list lock */ - spinlock_t nvme_buf_list_put_lock; /* NVME buf free list lock */ - struct list_head lpfc_nvme_buf_list_get; - struct list_head lpfc_nvme_buf_list_put; - uint32_t total_nvme_bufs; - uint32_t get_nvme_bufs; - uint32_t put_nvme_bufs; struct list_head lpfc_iocb_list; uint32_t total_iocbq_bufs; struct list_head active_rrq_list; @@ -1033,6 +1026,7 @@ struct lpfc_hba { #ifdef CONFIG_SCSI_LPFC_DEBUG_FS struct dentry *hba_debugfs_root; atomic_t debugfs_vport_count; + struct dentry *debug_multixri_pools; struct dentry *debug_hbqinfo; struct dentry *debug_dumpHostSlim; struct dentry *debug_dumpHBASlim; @@ -1050,6 +1044,10 @@ struct lpfc_hba { struct dentry *debug_nvmeio_trc; struct lpfc_debugfs_nvmeio_trc *nvmeio_trc; + struct dentry *debug_hdwqinfo; +#ifdef LPFC_HDWQ_LOCK_STAT + struct dentry *debug_lockstat; +#endif atomic_t nvmeio_trc_cnt; uint32_t nvmeio_trc_size; uint32_t nvmeio_trc_output_idx; @@ -1090,7 +1088,6 @@ struct lpfc_hba { uint8_t temp_sensor_support; /* Fields used for heart beat. */ - unsigned long last_eqdelay_time; unsigned long last_completion_time; unsigned long skipped_hb; struct timer_list hb_tmofunc; @@ -1164,16 +1161,12 @@ struct lpfc_hba { uint16_t sfp_warning; #ifdef CONFIG_SCSI_LPFC_DEBUG_FS -#define LPFC_CHECK_CPU_CNT 32 - uint32_t cpucheck_rcv_io[LPFC_CHECK_CPU_CNT]; - uint32_t cpucheck_xmt_io[LPFC_CHECK_CPU_CNT]; - uint32_t cpucheck_cmpl_io[LPFC_CHECK_CPU_CNT]; - uint32_t cpucheck_ccmpl_io[LPFC_CHECK_CPU_CNT]; uint16_t cpucheck_on; #define LPFC_CHECK_OFF 0 #define LPFC_CHECK_NVME_IO 1 #define LPFC_CHECK_NVMET_RCV 2 #define LPFC_CHECK_NVMET_IO 4 +#define LPFC_CHECK_SCSI_IO 8 uint16_t ktime_on; uint64_t ktime_data_samples; uint64_t ktime_status_samples; @@ -1297,3 +1290,23 @@ lpfc_phba_elsring(struct lpfc_hba *phba) } return &phba->sli.sli3_ring[LPFC_ELS_RING]; } + +/** + * lpfc_sli4_mod_hba_eq_delay - update EQ delay + * @phba: Pointer to HBA context object. + * @q: The Event Queue to update. + * @delay: The delay value (in us) to be written. + * + **/ +static inline void +lpfc_sli4_mod_hba_eq_delay(struct lpfc_hba *phba, struct lpfc_queue *eq, + u32 delay) +{ + struct lpfc_register reg_data; + + reg_data.word0 = 0; + bf_set(lpfc_sliport_eqdelay_id, ®_data, eq->queue_id); + bf_set(lpfc_sliport_eqdelay_delay, ®_data, delay); + writel(reg_data.word0, phba->sli4_hba.u.if_type2.EQDregaddr); + eq->q_mode = delay; +} diff --git a/drivers/scsi/lpfc/lpfc_attr.c b/drivers/scsi/lpfc/lpfc_attr.c index 4bae72cbf3f6..ce3e541434dc 100644 --- a/drivers/scsi/lpfc/lpfc_attr.c +++ b/drivers/scsi/lpfc/lpfc_attr.c @@ -1,7 +1,7 @@ /******************************************************************* * This file is part of the Emulex Linux Device Driver for * * Fibre Channel Host Bus Adapters. * - * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term * + * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term * * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * * Copyright (C) 2004-2016 Emulex. All rights reserved. * * EMULEX and SLI are trademarks of Emulex. * @@ -64,9 +64,6 @@ #define LPFC_MIN_MRQ_POST 512 #define LPFC_MAX_MRQ_POST 2048 -#define LPFC_MAX_NVME_INFO_TMP_LEN 100 -#define LPFC_NVME_INFO_MORE_STR "\nCould be more info...\n" - /* * Write key size should be multiple of 4. If write key is changed * make sure that library write key is also changed. @@ -155,7 +152,7 @@ lpfc_nvme_info_show(struct device *dev, struct device_attribute *attr, struct lpfc_nvme_rport *rport; struct lpfc_nodelist *ndlp; struct nvme_fc_remote_port *nrport; - struct lpfc_nvme_ctrl_stat *cstat; + struct lpfc_fc4_ctrl_stat *cstat; uint64_t data1, data2, data3; uint64_t totin, totout, tot; char *statep; @@ -163,7 +160,7 @@ lpfc_nvme_info_show(struct device *dev, struct device_attribute *attr, int len = 0; char tmp[LPFC_MAX_NVME_INFO_TMP_LEN] = {0}; - if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME)) { + if (!(vport->cfg_enable_fc4_type & LPFC_ENABLE_NVME)) { len = scnprintf(buf, PAGE_SIZE, "NVME Disabled\n"); return len; } @@ -334,11 +331,10 @@ lpfc_nvme_info_show(struct device *dev, struct device_attribute *attr, rcu_read_lock(); scnprintf(tmp, sizeof(tmp), - "XRI Dist lpfc%d Total %d NVME %d SCSI %d ELS %d\n", + "XRI Dist lpfc%d Total %d IO %d ELS %d\n", phba->brd_no, phba->sli4_hba.max_cfg_param.max_xri, - phba->sli4_hba.nvme_xri_max, - phba->sli4_hba.scsi_xri_max, + phba->sli4_hba.io_xri_max, lpfc_sli4_get_els_iocb_cnt(phba)); if (strlcat(buf, tmp, PAGE_SIZE) >= PAGE_SIZE) goto buffer_done; @@ -457,13 +453,13 @@ lpfc_nvme_info_show(struct device *dev, struct device_attribute *attr, totin = 0; totout = 0; - for (i = 0; i < phba->cfg_nvme_io_channel; i++) { - cstat = &lport->cstat[i]; - tot = atomic_read(&cstat->fc4NvmeIoCmpls); + for (i = 0; i < phba->cfg_hdw_queue; i++) { + cstat = &phba->sli4_hba.hdwq[i].nvme_cstat; + tot = cstat->io_cmpls; totin += tot; - data1 = atomic_read(&cstat->fc4NvmeInputRequests); - data2 = atomic_read(&cstat->fc4NvmeOutputRequests); - data3 = atomic_read(&cstat->fc4NvmeControlRequests); + data1 = cstat->input_requests; + data2 = cstat->output_requests; + data3 = cstat->control_requests; totout += (data1 + data2 + data3); } scnprintf(tmp, sizeof(tmp), @@ -510,6 +506,57 @@ buffer_done: } static ssize_t +lpfc_scsi_stat_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct Scsi_Host *shost = class_to_shost(dev); + struct lpfc_vport *vport = shost_priv(shost); + struct lpfc_hba *phba = vport->phba; + int len; + struct lpfc_fc4_ctrl_stat *cstat; + u64 data1, data2, data3; + u64 tot, totin, totout; + int i; + char tmp[LPFC_MAX_SCSI_INFO_TMP_LEN] = {0}; + + if (!(vport->cfg_enable_fc4_type & LPFC_ENABLE_FCP) || + (phba->sli_rev != LPFC_SLI_REV4)) + return 0; + + scnprintf(buf, PAGE_SIZE, "SCSI HDWQ Statistics\n"); + + totin = 0; + totout = 0; + for (i = 0; i < phba->cfg_hdw_queue; i++) { + cstat = &phba->sli4_hba.hdwq[i].scsi_cstat; + tot = cstat->io_cmpls; + totin += tot; + data1 = cstat->input_requests; + data2 = cstat->output_requests; + data3 = cstat->control_requests; + totout += (data1 + data2 + data3); + + scnprintf(tmp, sizeof(tmp), "HDWQ (%d): Rd %016llx Wr %016llx " + "IO %016llx ", i, data1, data2, data3); + if (strlcat(buf, tmp, PAGE_SIZE) >= PAGE_SIZE) + goto buffer_done; + + scnprintf(tmp, sizeof(tmp), "Cmpl %016llx OutIO %016llx\n", + tot, ((data1 + data2 + data3) - tot)); + if (strlcat(buf, tmp, PAGE_SIZE) >= PAGE_SIZE) + goto buffer_done; + } + scnprintf(tmp, sizeof(tmp), "Total FCP Cmpl %016llx Issue %016llx " + "OutIO %016llx\n", totin, totout, totout - totin); + strlcat(buf, tmp, PAGE_SIZE); + +buffer_done: + len = strnlen(buf, PAGE_SIZE); + + return len; +} + +static ssize_t lpfc_bg_info_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -2574,6 +2621,7 @@ lpfc_##attr##_store(struct device *dev, struct device_attribute *attr, \ static DEVICE_ATTR(nvme_info, 0444, lpfc_nvme_info_show, NULL); +static DEVICE_ATTR(scsi_stat, 0444, lpfc_scsi_stat_show, NULL); static DEVICE_ATTR(bg_info, S_IRUGO, lpfc_bg_info_show, NULL); static DEVICE_ATTR(bg_guard_err, S_IRUGO, lpfc_bg_guard_err_show, NULL); static DEVICE_ATTR(bg_apptag_err, S_IRUGO, lpfc_bg_apptag_err_show, NULL); @@ -3724,29 +3772,13 @@ LPFC_ATTR_R(nvmet_mrq_post, * lpfc_enable_fc4_type: Defines what FC4 types are supported. * Supported Values: 1 - register just FCP * 3 - register both FCP and NVME - * Supported values are [1,3]. Default value is 1 + * Supported values are [1,3]. Default value is 3 */ -LPFC_ATTR_R(enable_fc4_type, LPFC_ENABLE_FCP, +LPFC_ATTR_R(enable_fc4_type, LPFC_ENABLE_BOTH, LPFC_ENABLE_FCP, LPFC_ENABLE_BOTH, "Enable FC4 Protocol support - FCP / NVME"); /* - * lpfc_xri_split: Defines the division of XRI resources between SCSI and NVME - * This parameter is only used if: - * lpfc_enable_fc4_type is 3 - register both FCP and NVME and - * port is not configured for NVMET. - * - * ELS/CT always get 10% of XRIs, up to a maximum of 250 - * The remaining XRIs get split up based on lpfc_xri_split per port: - * - * Supported Values are in percentages - * the xri_split value is the percentage the SCSI port will get. The remaining - * percentage will go to NVME. - */ -LPFC_ATTR_R(xri_split, 50, 10, 90, - "Percentage of FCP XRI resources versus NVME"); - -/* # lpfc_log_verbose: Only turn this flag on if you are willing to risk being # deluged with LOTS of information. # You can set a bit mask to record specific types of verbose messages: @@ -4903,6 +4935,8 @@ lpfc_fcp_imax_store(struct device *dev, struct device_attribute *attr, struct Scsi_Host *shost = class_to_shost(dev); struct lpfc_vport *vport = (struct lpfc_vport *)shost->hostdata; struct lpfc_hba *phba = vport->phba; + struct lpfc_eq_intr_info *eqi; + uint32_t usdelay; int val = 0, i; /* fcp_imax is only valid for SLI4 */ @@ -4923,12 +4957,27 @@ lpfc_fcp_imax_store(struct device *dev, struct device_attribute *attr, if (val && (val < LPFC_MIN_IMAX || val > LPFC_MAX_IMAX)) return -EINVAL; + phba->cfg_auto_imax = (val) ? 0 : 1; + if (phba->cfg_fcp_imax && !val) { + queue_delayed_work(phba->wq, &phba->eq_delay_work, + msecs_to_jiffies(LPFC_EQ_DELAY_MSECS)); + + for_each_present_cpu(i) { + eqi = per_cpu_ptr(phba->sli4_hba.eq_info, i); + eqi->icnt = 0; + } + } + phba->cfg_fcp_imax = (uint32_t)val; - phba->initial_imax = phba->cfg_fcp_imax; - for (i = 0; i < phba->io_channel_irqs; i += LPFC_MAX_EQ_DELAY_EQID_CNT) + if (phba->cfg_fcp_imax) + usdelay = LPFC_SEC_TO_USEC / phba->cfg_fcp_imax; + else + usdelay = 0; + + for (i = 0; i < phba->cfg_irq_chann; i += LPFC_MAX_EQ_DELAY_EQID_CNT) lpfc_modify_hba_eq_delay(phba, i, LPFC_MAX_EQ_DELAY_EQID_CNT, - val); + usdelay); return strlen(buf); } @@ -4982,15 +5031,119 @@ lpfc_fcp_imax_init(struct lpfc_hba *phba, int val) static DEVICE_ATTR_RW(lpfc_fcp_imax); +/** + * lpfc_cq_max_proc_limit_store + * + * @dev: class device that is converted into a Scsi_host. + * @attr: device attribute, not used. + * @buf: string with the cq max processing limit of cqes + * @count: unused variable. + * + * Description: + * If val is in a valid range, then set value on each cq + * + * Returns: + * The length of the buf: if successful + * -ERANGE: if val is not in the valid range + * -EINVAL: if bad value format or intended mode is not supported. + **/ +static ssize_t +lpfc_cq_max_proc_limit_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct Scsi_Host *shost = class_to_shost(dev); + struct lpfc_vport *vport = (struct lpfc_vport *)shost->hostdata; + struct lpfc_hba *phba = vport->phba; + struct lpfc_queue *eq, *cq; + unsigned long val; + int i; + + /* cq_max_proc_limit is only valid for SLI4 */ + if (phba->sli_rev != LPFC_SLI_REV4) + return -EINVAL; + + /* Sanity check on user data */ + if (!isdigit(buf[0])) + return -EINVAL; + if (kstrtoul(buf, 0, &val)) + return -EINVAL; + + if (val < LPFC_CQ_MIN_PROC_LIMIT || val > LPFC_CQ_MAX_PROC_LIMIT) + return -ERANGE; + + phba->cfg_cq_max_proc_limit = (uint32_t)val; + + /* set the values on the cq's */ + for (i = 0; i < phba->cfg_irq_chann; i++) { + eq = phba->sli4_hba.hdwq[i].hba_eq; + if (!eq) + continue; + + list_for_each_entry(cq, &eq->child_list, list) + cq->max_proc_limit = min(phba->cfg_cq_max_proc_limit, + cq->entry_count); + } + + return strlen(buf); +} + /* - * lpfc_auto_imax: Controls Auto-interrupt coalescing values support. - * 0 No auto_imax support - * 1 auto imax on - * Auto imax will change the value of fcp_imax on a per EQ basis, using - * the EQ Delay Multiplier, depending on the activity for that EQ. - * Value range [0,1]. Default value is 1. + * lpfc_cq_max_proc_limit: The maximum number CQE entries processed in an + * itteration of CQ processing. */ -LPFC_ATTR_RW(auto_imax, 1, 0, 1, "Enable Auto imax"); +static int lpfc_cq_max_proc_limit = LPFC_CQ_DEF_MAX_PROC_LIMIT; +module_param(lpfc_cq_max_proc_limit, int, 0644); +MODULE_PARM_DESC(lpfc_cq_max_proc_limit, + "Set the maximum number CQEs processed in an iteration of " + "CQ processing"); +lpfc_param_show(cq_max_proc_limit) + +/* + * lpfc_cq_poll_threshold: Set the threshold of CQE completions in a + * single handler call which should request a polled completion rather + * than re-enabling interrupts. + */ +LPFC_ATTR_RW(cq_poll_threshold, LPFC_CQ_DEF_THRESHOLD_TO_POLL, + LPFC_CQ_MIN_THRESHOLD_TO_POLL, + LPFC_CQ_MAX_THRESHOLD_TO_POLL, + "CQE Processing Threshold to enable Polling"); + +/** + * lpfc_cq_max_proc_limit_init - Set the initial cq max_proc_limit + * @phba: lpfc_hba pointer. + * @val: entry limit + * + * Description: + * If val is in a valid range, then initialize the adapter's maximum + * value. + * + * Returns: + * Always returns 0 for success, even if value not always set to + * requested value. If value out of range or not supported, will fall + * back to default. + **/ +static int +lpfc_cq_max_proc_limit_init(struct lpfc_hba *phba, int val) +{ + phba->cfg_cq_max_proc_limit = LPFC_CQ_DEF_MAX_PROC_LIMIT; + + if (phba->sli_rev != LPFC_SLI_REV4) + return 0; + + if (val >= LPFC_CQ_MIN_PROC_LIMIT && val <= LPFC_CQ_MAX_PROC_LIMIT) { + phba->cfg_cq_max_proc_limit = val; + return 0; + } + + lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + "0371 "LPFC_DRIVER_NAME"_cq_max_proc_limit: " + "%d out of range, using default\n", + phba->cfg_cq_max_proc_limit); + + return 0; +} + +static DEVICE_ATTR_RW(lpfc_cq_max_proc_limit); /** * lpfc_state_show - Display current driver CPU affinity @@ -5023,50 +5176,70 @@ lpfc_fcp_cpu_map_show(struct device *dev, struct device_attribute *attr, case 1: len += snprintf(buf + len, PAGE_SIZE-len, "fcp_cpu_map: HBA centric mapping (%d): " - "%d online CPUs\n", - phba->cfg_fcp_cpu_map, - phba->sli4_hba.num_online_cpu); - break; - case 2: - len += snprintf(buf + len, PAGE_SIZE-len, - "fcp_cpu_map: Driver centric mapping (%d): " - "%d online CPUs\n", - phba->cfg_fcp_cpu_map, - phba->sli4_hba.num_online_cpu); + "%d of %d CPUs online from %d possible CPUs\n", + phba->cfg_fcp_cpu_map, num_online_cpus(), + num_present_cpus(), + phba->sli4_hba.num_possible_cpu); break; } - while (phba->sli4_hba.curr_disp_cpu < phba->sli4_hba.num_present_cpu) { + while (phba->sli4_hba.curr_disp_cpu < + phba->sli4_hba.num_possible_cpu) { cpup = &phba->sli4_hba.cpu_map[phba->sli4_hba.curr_disp_cpu]; - /* margin should fit in this and the truncated message */ - if (cpup->irq == LPFC_VECTOR_MAP_EMPTY) - len += snprintf(buf + len, PAGE_SIZE-len, - "CPU %02d io_chan %02d " - "physid %d coreid %d\n", + if (!cpu_present(phba->sli4_hba.curr_disp_cpu)) + len += snprintf(buf + len, PAGE_SIZE - len, + "CPU %02d not present\n", + phba->sli4_hba.curr_disp_cpu); + else if (cpup->irq == LPFC_VECTOR_MAP_EMPTY) { + if (cpup->hdwq == LPFC_VECTOR_MAP_EMPTY) + len += snprintf( + buf + len, PAGE_SIZE - len, + "CPU %02d hdwq None " + "physid %d coreid %d ht %d\n", phba->sli4_hba.curr_disp_cpu, - cpup->channel_id, cpup->phys_id, - cpup->core_id); - else - len += snprintf(buf + len, PAGE_SIZE-len, - "CPU %02d io_chan %02d " - "physid %d coreid %d IRQ %d\n", + cpup->phys_id, + cpup->core_id, cpup->hyper); + else + len += snprintf( + buf + len, PAGE_SIZE - len, + "CPU %02d EQ %04d hdwq %04d " + "physid %d coreid %d ht %d\n", + phba->sli4_hba.curr_disp_cpu, + cpup->eq, cpup->hdwq, cpup->phys_id, + cpup->core_id, cpup->hyper); + } else { + if (cpup->hdwq == LPFC_VECTOR_MAP_EMPTY) + len += snprintf( + buf + len, PAGE_SIZE - len, + "CPU %02d hdwq None " + "physid %d coreid %d ht %d IRQ %d\n", + phba->sli4_hba.curr_disp_cpu, + cpup->phys_id, + cpup->core_id, cpup->hyper, cpup->irq); + else + len += snprintf( + buf + len, PAGE_SIZE - len, + "CPU %02d EQ %04d hdwq %04d " + "physid %d coreid %d ht %d IRQ %d\n", phba->sli4_hba.curr_disp_cpu, - cpup->channel_id, cpup->phys_id, - cpup->core_id, cpup->irq); + cpup->eq, cpup->hdwq, cpup->phys_id, + cpup->core_id, cpup->hyper, cpup->irq); + } phba->sli4_hba.curr_disp_cpu++; /* display max number of CPUs keeping some margin */ if (phba->sli4_hba.curr_disp_cpu < - phba->sli4_hba.num_present_cpu && + phba->sli4_hba.num_possible_cpu && (len >= (PAGE_SIZE - 64))) { - len += snprintf(buf + len, PAGE_SIZE-len, "more...\n"); + len += snprintf(buf + len, + PAGE_SIZE - len, "more...\n"); break; } } - if (phba->sli4_hba.curr_disp_cpu == phba->sli4_hba.num_present_cpu) + if (phba->sli4_hba.curr_disp_cpu == phba->sli4_hba.num_possible_cpu) phba->sli4_hba.curr_disp_cpu = 0; return len; @@ -5094,14 +5267,13 @@ lpfc_fcp_cpu_map_store(struct device *dev, struct device_attribute *attr, # lpfc_fcp_cpu_map: Defines how to map CPUs to IRQ vectors # for the HBA. # -# Value range is [0 to 2]. Default value is LPFC_DRIVER_CPU_MAP (2). +# Value range is [0 to 1]. Default value is LPFC_HBA_CPU_MAP (1). # 0 - Do not affinitze IRQ vectors # 1 - Affintize HBA vectors with respect to each HBA # (start with CPU0 for each HBA) -# 2 - Affintize HBA vectors with respect to the entire driver -# (round robin thru all CPUs across all HBAs) +# This also defines how Hardware Queues are mapped to specific CPUs. */ -static int lpfc_fcp_cpu_map = LPFC_DRIVER_CPU_MAP; +static int lpfc_fcp_cpu_map = LPFC_HBA_CPU_MAP; module_param(lpfc_fcp_cpu_map, int, S_IRUGO|S_IWUSR); MODULE_PARM_DESC(lpfc_fcp_cpu_map, "Defines how to map CPUs to IRQ vectors per HBA"); @@ -5135,7 +5307,7 @@ lpfc_fcp_cpu_map_init(struct lpfc_hba *phba, int val) lpfc_printf_log(phba, KERN_ERR, LOG_INIT, "3326 lpfc_fcp_cpu_map: %d out of range, using " "default\n", val); - phba->cfg_fcp_cpu_map = LPFC_DRIVER_CPU_MAP; + phba->cfg_fcp_cpu_map = LPFC_HBA_CPU_MAP; return 0; } @@ -5235,13 +5407,20 @@ static DEVICE_ATTR_RW(lpfc_max_scsicmpl_time); LPFC_ATTR_R(ack0, 0, 0, 1, "Enable ACK0 support"); /* +# lpfc_xri_rebalancing: enable or disable XRI rebalancing feature +# range is [0,1]. Default value is 1. +*/ +LPFC_ATTR_R(xri_rebalancing, 1, 0, 1, "Enable/Disable XRI rebalancing"); + +/* * lpfc_io_sched: Determine scheduling algrithmn for issuing FCP cmds * range is [0,1]. Default value is 0. - * For [0], FCP commands are issued to Work Queues ina round robin fashion. + * For [0], FCP commands are issued to Work Queues based on upper layer + * hardware queue index. * For [1], FCP commands are issued to a Work Queue associated with the * current CPU. * - * LPFC_FCP_SCHED_ROUND_ROBIN == 0 + * LPFC_FCP_SCHED_BY_HDWQ == 0 * LPFC_FCP_SCHED_BY_CPU == 1 * * The driver dynamically sets this to 1 (BY_CPU) if it's able to set up cpu @@ -5249,11 +5428,11 @@ LPFC_ATTR_R(ack0, 0, 0, 1, "Enable ACK0 support"); * CPU. Otherwise, the default 0 (Round Robin) scheduling of FCP/NVME I/Os * through WQs will be used. */ -LPFC_ATTR_RW(fcp_io_sched, LPFC_FCP_SCHED_ROUND_ROBIN, - LPFC_FCP_SCHED_ROUND_ROBIN, +LPFC_ATTR_RW(fcp_io_sched, LPFC_FCP_SCHED_BY_CPU, + LPFC_FCP_SCHED_BY_HDWQ, LPFC_FCP_SCHED_BY_CPU, "Determine scheduling algorithm for " - "issuing commands [0] - Round Robin, [1] - Current CPU"); + "issuing commands [0] - Hardware Queue, [1] - Current CPU"); /* * lpfc_ns_query: Determine algrithmn for NameServer queries after RSCN @@ -5415,41 +5594,39 @@ LPFC_ATTR_RW(nvme_embed_cmd, 1, 0, 2, "Embed NVME Command in WQE"); /* - * lpfc_fcp_io_channel: Set the number of FCP IO channels the driver - * will advertise it supports to the SCSI layer. This also will map to - * the number of WQs the driver will create. + * lpfc_hdw_queue: Set the number of Hardware Queues the driver + * will advertise it supports to the NVME and SCSI layers. This also + * will map to the number of CQ/WQ pairs the driver will create. * - * 0 = Configure the number of io channels to the number of active CPUs. - * 1,32 = Manually specify how many io channels to use. + * The NVME Layer will try to create this many, plus 1 administrative + * hardware queue. The administrative queue will always map to WQ 0 + * A hardware IO queue maps (qidx) to a specific driver CQ/WQ. * - * Value range is [0,32]. Default value is 4. + * 0 = Configure the number of hdw queues to the number of active CPUs. + * 1,128 = Manually specify how many hdw queues to use. + * + * Value range is [0,128]. Default value is 0. */ -LPFC_ATTR_R(fcp_io_channel, - LPFC_FCP_IO_CHAN_DEF, - LPFC_HBA_IO_CHAN_MIN, LPFC_HBA_IO_CHAN_MAX, - "Set the number of FCP I/O channels"); +LPFC_ATTR_R(hdw_queue, + LPFC_HBA_HDWQ_DEF, + LPFC_HBA_HDWQ_MIN, LPFC_HBA_HDWQ_MAX, + "Set the number of I/O Hardware Queues"); /* - * lpfc_nvme_io_channel: Set the number of IO hardware queues the driver - * will advertise it supports to the NVME layer. This also will map to - * the number of WQs the driver will create. - * - * This module parameter is valid when lpfc_enable_fc4_type is set - * to support NVME. - * - * The NVME Layer will try to create this many, plus 1 administrative - * hardware queue. The administrative queue will always map to WQ 0 - * A hardware IO queue maps (qidx) to a specific driver WQ. + * lpfc_irq_chann: Set the number of IRQ vectors that are available + * for Hardware Queues to utilize. This also will map to the number + * of EQ / MSI-X vectors the driver will create. This should never be + * more than the number of Hardware Queues * - * 0 = Configure the number of io channels to the number of active CPUs. - * 1,32 = Manually specify how many io channels to use. + * 0 = Configure number of IRQ Channels to the number of active CPUs. + * 1,128 = Manually specify how many IRQ Channels to use. * - * Value range is [0,32]. Default value is 0. + * Value range is [0,128]. Default value is 0. */ -LPFC_ATTR_R(nvme_io_channel, - LPFC_NVME_IO_CHAN_DEF, - LPFC_HBA_IO_CHAN_MIN, LPFC_HBA_IO_CHAN_MAX, - "Set the number of NVME I/O channels"); +LPFC_ATTR_R(irq_chann, + LPFC_HBA_HDWQ_DEF, + LPFC_HBA_HDWQ_MIN, LPFC_HBA_HDWQ_MAX, + "Set the number of I/O IRQ Channels"); /* # lpfc_enable_hba_reset: Allow or prevent HBA resets to the hardware. @@ -5492,16 +5669,6 @@ LPFC_ATTR_RW(XLanePriority, 0, 0x0, 0x7f, "CS_CTL for Express Lane Feature."); LPFC_ATTR_R(enable_bg, 0, 0, 1, "Enable BlockGuard Support"); /* -# lpfc_fcp_look_ahead: Look ahead for completions in FCP start routine -# 0 = disabled (default) -# 1 = enabled -# Value range is [0,1]. Default value is 0. -# -# This feature in under investigation and may be supported in the future. -*/ -unsigned int lpfc_fcp_look_ahead = LPFC_LOOK_AHEAD_OFF; - -/* # lpfc_prot_mask: i # - Bit mask of host protection capabilities used to register with the # SCSI mid-layer @@ -5677,6 +5844,7 @@ LPFC_ATTR_RW(enable_dpp, 1, 0, 1, "Enable Direct Packet Push"); struct device_attribute *lpfc_hba_attrs[] = { &dev_attr_nvme_info, + &dev_attr_scsi_stat, &dev_attr_bg_info, &dev_attr_bg_guard_err, &dev_attr_bg_apptag_err, @@ -5704,11 +5872,11 @@ struct device_attribute *lpfc_hba_attrs[] = { &dev_attr_lpfc_nodev_tmo, &dev_attr_lpfc_devloss_tmo, &dev_attr_lpfc_enable_fc4_type, - &dev_attr_lpfc_xri_split, &dev_attr_lpfc_fcp_class, &dev_attr_lpfc_use_adisc, &dev_attr_lpfc_first_burst_size, &dev_attr_lpfc_ack0, + &dev_attr_lpfc_xri_rebalancing, &dev_attr_lpfc_topology, &dev_attr_lpfc_scan_down, &dev_attr_lpfc_link_speed, @@ -5742,12 +5910,13 @@ struct device_attribute *lpfc_hba_attrs[] = { &dev_attr_lpfc_use_msi, &dev_attr_lpfc_nvme_oas, &dev_attr_lpfc_nvme_embed_cmd, - &dev_attr_lpfc_auto_imax, &dev_attr_lpfc_fcp_imax, + &dev_attr_lpfc_cq_poll_threshold, + &dev_attr_lpfc_cq_max_proc_limit, &dev_attr_lpfc_fcp_cpu_map, - &dev_attr_lpfc_fcp_io_channel, + &dev_attr_lpfc_hdw_queue, + &dev_attr_lpfc_irq_chann, &dev_attr_lpfc_suppress_rsp, - &dev_attr_lpfc_nvme_io_channel, &dev_attr_lpfc_nvmet_mrq, &dev_attr_lpfc_nvmet_mrq_post, &dev_attr_lpfc_nvme_enable_fb, @@ -6775,6 +6944,7 @@ lpfc_get_cfgparam(struct lpfc_hba *phba) lpfc_multi_ring_rctl_init(phba, lpfc_multi_ring_rctl); lpfc_multi_ring_type_init(phba, lpfc_multi_ring_type); lpfc_ack0_init(phba, lpfc_ack0); + lpfc_xri_rebalancing_init(phba, lpfc_xri_rebalancing); lpfc_topology_init(phba, lpfc_topology); lpfc_link_speed_init(phba, lpfc_link_speed); lpfc_poll_tmo_init(phba, lpfc_poll_tmo); @@ -6787,8 +6957,9 @@ lpfc_get_cfgparam(struct lpfc_hba *phba) lpfc_use_msi_init(phba, lpfc_use_msi); lpfc_nvme_oas_init(phba, lpfc_nvme_oas); lpfc_nvme_embed_cmd_init(phba, lpfc_nvme_embed_cmd); - lpfc_auto_imax_init(phba, lpfc_auto_imax); lpfc_fcp_imax_init(phba, lpfc_fcp_imax); + lpfc_cq_poll_threshold_init(phba, lpfc_cq_poll_threshold); + lpfc_cq_max_proc_limit_init(phba, lpfc_cq_max_proc_limit); lpfc_fcp_cpu_map_init(phba, lpfc_fcp_cpu_map); lpfc_enable_hba_reset_init(phba, lpfc_enable_hba_reset); lpfc_enable_hba_heartbeat_init(phba, lpfc_enable_hba_heartbeat); @@ -6824,8 +6995,8 @@ lpfc_get_cfgparam(struct lpfc_hba *phba) /* Initialize first burst. Target vs Initiator are different. */ lpfc_nvme_enable_fb_init(phba, lpfc_nvme_enable_fb); lpfc_nvmet_fb_size_init(phba, lpfc_nvmet_fb_size); - lpfc_fcp_io_channel_init(phba, lpfc_fcp_io_channel); - lpfc_nvme_io_channel_init(phba, lpfc_nvme_io_channel); + lpfc_hdw_queue_init(phba, lpfc_hdw_queue); + lpfc_irq_chann_init(phba, lpfc_irq_chann); lpfc_enable_bbcr_init(phba, lpfc_enable_bbcr); lpfc_enable_dpp_init(phba, lpfc_enable_dpp); @@ -6834,38 +7005,27 @@ lpfc_get_cfgparam(struct lpfc_hba *phba) phba->nvmet_support = 0; phba->cfg_enable_fc4_type = LPFC_ENABLE_FCP; phba->cfg_enable_bbcr = 0; + phba->cfg_xri_rebalancing = 0; } else { /* We MUST have FCP support */ if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_FCP)) phba->cfg_enable_fc4_type |= LPFC_ENABLE_FCP; } - if (phba->cfg_auto_imax && !phba->cfg_fcp_imax) - phba->cfg_auto_imax = 0; - phba->initial_imax = phba->cfg_fcp_imax; + phba->cfg_auto_imax = (phba->cfg_fcp_imax) ? 0 : 1; phba->cfg_enable_pbde = 0; /* A value of 0 means use the number of CPUs found in the system */ - if (phba->cfg_fcp_io_channel == 0) - phba->cfg_fcp_io_channel = phba->sli4_hba.num_present_cpu; - if (phba->cfg_nvme_io_channel == 0) - phba->cfg_nvme_io_channel = phba->sli4_hba.num_present_cpu; - - if (phba->cfg_enable_fc4_type == LPFC_ENABLE_NVME) - phba->cfg_fcp_io_channel = 0; - - if (phba->cfg_enable_fc4_type == LPFC_ENABLE_FCP) - phba->cfg_nvme_io_channel = 0; - - if (phba->cfg_fcp_io_channel > phba->cfg_nvme_io_channel) - phba->io_channel_irqs = phba->cfg_fcp_io_channel; - else - phba->io_channel_irqs = phba->cfg_nvme_io_channel; + if (phba->cfg_hdw_queue == 0) + phba->cfg_hdw_queue = phba->sli4_hba.num_present_cpu; + if (phba->cfg_irq_chann == 0) + phba->cfg_irq_chann = phba->sli4_hba.num_present_cpu; + if (phba->cfg_irq_chann > phba->cfg_hdw_queue) + phba->cfg_irq_chann = phba->cfg_hdw_queue; phba->cfg_soft_wwnn = 0L; phba->cfg_soft_wwpn = 0L; - lpfc_xri_split_init(phba, lpfc_xri_split); lpfc_sg_seg_cnt_init(phba, lpfc_sg_seg_cnt); lpfc_hba_queue_depth_init(phba, lpfc_hba_queue_depth); lpfc_hba_log_verbose_init(phba, lpfc_log_verbose); @@ -6903,16 +7063,16 @@ lpfc_get_cfgparam(struct lpfc_hba *phba) void lpfc_nvme_mod_param_dep(struct lpfc_hba *phba) { - if (phba->cfg_nvme_io_channel > phba->sli4_hba.num_present_cpu) - phba->cfg_nvme_io_channel = phba->sli4_hba.num_present_cpu; - - if (phba->cfg_fcp_io_channel > phba->sli4_hba.num_present_cpu) - phba->cfg_fcp_io_channel = phba->sli4_hba.num_present_cpu; + if (phba->cfg_hdw_queue > phba->sli4_hba.num_present_cpu) + phba->cfg_hdw_queue = phba->sli4_hba.num_present_cpu; + if (phba->cfg_irq_chann > phba->sli4_hba.num_present_cpu) + phba->cfg_irq_chann = phba->sli4_hba.num_present_cpu; + if (phba->cfg_irq_chann > phba->cfg_hdw_queue) + phba->cfg_irq_chann = phba->cfg_hdw_queue; if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME && phba->nvmet_support) { phba->cfg_enable_fc4_type &= ~LPFC_ENABLE_FCP; - phba->cfg_fcp_io_channel = 0; lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC, "6013 %s x%x fb_size x%x, fb_max x%x\n", @@ -6929,11 +7089,11 @@ lpfc_nvme_mod_param_dep(struct lpfc_hba *phba) } if (!phba->cfg_nvmet_mrq) - phba->cfg_nvmet_mrq = phba->cfg_nvme_io_channel; + phba->cfg_nvmet_mrq = phba->cfg_irq_chann; /* Adjust lpfc_nvmet_mrq to avoid running out of WQE slots */ - if (phba->cfg_nvmet_mrq > phba->cfg_nvme_io_channel) { - phba->cfg_nvmet_mrq = phba->cfg_nvme_io_channel; + if (phba->cfg_nvmet_mrq > phba->cfg_irq_chann) { + phba->cfg_nvmet_mrq = phba->cfg_irq_chann; lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC, "6018 Adjust lpfc_nvmet_mrq to %d\n", phba->cfg_nvmet_mrq); @@ -6947,11 +7107,6 @@ lpfc_nvme_mod_param_dep(struct lpfc_hba *phba) phba->cfg_nvmet_mrq = LPFC_NVMET_MRQ_OFF; phba->cfg_nvmet_fb_size = 0; } - - if (phba->cfg_fcp_io_channel > phba->cfg_nvme_io_channel) - phba->io_channel_irqs = phba->cfg_fcp_io_channel; - else - phba->io_channel_irqs = phba->cfg_nvme_io_channel; } /** diff --git a/drivers/scsi/lpfc/lpfc_bsg.c b/drivers/scsi/lpfc/lpfc_bsg.c index 2dc564e59430..f2494d3b365c 100644 --- a/drivers/scsi/lpfc/lpfc_bsg.c +++ b/drivers/scsi/lpfc/lpfc_bsg.c @@ -2947,7 +2947,7 @@ static int lpfcdiag_loop_post_rxbufs(struct lpfc_hba *phba, uint16_t rxxri, cmd->un.cont64[i].addrLow = putPaddrLow(mp[i]->phys); cmd->un.cont64[i].tus.f.bdeSize = ((struct lpfc_dmabufext *)mp[i])->size; - cmd->ulpBdeCount = ++i; + cmd->ulpBdeCount = ++i; if ((--num_bde > 0) && (i < 2)) continue; @@ -4682,7 +4682,7 @@ lpfc_bsg_issue_mbox(struct lpfc_hba *phba, struct bsg_job *job, * Don't allow mailbox commands to be sent when blocked or when in * the middle of discovery */ - if (phba->sli.sli_flag & LPFC_BLOCK_MGMT_IO) { + if (phba->sli.sli_flag & LPFC_BLOCK_MGMT_IO) { rc = -EAGAIN; goto job_done; } diff --git a/drivers/scsi/lpfc/lpfc_crtn.h b/drivers/scsi/lpfc/lpfc_crtn.h index 39f3fa988732..e0b14d791b8c 100644 --- a/drivers/scsi/lpfc/lpfc_crtn.h +++ b/drivers/scsi/lpfc/lpfc_crtn.h @@ -1,7 +1,7 @@ /******************************************************************* * This file is part of the Emulex Linux Device Driver for * * Fibre Channel Host Bus Adapters. * - * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term * + * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term * * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * * Copyright (C) 2004-2016 Emulex. All rights reserved. * * EMULEX and SLI are trademarks of Emulex. * @@ -199,11 +199,6 @@ void lpfc_reset_hba(struct lpfc_hba *); int lpfc_emptyq_wait(struct lpfc_hba *phba, struct list_head *hd, spinlock_t *slock); -int lpfc_fof_queue_create(struct lpfc_hba *); -int lpfc_fof_queue_setup(struct lpfc_hba *); -int lpfc_fof_queue_destroy(struct lpfc_hba *); -irqreturn_t lpfc_sli4_fof_intr_handler(int, void *); - int lpfc_sli_setup(struct lpfc_hba *); int lpfc_sli4_setup(struct lpfc_hba *phba); void lpfc_sli_queue_init(struct lpfc_hba *phba); @@ -320,8 +315,8 @@ void lpfc_sli_def_mbox_cmpl(struct lpfc_hba *, LPFC_MBOXQ_t *); void lpfc_sli4_unreg_rpi_cmpl_clr(struct lpfc_hba *, LPFC_MBOXQ_t *); int lpfc_sli_issue_iocb(struct lpfc_hba *, uint32_t, struct lpfc_iocbq *, uint32_t); -int lpfc_sli4_issue_wqe(struct lpfc_hba *phba, uint32_t rnum, - struct lpfc_iocbq *iocbq); +int lpfc_sli4_issue_wqe(struct lpfc_hba *phba, struct lpfc_sli4_hdw_queue *qp, + struct lpfc_iocbq *pwqe); struct lpfc_sglq *__lpfc_clear_active_sglq(struct lpfc_hba *phba, uint16_t xri); struct lpfc_sglq *__lpfc_sli_get_nvmet_sglq(struct lpfc_hba *phba, struct lpfc_iocbq *piocbq); @@ -445,7 +440,6 @@ extern spinlock_t _dump_buf_lock; extern int _dump_buf_done; extern spinlock_t pgcnt_lock; extern unsigned int pgcnt; -extern unsigned int lpfc_fcp_look_ahead; /* Interface exported by fabric iocb scheduler */ void lpfc_fabric_abort_nport(struct lpfc_nodelist *); @@ -520,8 +514,13 @@ int lpfc_sli4_read_config(struct lpfc_hba *); void lpfc_sli4_node_prep(struct lpfc_hba *); int lpfc_sli4_els_sgl_update(struct lpfc_hba *phba); int lpfc_sli4_nvmet_sgl_update(struct lpfc_hba *phba); -int lpfc_sli4_scsi_sgl_update(struct lpfc_hba *phba); -int lpfc_sli4_nvme_sgl_update(struct lpfc_hba *phba); +int lpfc_io_buf_flush(struct lpfc_hba *phba, struct list_head *sglist); +int lpfc_io_buf_replenish(struct lpfc_hba *phba, struct list_head *cbuf); +int lpfc_sli4_io_sgl_update(struct lpfc_hba *phba); +int lpfc_sli4_post_io_sgl_list(struct lpfc_hba *phba, + struct list_head *blist, int xricnt); +int lpfc_new_io_buf(struct lpfc_hba *phba, int num_to_alloc); +void lpfc_io_free(struct lpfc_hba *phba); void lpfc_free_sgl_list(struct lpfc_hba *, struct list_head *); uint32_t lpfc_sli_port_speed_get(struct lpfc_hba *); int lpfc_sli4_request_firmware_update(struct lpfc_hba *, uint8_t); @@ -574,6 +573,21 @@ void lpfc_nvme_mod_param_dep(struct lpfc_hba *phba); void lpfc_nvme_abort_fcreq_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb, struct lpfc_wcqe_complete *abts_cmpl); +void lpfc_create_multixri_pools(struct lpfc_hba *phba); +void lpfc_create_destroy_pools(struct lpfc_hba *phba); +void lpfc_move_xri_pvt_to_pbl(struct lpfc_hba *phba, u32 hwqid); +void lpfc_move_xri_pbl_to_pvt(struct lpfc_hba *phba, u32 hwqid, u32 cnt); +void lpfc_adjust_high_watermark(struct lpfc_hba *phba, u32 hwqid); +void lpfc_keep_pvt_pool_above_lowwm(struct lpfc_hba *phba, u32 hwqid); +void lpfc_adjust_pvt_pool_count(struct lpfc_hba *phba, u32 hwqid); +#ifdef LPFC_MXP_STAT +void lpfc_snapshot_mxp(struct lpfc_hba *, u32); +#endif +struct lpfc_io_buf *lpfc_get_io_buf(struct lpfc_hba *phba, + struct lpfc_nodelist *ndlp, u32 hwqid, + int); +void lpfc_release_io_buf(struct lpfc_hba *phba, struct lpfc_io_buf *ncmd, + struct lpfc_sli4_hdw_queue *qp); void lpfc_nvme_cmd_template(void); void lpfc_nvmet_cmd_template(void); extern int lpfc_enable_nvmet_cnt; diff --git a/drivers/scsi/lpfc/lpfc_ct.c b/drivers/scsi/lpfc/lpfc_ct.c index 552da8bf43e4..7290573110fe 100644 --- a/drivers/scsi/lpfc/lpfc_ct.c +++ b/drivers/scsi/lpfc/lpfc_ct.c @@ -1,7 +1,7 @@ /******************************************************************* * This file is part of the Emulex Linux Device Driver for * * Fibre Channel Host Bus Adapters. * - * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term * + * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term * * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * * Copyright (C) 2004-2016 Emulex. All rights reserved. * * EMULEX and SLI are trademarks of Emulex. * @@ -1656,16 +1656,16 @@ lpfc_ns_cmd(struct lpfc_vport *vport, int cmdcode, CtReq->un.rft.PortId = cpu_to_be32(vport->fc_myDID); /* Register FC4 FCP type if enabled. */ - if ((phba->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) || - (phba->cfg_enable_fc4_type == LPFC_ENABLE_FCP)) + if (vport->cfg_enable_fc4_type == LPFC_ENABLE_BOTH || + vport->cfg_enable_fc4_type == LPFC_ENABLE_FCP) CtReq->un.rft.fcpReg = 1; /* Register NVME type if enabled. Defined LE and swapped. * rsvd[0] is used as word1 because of the hard-coded * word0 usage in the ct_request data structure. */ - if ((phba->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) || - (phba->cfg_enable_fc4_type == LPFC_ENABLE_NVME)) + if (vport->cfg_enable_fc4_type == LPFC_ENABLE_BOTH || + vport->cfg_enable_fc4_type == LPFC_ENABLE_NVME) CtReq->un.rft.rsvd[0] = cpu_to_be32(LPFC_FC4_TYPE_BITMASK); @@ -1732,8 +1732,8 @@ lpfc_ns_cmd(struct lpfc_vport *vport, int cmdcode, * caller can specify NVME (type x28) as well. But only * these that FC4 type is supported. */ - if (((phba->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) || - (phba->cfg_enable_fc4_type == LPFC_ENABLE_NVME)) && + if (((vport->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) || + (vport->cfg_enable_fc4_type == LPFC_ENABLE_NVME)) && (context == FC_TYPE_NVME)) { if ((vport == phba->pport) && phba->nvmet_support) { CtReq->un.rff.fbits = (FC4_FEATURE_TARGET | @@ -1744,8 +1744,8 @@ lpfc_ns_cmd(struct lpfc_vport *vport, int cmdcode, } CtReq->un.rff.type_code = context; - } else if (((phba->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) || - (phba->cfg_enable_fc4_type == LPFC_ENABLE_FCP)) && + } else if (((vport->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) || + (vport->cfg_enable_fc4_type == LPFC_ENABLE_FCP)) && (context == FC_TYPE_FCP)) CtReq->un.rff.type_code = context; diff --git a/drivers/scsi/lpfc/lpfc_debugfs.c b/drivers/scsi/lpfc/lpfc_debugfs.c index a58f0b3f03a9..1215eaa530db 100644 --- a/drivers/scsi/lpfc/lpfc_debugfs.c +++ b/drivers/scsi/lpfc/lpfc_debugfs.c @@ -1,7 +1,7 @@ /******************************************************************* * This file is part of the Emulex Linux Device Driver for * * Fibre Channel Host Bus Adapters. * - * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term * + * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term * * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * * Copyright (C) 2007-2015 Emulex. All rights reserved. * * EMULEX and SLI are trademarks of Emulex. * @@ -378,6 +378,271 @@ skipit: return len; } +static int lpfc_debugfs_last_xripool; + +/** + * lpfc_debugfs_common_xri_data - Dump Hardware Queue info to a buffer + * @phba: The HBA to gather host buffer info from. + * @buf: The buffer to dump log into. + * @size: The maximum amount of data to process. + * + * Description: + * This routine dumps the Hardware Queue info from the @phba to @buf up to + * @size number of bytes. A header that describes the current hdwq state will be + * dumped to @buf first and then info on each hdwq entry will be dumped to @buf + * until @size bytes have been dumped or all the hdwq info has been dumped. + * + * Notes: + * This routine will rotate through each configured Hardware Queue each + * time called. + * + * Return Value: + * This routine returns the amount of bytes that were dumped into @buf and will + * not exceed @size. + **/ +static int +lpfc_debugfs_commonxripools_data(struct lpfc_hba *phba, char *buf, int size) +{ + struct lpfc_sli4_hdw_queue *qp; + int len = 0; + int i, out; + unsigned long iflag; + + for (i = 0; i < phba->cfg_hdw_queue; i++) { + if (len > (LPFC_DUMP_MULTIXRIPOOL_SIZE - 80)) + break; + qp = &phba->sli4_hba.hdwq[lpfc_debugfs_last_xripool]; + + len += snprintf(buf + len, size - len, "HdwQ %d Info ", i); + spin_lock_irqsave(&qp->abts_scsi_buf_list_lock, iflag); + spin_lock(&qp->abts_nvme_buf_list_lock); + spin_lock(&qp->io_buf_list_get_lock); + spin_lock(&qp->io_buf_list_put_lock); + out = qp->total_io_bufs - (qp->get_io_bufs + qp->put_io_bufs + + qp->abts_scsi_io_bufs + qp->abts_nvme_io_bufs); + len += snprintf(buf + len, size - len, + "tot:%d get:%d put:%d mt:%d " + "ABTS scsi:%d nvme:%d Out:%d\n", + qp->total_io_bufs, qp->get_io_bufs, qp->put_io_bufs, + qp->empty_io_bufs, qp->abts_scsi_io_bufs, + qp->abts_nvme_io_bufs, out); + spin_unlock(&qp->io_buf_list_put_lock); + spin_unlock(&qp->io_buf_list_get_lock); + spin_unlock(&qp->abts_nvme_buf_list_lock); + spin_unlock_irqrestore(&qp->abts_scsi_buf_list_lock, iflag); + + lpfc_debugfs_last_xripool++; + if (lpfc_debugfs_last_xripool >= phba->cfg_hdw_queue) + lpfc_debugfs_last_xripool = 0; + } + + return len; +} + +/** + * lpfc_debugfs_multixripools_data - Display multi-XRI pools information + * @phba: The HBA to gather host buffer info from. + * @buf: The buffer to dump log into. + * @size: The maximum amount of data to process. + * + * Description: + * This routine displays current multi-XRI pools information including XRI + * count in public, private and txcmplq. It also displays current high and + * low watermark. + * + * Return Value: + * This routine returns the amount of bytes that were dumped into @buf and will + * not exceed @size. + **/ +static int +lpfc_debugfs_multixripools_data(struct lpfc_hba *phba, char *buf, int size) +{ + u32 i; + u32 hwq_count; + struct lpfc_sli4_hdw_queue *qp; + struct lpfc_multixri_pool *multixri_pool; + struct lpfc_pvt_pool *pvt_pool; + struct lpfc_pbl_pool *pbl_pool; + u32 txcmplq_cnt; + char tmp[LPFC_DEBUG_OUT_LINE_SZ] = {0}; + + if (phba->sli_rev != LPFC_SLI_REV4) + return 0; + + if (!phba->sli4_hba.hdwq) + return 0; + + if (!phba->cfg_xri_rebalancing) { + i = lpfc_debugfs_commonxripools_data(phba, buf, size); + return i; + } + + /* + * Pbl: Current number of free XRIs in public pool + * Pvt: Current number of free XRIs in private pool + * Busy: Current number of outstanding XRIs + * HWM: Current high watermark + * pvt_empty: Incremented by 1 when IO submission fails (no xri) + * pbl_empty: Incremented by 1 when all pbl_pool are empty during + * IO submission + */ + scnprintf(tmp, sizeof(tmp), + "HWQ: Pbl Pvt Busy HWM | pvt_empty pbl_empty "); + if (strlcat(buf, tmp, size) >= size) + return strnlen(buf, size); + +#ifdef LPFC_MXP_STAT + /* + * MAXH: Max high watermark seen so far + * above_lmt: Incremented by 1 if xri_owned > xri_limit during + * IO submission + * below_lmt: Incremented by 1 if xri_owned <= xri_limit during + * IO submission + * locPbl_hit: Incremented by 1 if successfully get a batch of XRI from + * local pbl_pool + * othPbl_hit: Incremented by 1 if successfully get a batch of XRI from + * other pbl_pool + */ + scnprintf(tmp, sizeof(tmp), + "MAXH above_lmt below_lmt locPbl_hit othPbl_hit"); + if (strlcat(buf, tmp, size) >= size) + return strnlen(buf, size); + + /* + * sPbl: snapshot of Pbl 15 sec after stat gets cleared + * sPvt: snapshot of Pvt 15 sec after stat gets cleared + * sBusy: snapshot of Busy 15 sec after stat gets cleared + */ + scnprintf(tmp, sizeof(tmp), + " | sPbl sPvt sBusy"); + if (strlcat(buf, tmp, size) >= size) + return strnlen(buf, size); +#endif + + scnprintf(tmp, sizeof(tmp), "\n"); + if (strlcat(buf, tmp, size) >= size) + return strnlen(buf, size); + + hwq_count = phba->cfg_hdw_queue; + for (i = 0; i < hwq_count; i++) { + qp = &phba->sli4_hba.hdwq[i]; + multixri_pool = qp->p_multixri_pool; + if (!multixri_pool) + continue; + pbl_pool = &multixri_pool->pbl_pool; + pvt_pool = &multixri_pool->pvt_pool; + txcmplq_cnt = qp->fcp_wq->pring->txcmplq_cnt; + if (qp->nvme_wq) + txcmplq_cnt += qp->nvme_wq->pring->txcmplq_cnt; + + scnprintf(tmp, sizeof(tmp), + "%03d: %4d %4d %4d %4d | %10d %10d ", + i, pbl_pool->count, pvt_pool->count, + txcmplq_cnt, pvt_pool->high_watermark, + qp->empty_io_bufs, multixri_pool->pbl_empty_count); + if (strlcat(buf, tmp, size) >= size) + break; + +#ifdef LPFC_MXP_STAT + scnprintf(tmp, sizeof(tmp), + "%4d %10d %10d %10d %10d", + multixri_pool->stat_max_hwm, + multixri_pool->above_limit_count, + multixri_pool->below_limit_count, + multixri_pool->local_pbl_hit_count, + multixri_pool->other_pbl_hit_count); + if (strlcat(buf, tmp, size) >= size) + break; + + scnprintf(tmp, sizeof(tmp), + " | %4d %4d %5d", + multixri_pool->stat_pbl_count, + multixri_pool->stat_pvt_count, + multixri_pool->stat_busy_count); + if (strlcat(buf, tmp, size) >= size) + break; +#endif + + scnprintf(tmp, sizeof(tmp), "\n"); + if (strlcat(buf, tmp, size) >= size) + break; + } + return strnlen(buf, size); +} + + +#ifdef LPFC_HDWQ_LOCK_STAT +static int lpfc_debugfs_last_lock; + +/** + * lpfc_debugfs_lockstat_data - Dump Hardware Queue info to a buffer + * @phba: The HBA to gather host buffer info from. + * @buf: The buffer to dump log into. + * @size: The maximum amount of data to process. + * + * Description: + * This routine dumps the Hardware Queue info from the @phba to @buf up to + * @size number of bytes. A header that describes the current hdwq state will be + * dumped to @buf first and then info on each hdwq entry will be dumped to @buf + * until @size bytes have been dumped or all the hdwq info has been dumped. + * + * Notes: + * This routine will rotate through each configured Hardware Queue each + * time called. + * + * Return Value: + * This routine returns the amount of bytes that were dumped into @buf and will + * not exceed @size. + **/ +static int +lpfc_debugfs_lockstat_data(struct lpfc_hba *phba, char *buf, int size) +{ + struct lpfc_sli4_hdw_queue *qp; + int len = 0; + int i; + + if (phba->sli_rev != LPFC_SLI_REV4) + return 0; + + if (!phba->sli4_hba.hdwq) + return 0; + + for (i = 0; i < phba->cfg_hdw_queue; i++) { + if (len > (LPFC_HDWQINFO_SIZE - 100)) + break; + qp = &phba->sli4_hba.hdwq[lpfc_debugfs_last_lock]; + + len += snprintf(buf + len, size - len, "HdwQ %03d Lock ", i); + if (phba->cfg_xri_rebalancing) { + len += snprintf(buf + len, size - len, + "get_pvt:%d mv_pvt:%d " + "mv2pub:%d mv2pvt:%d " + "put_pvt:%d put_pub:%d wq:%d\n", + qp->lock_conflict.alloc_pvt_pool, + qp->lock_conflict.mv_from_pvt_pool, + qp->lock_conflict.mv_to_pub_pool, + qp->lock_conflict.mv_to_pvt_pool, + qp->lock_conflict.free_pvt_pool, + qp->lock_conflict.free_pub_pool, + qp->lock_conflict.wq_access); + } else { + len += snprintf(buf + len, size - len, + "get:%d put:%d free:%d wq:%d\n", + qp->lock_conflict.alloc_xri_get, + qp->lock_conflict.alloc_xri_put, + qp->lock_conflict.free_xri, + qp->lock_conflict.wq_access); + } + + lpfc_debugfs_last_lock++; + if (lpfc_debugfs_last_lock >= phba->cfg_hdw_queue) + lpfc_debugfs_last_lock = 0; + } + + return len; +} +#endif + static int lpfc_debugfs_last_hba_slim_off; /** @@ -773,11 +1038,11 @@ lpfc_debugfs_nvmestat_data(struct lpfc_vport *vport, char *buf, int size) struct lpfc_nvmet_tgtport *tgtp; struct lpfc_nvmet_rcv_ctx *ctxp, *next_ctxp; struct nvme_fc_local_port *localport; - struct lpfc_nvme_ctrl_stat *cstat; + struct lpfc_fc4_ctrl_stat *cstat; struct lpfc_nvme_lport *lport; uint64_t data1, data2, data3; uint64_t tot, totin, totout; - int cnt, i, maxch; + int cnt, i; int len = 0; if (phba->nvmet_support) { @@ -863,17 +1128,17 @@ lpfc_debugfs_nvmestat_data(struct lpfc_vport *vport, char *buf, int size) len += snprintf(buf + len, size - len, "\n"); cnt = 0; - spin_lock(&phba->sli4_hba.abts_nvme_buf_list_lock); + spin_lock(&phba->sli4_hba.abts_nvmet_buf_list_lock); list_for_each_entry_safe(ctxp, next_ctxp, &phba->sli4_hba.lpfc_abts_nvmet_ctx_list, list) { cnt++; } - spin_unlock(&phba->sli4_hba.abts_nvme_buf_list_lock); + spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock); if (cnt) { len += snprintf(buf + len, size - len, "ABORT: %d ctx entries\n", cnt); - spin_lock(&phba->sli4_hba.abts_nvme_buf_list_lock); + spin_lock(&phba->sli4_hba.abts_nvmet_buf_list_lock); list_for_each_entry_safe(ctxp, next_ctxp, &phba->sli4_hba.lpfc_abts_nvmet_ctx_list, list) { @@ -885,7 +1150,7 @@ lpfc_debugfs_nvmestat_data(struct lpfc_vport *vport, char *buf, int size) ctxp->oxid, ctxp->state, ctxp->flag); } - spin_unlock(&phba->sli4_hba.abts_nvme_buf_list_lock); + spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock); } /* Calculate outstanding IOs */ @@ -901,7 +1166,7 @@ lpfc_debugfs_nvmestat_data(struct lpfc_vport *vport, char *buf, int size) phba->sli4_hba.nvmet_io_wait_total, tot); } else { - if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME)) + if (!(vport->cfg_enable_fc4_type & LPFC_ENABLE_NVME)) return len; localport = vport->localport; @@ -912,26 +1177,22 @@ lpfc_debugfs_nvmestat_data(struct lpfc_vport *vport, char *buf, int size) return len; len += snprintf(buf + len, size - len, - "\nNVME Lport Statistics\n"); + "\nNVME HDWQ Statistics\n"); len += snprintf(buf + len, size - len, "LS: Xmt %016x Cmpl %016x\n", atomic_read(&lport->fc4NvmeLsRequests), atomic_read(&lport->fc4NvmeLsCmpls)); - if (phba->cfg_nvme_io_channel < 32) - maxch = phba->cfg_nvme_io_channel; - else - maxch = 32; totin = 0; totout = 0; - for (i = 0; i < phba->cfg_nvme_io_channel; i++) { - cstat = &lport->cstat[i]; - tot = atomic_read(&cstat->fc4NvmeIoCmpls); + for (i = 0; i < phba->cfg_hdw_queue; i++) { + cstat = &phba->sli4_hba.hdwq[i].nvme_cstat; + tot = cstat->io_cmpls; totin += tot; - data1 = atomic_read(&cstat->fc4NvmeInputRequests); - data2 = atomic_read(&cstat->fc4NvmeOutputRequests); - data3 = atomic_read(&cstat->fc4NvmeControlRequests); + data1 = cstat->input_requests; + data2 = cstat->output_requests; + data3 = cstat->control_requests; totout += (data1 + data2 + data3); /* Limit to 32, debugfs display buffer limitation */ @@ -939,7 +1200,7 @@ lpfc_debugfs_nvmestat_data(struct lpfc_vport *vport, char *buf, int size) continue; len += snprintf(buf + len, PAGE_SIZE - len, - "FCP (%d): Rd %016llx Wr %016llx " + "HDWQ (%d): Rd %016llx Wr %016llx " "IO %016llx ", i, data1, data2, data3); len += snprintf(buf + len, PAGE_SIZE - len, @@ -979,6 +1240,66 @@ lpfc_debugfs_nvmestat_data(struct lpfc_vport *vport, char *buf, int size) return len; } +/** + * lpfc_debugfs_scsistat_data - Dump target node list to a buffer + * @vport: The vport to gather target node info from. + * @buf: The buffer to dump log into. + * @size: The maximum amount of data to process. + * + * Description: + * This routine dumps the SCSI statistics associated with @vport + * + * Return Value: + * This routine returns the amount of bytes that were dumped into @buf and will + * not exceed @size. + **/ +static int +lpfc_debugfs_scsistat_data(struct lpfc_vport *vport, char *buf, int size) +{ + int len; + struct lpfc_hba *phba = vport->phba; + struct lpfc_fc4_ctrl_stat *cstat; + u64 data1, data2, data3; + u64 tot, totin, totout; + int i; + char tmp[LPFC_MAX_SCSI_INFO_TMP_LEN] = {0}; + + if (!(vport->cfg_enable_fc4_type & LPFC_ENABLE_FCP) || + (phba->sli_rev != LPFC_SLI_REV4)) + return 0; + + scnprintf(buf, size, "SCSI HDWQ Statistics\n"); + + totin = 0; + totout = 0; + for (i = 0; i < phba->cfg_hdw_queue; i++) { + cstat = &phba->sli4_hba.hdwq[i].scsi_cstat; + tot = cstat->io_cmpls; + totin += tot; + data1 = cstat->input_requests; + data2 = cstat->output_requests; + data3 = cstat->control_requests; + totout += (data1 + data2 + data3); + + scnprintf(tmp, sizeof(tmp), "HDWQ (%d): Rd %016llx Wr %016llx " + "IO %016llx ", i, data1, data2, data3); + if (strlcat(buf, tmp, size) >= size) + goto buffer_done; + + scnprintf(tmp, sizeof(tmp), "Cmpl %016llx OutIO %016llx\n", + tot, ((data1 + data2 + data3) - tot)); + if (strlcat(buf, tmp, size) >= size) + goto buffer_done; + } + scnprintf(tmp, sizeof(tmp), "Total FCP Cmpl %016llx Issue %016llx " + "OutIO %016llx\n", totin, totout, totout - totin); + strlcat(buf, tmp, size); + +buffer_done: + len = strnlen(buf, size); + + return len; +} /** * lpfc_debugfs_nvmektime_data - Dump target node list to a buffer @@ -1299,62 +1620,73 @@ static int lpfc_debugfs_cpucheck_data(struct lpfc_vport *vport, char *buf, int size) { struct lpfc_hba *phba = vport->phba; - int i; + struct lpfc_sli4_hdw_queue *qp; + int i, j, max_cnt; int len = 0; - uint32_t tot_xmt = 0; - uint32_t tot_rcv = 0; - uint32_t tot_cmpl = 0; - uint32_t tot_ccmpl = 0; + uint32_t tot_xmt; + uint32_t tot_rcv; + uint32_t tot_cmpl; - if (phba->nvmet_support == 0) { - /* NVME Initiator */ - len += snprintf(buf + len, PAGE_SIZE - len, - "CPUcheck %s\n", - (phba->cpucheck_on & LPFC_CHECK_NVME_IO ? - "Enabled" : "Disabled")); - for (i = 0; i < phba->sli4_hba.num_present_cpu; i++) { - if (i >= LPFC_CHECK_CPU_CNT) - break; - len += snprintf(buf + len, PAGE_SIZE - len, - "%02d: xmit x%08x cmpl x%08x\n", - i, phba->cpucheck_xmt_io[i], - phba->cpucheck_cmpl_io[i]); - tot_xmt += phba->cpucheck_xmt_io[i]; - tot_cmpl += phba->cpucheck_cmpl_io[i]; - } + len += snprintf(buf + len, PAGE_SIZE - len, + "CPUcheck %s ", + (phba->cpucheck_on & LPFC_CHECK_NVME_IO ? + "Enabled" : "Disabled")); + if (phba->nvmet_support) { len += snprintf(buf + len, PAGE_SIZE - len, - "tot:xmit x%08x cmpl x%08x\n", - tot_xmt, tot_cmpl); - return len; + "%s\n", + (phba->cpucheck_on & LPFC_CHECK_NVMET_RCV ? + "Rcv Enabled\n" : "Rcv Disabled\n")); + } else { + len += snprintf(buf + len, PAGE_SIZE - len, "\n"); } + max_cnt = size - LPFC_DEBUG_OUT_LINE_SZ; + + for (i = 0; i < phba->cfg_hdw_queue; i++) { + qp = &phba->sli4_hba.hdwq[i]; + + tot_rcv = 0; + tot_xmt = 0; + tot_cmpl = 0; + for (j = 0; j < LPFC_CHECK_CPU_CNT; j++) { + tot_xmt += qp->cpucheck_xmt_io[j]; + tot_cmpl += qp->cpucheck_cmpl_io[j]; + if (phba->nvmet_support) + tot_rcv += qp->cpucheck_rcv_io[j]; + } + + /* Only display Hardware Qs with something */ + if (!tot_xmt && !tot_cmpl && !tot_rcv) + continue; - /* NVME Target */ - len += snprintf(buf + len, PAGE_SIZE - len, - "CPUcheck %s ", - (phba->cpucheck_on & LPFC_CHECK_NVMET_IO ? - "IO Enabled - " : "IO Disabled - ")); - len += snprintf(buf + len, PAGE_SIZE - len, - "%s\n", - (phba->cpucheck_on & LPFC_CHECK_NVMET_RCV ? - "Rcv Enabled\n" : "Rcv Disabled\n")); - for (i = 0; i < phba->sli4_hba.num_present_cpu; i++) { - if (i >= LPFC_CHECK_CPU_CNT) - break; len += snprintf(buf + len, PAGE_SIZE - len, - "%02d: xmit x%08x ccmpl x%08x " - "cmpl x%08x rcv x%08x\n", - i, phba->cpucheck_xmt_io[i], - phba->cpucheck_ccmpl_io[i], - phba->cpucheck_cmpl_io[i], - phba->cpucheck_rcv_io[i]); - tot_xmt += phba->cpucheck_xmt_io[i]; - tot_rcv += phba->cpucheck_rcv_io[i]; - tot_cmpl += phba->cpucheck_cmpl_io[i]; - tot_ccmpl += phba->cpucheck_ccmpl_io[i]; + "HDWQ %03d: ", i); + for (j = 0; j < LPFC_CHECK_CPU_CNT; j++) { + /* Only display non-zero counters */ + if (!qp->cpucheck_xmt_io[j] && + !qp->cpucheck_cmpl_io[j] && + !qp->cpucheck_rcv_io[j]) + continue; + if (phba->nvmet_support) { + len += snprintf(buf + len, PAGE_SIZE - len, + "CPU %03d: %x/%x/%x ", j, + qp->cpucheck_rcv_io[j], + qp->cpucheck_xmt_io[j], + qp->cpucheck_cmpl_io[j]); + } else { + len += snprintf(buf + len, PAGE_SIZE - len, + "CPU %03d: %x/%x ", j, + qp->cpucheck_xmt_io[j], + qp->cpucheck_cmpl_io[j]); + } + } + len += snprintf(buf + len, PAGE_SIZE - len, + "Total: %x\n", tot_xmt); + if (len >= max_cnt) { + len += snprintf(buf + len, PAGE_SIZE - len, + "Truncated ...\n"); + return len; + } } - len += snprintf(buf + len, PAGE_SIZE - len, - "tot:xmit x%08x ccmpl x%08x cmpl x%08x rcv x%08x\n", - tot_xmt, tot_ccmpl, tot_cmpl, tot_rcv); return len; } @@ -1501,7 +1833,7 @@ lpfc_debugfs_disc_trc_open(struct inode *inode, struct file *file) int rc = -ENOMEM; if (!lpfc_debugfs_max_disc_trc) { - rc = -ENOSPC; + rc = -ENOSPC; goto out; } @@ -1551,7 +1883,7 @@ lpfc_debugfs_slow_ring_trc_open(struct inode *inode, struct file *file) int rc = -ENOMEM; if (!lpfc_debugfs_max_slow_ring_trc) { - rc = -ENOSPC; + rc = -ENOSPC; goto out; } @@ -1620,6 +1952,135 @@ out: } /** + * lpfc_debugfs_multixripools_open - Open the multixripool debugfs buffer + * @inode: The inode pointer that contains a hba pointer. + * @file: The file pointer to attach the log output. + * + * Description: + * This routine is the entry point for the debugfs open file operation. It gets + * the hba from the i_private field in @inode, allocates the necessary buffer + * for the log, fills the buffer from the in-memory log for this hba, and then + * returns a pointer to that log in the private_data field in @file. + * + * Returns: + * This function returns zero if successful. On error it will return a negative + * error value. + **/ +static int +lpfc_debugfs_multixripools_open(struct inode *inode, struct file *file) +{ + struct lpfc_hba *phba = inode->i_private; + struct lpfc_debug *debug; + int rc = -ENOMEM; + + debug = kmalloc(sizeof(*debug), GFP_KERNEL); + if (!debug) + goto out; + + /* Round to page boundary */ + debug->buffer = kzalloc(LPFC_DUMP_MULTIXRIPOOL_SIZE, GFP_KERNEL); + if (!debug->buffer) { + kfree(debug); + goto out; + } + + debug->len = lpfc_debugfs_multixripools_data( + phba, debug->buffer, LPFC_DUMP_MULTIXRIPOOL_SIZE); + + debug->i_private = inode->i_private; + file->private_data = debug; + + rc = 0; +out: + return rc; +} + +#ifdef LPFC_HDWQ_LOCK_STAT +/** + * lpfc_debugfs_lockstat_open - Open the lockstat debugfs buffer + * @inode: The inode pointer that contains a vport pointer. + * @file: The file pointer to attach the log output. + * + * Description: + * This routine is the entry point for the debugfs open file operation. It gets + * the vport from the i_private field in @inode, allocates the necessary buffer + * for the log, fills the buffer from the in-memory log for this vport, and then + * returns a pointer to that log in the private_data field in @file. + * + * Returns: + * This function returns zero if successful. On error it will return a negative + * error value. + **/ +static int +lpfc_debugfs_lockstat_open(struct inode *inode, struct file *file) +{ + struct lpfc_hba *phba = inode->i_private; + struct lpfc_debug *debug; + int rc = -ENOMEM; + + debug = kmalloc(sizeof(*debug), GFP_KERNEL); + if (!debug) + goto out; + + /* Round to page boundary */ + debug->buffer = kmalloc(LPFC_HDWQINFO_SIZE, GFP_KERNEL); + if (!debug->buffer) { + kfree(debug); + goto out; + } + + debug->len = lpfc_debugfs_lockstat_data(phba, debug->buffer, + LPFC_HBQINFO_SIZE); + file->private_data = debug; + + rc = 0; +out: + return rc; +} + +static ssize_t +lpfc_debugfs_lockstat_write(struct file *file, const char __user *buf, + size_t nbytes, loff_t *ppos) +{ + struct lpfc_debug *debug = file->private_data; + struct lpfc_hba *phba = (struct lpfc_hba *)debug->i_private; + struct lpfc_sli4_hdw_queue *qp; + char mybuf[64]; + char *pbuf; + int i; + + /* Protect copy from user */ + if (!access_ok(buf, nbytes)) + return -EFAULT; + + memset(mybuf, 0, sizeof(mybuf)); + + if (copy_from_user(mybuf, buf, nbytes)) + return -EFAULT; + pbuf = &mybuf[0]; + + if ((strncmp(pbuf, "reset", strlen("reset")) == 0) || + (strncmp(pbuf, "zero", strlen("zero")) == 0)) { + for (i = 0; i < phba->cfg_hdw_queue; i++) { + qp = &phba->sli4_hba.hdwq[i]; + qp->lock_conflict.alloc_xri_get = 0; + qp->lock_conflict.alloc_xri_put = 0; + qp->lock_conflict.free_xri = 0; + qp->lock_conflict.wq_access = 0; + qp->lock_conflict.alloc_pvt_pool = 0; + qp->lock_conflict.mv_from_pvt_pool = 0; + qp->lock_conflict.mv_to_pub_pool = 0; + qp->lock_conflict.mv_to_pvt_pool = 0; + qp->lock_conflict.free_pvt_pool = 0; + qp->lock_conflict.free_pub_pool = 0; + qp->lock_conflict.wq_access = 0; + } + } + return nbytes; +} +#endif + +/** * lpfc_debugfs_dumpHBASlim_open - Open the Dump HBA SLIM debugfs buffer * @inode: The inode pointer that contains a vport pointer. * @file: The file pointer to attach the log output. @@ -2008,6 +2469,75 @@ lpfc_debugfs_dumpDataDif_release(struct inode *inode, struct file *file) return 0; } +/** + * lpfc_debugfs_multixripools_write - Clear multi-XRI pools statistics + * @file: The file pointer to read from. + * @buf: The buffer to copy the user data from. + * @nbytes: The number of bytes to get. + * @ppos: The position in the file to start reading from. + * + * Description: + * This routine clears multi-XRI pools statistics when buf contains "clear". + * + * Return Value: + * It returns the @nbytges passing in from debugfs user space when successful. + * In case of error conditions, it returns proper error code back to the user + * space. + **/ +static ssize_t +lpfc_debugfs_multixripools_write(struct file *file, const char __user *buf, + size_t nbytes, loff_t *ppos) +{ + struct lpfc_debug *debug = file->private_data; + struct lpfc_hba *phba = (struct lpfc_hba *)debug->i_private; + char mybuf[64]; + char *pbuf; + u32 i; + u32 hwq_count; + struct lpfc_sli4_hdw_queue *qp; + struct lpfc_multixri_pool *multixri_pool; + + if (nbytes > 64) + nbytes = 64; + + /* Protect copy from user */ + if (!access_ok(buf, nbytes)) + return -EFAULT; + + memset(mybuf, 0, sizeof(mybuf)); + + if (copy_from_user(mybuf, buf, nbytes)) + return -EFAULT; + pbuf = &mybuf[0]; + + if ((strncmp(pbuf, "clear", strlen("clear"))) == 0) { + hwq_count = phba->cfg_hdw_queue; + for (i = 0; i < hwq_count; i++) { + qp = &phba->sli4_hba.hdwq[i]; + multixri_pool = qp->p_multixri_pool; + if (!multixri_pool) + continue; + + qp->empty_io_bufs = 0; + multixri_pool->pbl_empty_count = 0; +#ifdef LPFC_MXP_STAT + multixri_pool->above_limit_count = 0; + multixri_pool->below_limit_count = 0; + multixri_pool->stat_max_hwm = 0; + multixri_pool->local_pbl_hit_count = 0; + multixri_pool->other_pbl_hit_count = 0; + + multixri_pool->stat_pbl_count = 0; + multixri_pool->stat_pvt_count = 0; + multixri_pool->stat_busy_count = 0; + multixri_pool->stat_snapshot_taken = 0; +#endif + } + return strlen(pbuf); + } + + return -EINVAL; +} static int lpfc_debugfs_nvmestat_open(struct inode *inode, struct file *file) @@ -2098,6 +2628,64 @@ lpfc_debugfs_nvmestat_write(struct file *file, const char __user *buf, } static int +lpfc_debugfs_scsistat_open(struct inode *inode, struct file *file) +{ + struct lpfc_vport *vport = inode->i_private; + struct lpfc_debug *debug; + int rc = -ENOMEM; + + debug = kmalloc(sizeof(*debug), GFP_KERNEL); + if (!debug) + goto out; + + /* Round to page boundary */ + debug->buffer = kzalloc(LPFC_SCSISTAT_SIZE, GFP_KERNEL); + if (!debug->buffer) { + kfree(debug); + goto out; + } + + debug->len = lpfc_debugfs_scsistat_data(vport, debug->buffer, + LPFC_SCSISTAT_SIZE); + + debug->i_private = inode->i_private; + file->private_data = debug; + + rc = 0; +out: + return rc; +} + +static ssize_t +lpfc_debugfs_scsistat_write(struct file *file, const char __user *buf, + size_t nbytes, loff_t *ppos) +{ + struct lpfc_debug *debug = file->private_data; + struct lpfc_vport *vport = (struct lpfc_vport *)debug->i_private; + struct lpfc_hba *phba = vport->phba; + char mybuf[6] = {0}; + int i; + + /* Protect copy from user */ + if (!access_ok(buf, nbytes)) + return -EFAULT; + + if (copy_from_user(mybuf, buf, (nbytes >= sizeof(mybuf)) ? + (sizeof(mybuf) - 1) : nbytes)) + return -EFAULT; + + if ((strncmp(&mybuf[0], "reset", strlen("reset")) == 0) || + (strncmp(&mybuf[0], "zero", strlen("zero")) == 0)) { + for (i = 0; i < phba->cfg_hdw_queue; i++) { + memset(&phba->sli4_hba.hdwq[i].scsi_cstat, 0, + sizeof(phba->sli4_hba.hdwq[i].scsi_cstat)); + } + } + + return nbytes; +} + +static int lpfc_debugfs_nvmektime_open(struct inode *inode, struct file *file) { struct lpfc_vport *vport = inode->i_private; @@ -2348,7 +2936,7 @@ lpfc_debugfs_cpucheck_open(struct inode *inode, struct file *file) } debug->len = lpfc_debugfs_cpucheck_data(vport, debug->buffer, - LPFC_NVMEKTIME_SIZE); + LPFC_CPUCHECK_SIZE); debug->i_private = inode->i_private; file->private_data = debug; @@ -2365,9 +2953,10 @@ lpfc_debugfs_cpucheck_write(struct file *file, const char __user *buf, struct lpfc_debug *debug = file->private_data; struct lpfc_vport *vport = (struct lpfc_vport *)debug->i_private; struct lpfc_hba *phba = vport->phba; + struct lpfc_sli4_hdw_queue *qp; char mybuf[64]; char *pbuf; - int i; + int i, j; if (nbytes > 64) nbytes = 64; @@ -2382,8 +2971,18 @@ lpfc_debugfs_cpucheck_write(struct file *file, const char __user *buf, if (phba->nvmet_support) phba->cpucheck_on |= LPFC_CHECK_NVMET_IO; else + phba->cpucheck_on |= (LPFC_CHECK_NVME_IO | + LPFC_CHECK_SCSI_IO); + return strlen(pbuf); + } else if ((strncmp(pbuf, "nvme_on", sizeof("nvme_on") - 1) == 0)) { + if (phba->nvmet_support) + phba->cpucheck_on |= LPFC_CHECK_NVMET_IO; + else phba->cpucheck_on |= LPFC_CHECK_NVME_IO; return strlen(pbuf); + } else if ((strncmp(pbuf, "scsi_on", sizeof("scsi_on") - 1) == 0)) { + phba->cpucheck_on |= LPFC_CHECK_SCSI_IO; + return strlen(pbuf); } else if ((strncmp(pbuf, "rcv", sizeof("rcv") - 1) == 0)) { if (phba->nvmet_support) @@ -2397,13 +2996,14 @@ lpfc_debugfs_cpucheck_write(struct file *file, const char __user *buf, return strlen(pbuf); } else if ((strncmp(pbuf, "zero", sizeof("zero") - 1) == 0)) { - for (i = 0; i < phba->sli4_hba.num_present_cpu; i++) { - if (i >= LPFC_CHECK_CPU_CNT) - break; - phba->cpucheck_rcv_io[i] = 0; - phba->cpucheck_xmt_io[i] = 0; - phba->cpucheck_cmpl_io[i] = 0; - phba->cpucheck_ccmpl_io[i] = 0; + for (i = 0; i < phba->cfg_hdw_queue; i++) { + qp = &phba->sli4_hba.hdwq[i]; + + for (j = 0; j < LPFC_CHECK_CPU_CNT; j++) { + qp->cpucheck_rcv_io[j] = 0; + qp->cpucheck_xmt_io[j] = 0; + qp->cpucheck_cmpl_io[j] = 0; + } } return strlen(pbuf); } @@ -3166,10 +3766,10 @@ __lpfc_idiag_print_wq(struct lpfc_queue *qp, char *wqtype, (unsigned long long)qp->q_cnt_4); len += snprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len, "\t\tWQID[%02d], QE-CNT[%04d], QE-SZ[%04d], " - "HST-IDX[%04d], PRT-IDX[%04d], PST[%03d]", + "HST-IDX[%04d], PRT-IDX[%04d], NTFI[%03d]", qp->queue_id, qp->entry_count, qp->entry_size, qp->host_index, - qp->hba_index, qp->entry_repost); + qp->hba_index, qp->notify_interval); len += snprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len, "\n"); return len; @@ -3182,21 +3782,23 @@ lpfc_idiag_wqs_for_cq(struct lpfc_hba *phba, char *wqtype, char *pbuffer, struct lpfc_queue *qp; int qidx; - for (qidx = 0; qidx < phba->cfg_fcp_io_channel; qidx++) { - qp = phba->sli4_hba.fcp_wq[qidx]; + for (qidx = 0; qidx < phba->cfg_hdw_queue; qidx++) { + qp = phba->sli4_hba.hdwq[qidx].fcp_wq; if (qp->assoc_qid != cq_id) continue; *len = __lpfc_idiag_print_wq(qp, wqtype, pbuffer, *len); if (*len >= max_cnt) return 1; } - for (qidx = 0; qidx < phba->cfg_nvme_io_channel; qidx++) { - qp = phba->sli4_hba.nvme_wq[qidx]; - if (qp->assoc_qid != cq_id) - continue; - *len = __lpfc_idiag_print_wq(qp, wqtype, pbuffer, *len); - if (*len >= max_cnt) - return 1; + if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { + for (qidx = 0; qidx < phba->cfg_hdw_queue; qidx++) { + qp = phba->sli4_hba.hdwq[qidx].nvme_wq; + if (qp->assoc_qid != cq_id) + continue; + *len = __lpfc_idiag_print_wq(qp, wqtype, pbuffer, *len); + if (*len >= max_cnt) + return 1; + } } return 0; } @@ -3217,10 +3819,10 @@ __lpfc_idiag_print_cq(struct lpfc_queue *qp, char *cqtype, qp->q_cnt_3, (unsigned long long)qp->q_cnt_4); len += snprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len, "\tCQID[%02d], QE-CNT[%04d], QE-SZ[%04d], " - "HST-IDX[%04d], PRT-IDX[%04d], PST[%03d]", + "HST-IDX[%04d], NTFI[%03d], PLMT[%03d]", qp->queue_id, qp->entry_count, qp->entry_size, qp->host_index, - qp->hba_index, qp->entry_repost); + qp->notify_interval, qp->max_proc_limit); len += snprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len, "\n"); @@ -3243,15 +3845,15 @@ __lpfc_idiag_print_rqpair(struct lpfc_queue *qp, struct lpfc_queue *datqp, qp->q_cnt_3, (unsigned long long)qp->q_cnt_4); len += snprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len, "\t\tHQID[%02d], QE-CNT[%04d], QE-SZ[%04d], " - "HST-IDX[%04d], PRT-IDX[%04d], PST[%03d]\n", + "HST-IDX[%04d], PRT-IDX[%04d], NTFI[%03d]\n", qp->queue_id, qp->entry_count, qp->entry_size, - qp->host_index, qp->hba_index, qp->entry_repost); + qp->host_index, qp->hba_index, qp->notify_interval); len += snprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len, "\t\tDQID[%02d], QE-CNT[%04d], QE-SZ[%04d], " - "HST-IDX[%04d], PRT-IDX[%04d], PST[%03d]\n", + "HST-IDX[%04d], PRT-IDX[%04d], NTFI[%03d]\n", datqp->queue_id, datqp->entry_count, datqp->entry_size, datqp->host_index, - datqp->hba_index, datqp->entry_repost); + datqp->hba_index, datqp->notify_interval); return len; } @@ -3260,31 +3862,25 @@ lpfc_idiag_cqs_for_eq(struct lpfc_hba *phba, char *pbuffer, int *len, int max_cnt, int eqidx, int eq_id) { struct lpfc_queue *qp; - int qidx, rc; + int rc; - for (qidx = 0; qidx < phba->cfg_fcp_io_channel; qidx++) { - qp = phba->sli4_hba.fcp_cq[qidx]; - if (qp->assoc_qid != eq_id) - continue; + qp = phba->sli4_hba.hdwq[eqidx].fcp_cq; - *len = __lpfc_idiag_print_cq(qp, "FCP", pbuffer, *len); + *len = __lpfc_idiag_print_cq(qp, "FCP", pbuffer, *len); - /* Reset max counter */ - qp->CQ_max_cqe = 0; + /* Reset max counter */ + qp->CQ_max_cqe = 0; - if (*len >= max_cnt) - return 1; + if (*len >= max_cnt) + return 1; - rc = lpfc_idiag_wqs_for_cq(phba, "FCP", pbuffer, len, - max_cnt, qp->queue_id); - if (rc) - return 1; - } + rc = lpfc_idiag_wqs_for_cq(phba, "FCP", pbuffer, len, + max_cnt, qp->queue_id); + if (rc) + return 1; - for (qidx = 0; qidx < phba->cfg_nvme_io_channel; qidx++) { - qp = phba->sli4_hba.nvme_cq[qidx]; - if (qp->assoc_qid != eq_id) - continue; + if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { + qp = phba->sli4_hba.hdwq[eqidx].nvme_cq; *len = __lpfc_idiag_print_cq(qp, "NVME", pbuffer, *len); @@ -3295,7 +3891,7 @@ lpfc_idiag_cqs_for_eq(struct lpfc_hba *phba, char *pbuffer, return 1; rc = lpfc_idiag_wqs_for_cq(phba, "NVME", pbuffer, len, - max_cnt, qp->queue_id); + max_cnt, qp->queue_id); if (rc) return 1; } @@ -3338,9 +3934,10 @@ __lpfc_idiag_print_eq(struct lpfc_queue *qp, char *eqtype, (unsigned long long)qp->q_cnt_4, qp->q_mode); len += snprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len, "EQID[%02d], QE-CNT[%04d], QE-SZ[%04d], " - "HST-IDX[%04d], PRT-IDX[%04d], PST[%03d]", + "HST-IDX[%04d], NTFI[%03d], PLMT[%03d], AFFIN[%03d]", qp->queue_id, qp->entry_count, qp->entry_size, - qp->host_index, qp->hba_index, qp->entry_repost); + qp->host_index, qp->notify_interval, + qp->max_proc_limit, qp->chann); len += snprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len, "\n"); return len; @@ -3387,24 +3984,19 @@ lpfc_idiag_queinfo_read(struct file *file, char __user *buf, size_t nbytes, spin_lock_irq(&phba->hbalock); /* Fast-path event queue */ - if (phba->sli4_hba.hba_eq && phba->io_channel_irqs) { + if (phba->sli4_hba.hdwq && phba->cfg_hdw_queue) { x = phba->lpfc_idiag_last_eq; - if (phba->cfg_fof && (x >= phba->io_channel_irqs)) { - phba->lpfc_idiag_last_eq = 0; - goto fof; - } phba->lpfc_idiag_last_eq++; - if (phba->lpfc_idiag_last_eq >= phba->io_channel_irqs) - if (phba->cfg_fof == 0) - phba->lpfc_idiag_last_eq = 0; + if (phba->lpfc_idiag_last_eq >= phba->cfg_hdw_queue) + phba->lpfc_idiag_last_eq = 0; len += snprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len, - "EQ %d out of %d HBA EQs\n", - x, phba->io_channel_irqs); + "HDWQ %d out of %d HBA HDWQs\n", + x, phba->cfg_hdw_queue); /* Fast-path EQ */ - qp = phba->sli4_hba.hba_eq[x]; + qp = phba->sli4_hba.hdwq[x].hba_eq; if (!qp) goto out; @@ -3479,35 +4071,6 @@ lpfc_idiag_queinfo_read(struct file *file, char __user *buf, size_t nbytes, goto out; } -fof: - if (phba->cfg_fof) { - /* FOF EQ */ - qp = phba->sli4_hba.fof_eq; - len = __lpfc_idiag_print_eq(qp, "FOF", pbuffer, len); - - /* Reset max counter */ - if (qp) - qp->EQ_max_eqe = 0; - - if (len >= max_cnt) - goto too_big; - - /* OAS CQ */ - qp = phba->sli4_hba.oas_cq; - len = __lpfc_idiag_print_cq(qp, "OAS", pbuffer, len); - /* Reset max counter */ - if (qp) - qp->CQ_max_cqe = 0; - if (len >= max_cnt) - goto too_big; - - /* OAS WQ */ - qp = phba->sli4_hba.oas_wq; - len = __lpfc_idiag_print_wq(qp, "OAS", pbuffer, len); - if (len >= max_cnt) - goto too_big; - } - spin_unlock_irq(&phba->hbalock); return simple_read_from_buffer(buf, nbytes, ppos, pbuffer, len); @@ -3725,9 +4288,9 @@ lpfc_idiag_queacc_write(struct file *file, const char __user *buf, switch (quetp) { case LPFC_IDIAG_EQ: /* HBA event queue */ - if (phba->sli4_hba.hba_eq) { - for (qidx = 0; qidx < phba->io_channel_irqs; qidx++) { - qp = phba->sli4_hba.hba_eq[qidx]; + if (phba->sli4_hba.hdwq) { + for (qidx = 0; qidx < phba->cfg_hdw_queue; qidx++) { + qp = phba->sli4_hba.hdwq[qidx].hba_eq; if (qp && qp->queue_id == queid) { /* Sanity check */ rc = lpfc_idiag_que_param_check(qp, @@ -3776,10 +4339,10 @@ lpfc_idiag_queacc_write(struct file *file, const char __user *buf, goto pass_check; } /* FCP complete queue */ - if (phba->sli4_hba.fcp_cq) { - for (qidx = 0; qidx < phba->cfg_fcp_io_channel; + if (phba->sli4_hba.hdwq) { + for (qidx = 0; qidx < phba->cfg_hdw_queue; qidx++) { - qp = phba->sli4_hba.fcp_cq[qidx]; + qp = phba->sli4_hba.hdwq[qidx].fcp_cq; if (qp && qp->queue_id == queid) { /* Sanity check */ rc = lpfc_idiag_que_param_check( @@ -3792,23 +4355,20 @@ lpfc_idiag_queacc_write(struct file *file, const char __user *buf, } } /* NVME complete queue */ - if (phba->sli4_hba.nvme_cq) { + if (phba->sli4_hba.hdwq) { qidx = 0; do { - if (phba->sli4_hba.nvme_cq[qidx] && - phba->sli4_hba.nvme_cq[qidx]->queue_id == - queid) { + qp = phba->sli4_hba.hdwq[qidx].nvme_cq; + if (qp && qp->queue_id == queid) { /* Sanity check */ rc = lpfc_idiag_que_param_check( - phba->sli4_hba.nvme_cq[qidx], - index, count); + qp, index, count); if (rc) goto error_out; - idiag.ptr_private = - phba->sli4_hba.nvme_cq[qidx]; + idiag.ptr_private = qp; goto pass_check; } - } while (++qidx < phba->cfg_nvme_io_channel); + } while (++qidx < phba->cfg_hdw_queue); } goto error_out; break; @@ -3849,11 +4409,11 @@ lpfc_idiag_queacc_write(struct file *file, const char __user *buf, idiag.ptr_private = phba->sli4_hba.nvmels_wq; goto pass_check; } - /* FCP work queue */ - if (phba->sli4_hba.fcp_wq) { - for (qidx = 0; qidx < phba->cfg_fcp_io_channel; - qidx++) { - qp = phba->sli4_hba.fcp_wq[qidx]; + + if (phba->sli4_hba.hdwq) { + /* FCP/SCSI work queue */ + for (qidx = 0; qidx < phba->cfg_hdw_queue; qidx++) { + qp = phba->sli4_hba.hdwq[qidx].fcp_wq; if (qp && qp->queue_id == queid) { /* Sanity check */ rc = lpfc_idiag_que_param_check( @@ -3864,12 +4424,9 @@ lpfc_idiag_queacc_write(struct file *file, const char __user *buf, goto pass_check; } } - } - /* NVME work queue */ - if (phba->sli4_hba.nvme_wq) { - for (qidx = 0; qidx < phba->cfg_nvme_io_channel; - qidx++) { - qp = phba->sli4_hba.nvme_wq[qidx]; + /* NVME work queue */ + for (qidx = 0; qidx < phba->cfg_hdw_queue; qidx++) { + qp = phba->sli4_hba.hdwq[qidx].nvme_wq; if (qp && qp->queue_id == queid) { /* Sanity check */ rc = lpfc_idiag_que_param_check( @@ -3882,26 +4439,6 @@ lpfc_idiag_queacc_write(struct file *file, const char __user *buf, } } - /* NVME work queues */ - if (phba->sli4_hba.nvme_wq) { - for (qidx = 0; qidx < phba->cfg_nvme_io_channel; - qidx++) { - if (!phba->sli4_hba.nvme_wq[qidx]) - continue; - if (phba->sli4_hba.nvme_wq[qidx]->queue_id == - queid) { - /* Sanity check */ - rc = lpfc_idiag_que_param_check( - phba->sli4_hba.nvme_wq[qidx], - index, count); - if (rc) - goto error_out; - idiag.ptr_private = - phba->sli4_hba.nvme_wq[qidx]; - goto pass_check; - } - } - } goto error_out; break; case LPFC_IDIAG_RQ: @@ -4866,6 +5403,16 @@ static const struct file_operations lpfc_debugfs_op_nodelist = { .release = lpfc_debugfs_release, }; +#undef lpfc_debugfs_op_multixripools +static const struct file_operations lpfc_debugfs_op_multixripools = { + .owner = THIS_MODULE, + .open = lpfc_debugfs_multixripools_open, + .llseek = lpfc_debugfs_lseek, + .read = lpfc_debugfs_read, + .write = lpfc_debugfs_multixripools_write, + .release = lpfc_debugfs_release, +}; + #undef lpfc_debugfs_op_hbqinfo static const struct file_operations lpfc_debugfs_op_hbqinfo = { .owner = THIS_MODULE, @@ -4875,6 +5422,18 @@ static const struct file_operations lpfc_debugfs_op_hbqinfo = { .release = lpfc_debugfs_release, }; +#ifdef LPFC_HDWQ_LOCK_STAT +#undef lpfc_debugfs_op_lockstat +static const struct file_operations lpfc_debugfs_op_lockstat = { + .owner = THIS_MODULE, + .open = lpfc_debugfs_lockstat_open, + .llseek = lpfc_debugfs_lseek, + .read = lpfc_debugfs_read, + .write = lpfc_debugfs_lockstat_write, + .release = lpfc_debugfs_release, +}; +#endif + #undef lpfc_debugfs_op_dumpHBASlim static const struct file_operations lpfc_debugfs_op_dumpHBASlim = { .owner = THIS_MODULE, @@ -4903,6 +5462,16 @@ static const struct file_operations lpfc_debugfs_op_nvmestat = { .release = lpfc_debugfs_release, }; +#undef lpfc_debugfs_op_scsistat +static const struct file_operations lpfc_debugfs_op_scsistat = { + .owner = THIS_MODULE, + .open = lpfc_debugfs_scsistat_open, + .llseek = lpfc_debugfs_lseek, + .read = lpfc_debugfs_read, + .write = lpfc_debugfs_scsistat_write, + .release = lpfc_debugfs_release, +}; + #undef lpfc_debugfs_op_nvmektime static const struct file_operations lpfc_debugfs_op_nvmektime = { .owner = THIS_MODULE, @@ -5280,11 +5849,6 @@ lpfc_debugfs_initialize(struct lpfc_vport *vport) if (!lpfc_debugfs_root) { lpfc_debugfs_root = debugfs_create_dir("lpfc", NULL); atomic_set(&lpfc_debugfs_hba_count, 0); - if (!lpfc_debugfs_root) { - lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT, - "0408 Cannot create debugfs root\n"); - goto debug_failed; - } } if (!lpfc_debugfs_start_time) lpfc_debugfs_start_time = jiffies; @@ -5295,25 +5859,42 @@ lpfc_debugfs_initialize(struct lpfc_vport *vport) pport_setup = true; phba->hba_debugfs_root = debugfs_create_dir(name, lpfc_debugfs_root); - if (!phba->hba_debugfs_root) { + atomic_inc(&lpfc_debugfs_hba_count); + atomic_set(&phba->debugfs_vport_count, 0); + + /* Multi-XRI pools */ + snprintf(name, sizeof(name), "multixripools"); + phba->debug_multixri_pools = + debugfs_create_file(name, S_IFREG | 0644, + phba->hba_debugfs_root, + phba, + &lpfc_debugfs_op_multixripools); + if (!phba->debug_multixri_pools) { lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT, - "0412 Cannot create debugfs hba\n"); + "0527 Cannot create debugfs multixripools\n"); goto debug_failed; } - atomic_inc(&lpfc_debugfs_hba_count); - atomic_set(&phba->debugfs_vport_count, 0); /* Setup hbqinfo */ snprintf(name, sizeof(name), "hbqinfo"); phba->debug_hbqinfo = - debugfs_create_file(name, S_IFREG|S_IRUGO|S_IWUSR, - phba->hba_debugfs_root, - phba, &lpfc_debugfs_op_hbqinfo); - if (!phba->debug_hbqinfo) { + debugfs_create_file(name, S_IFREG | 0644, + phba->hba_debugfs_root, + phba, &lpfc_debugfs_op_hbqinfo); + +#ifdef LPFC_HDWQ_LOCK_STAT + /* Setup lockstat */ + snprintf(name, sizeof(name), "lockstat"); + phba->debug_lockstat = + debugfs_create_file(name, S_IFREG | 0644, + phba->hba_debugfs_root, + phba, &lpfc_debugfs_op_lockstat); + if (!phba->debug_lockstat) { lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT, - "0411 Cannot create debugfs hbqinfo\n"); + "0913 Cant create debugfs lockstat\n"); goto debug_failed; } +#endif /* Setup dumpHBASlim */ if (phba->sli_rev < LPFC_SLI_REV4) { @@ -5323,12 +5904,6 @@ lpfc_debugfs_initialize(struct lpfc_vport *vport) S_IFREG|S_IRUGO|S_IWUSR, phba->hba_debugfs_root, phba, &lpfc_debugfs_op_dumpHBASlim); - if (!phba->debug_dumpHBASlim) { - lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT, - "0413 Cannot create debugfs " - "dumpHBASlim\n"); - goto debug_failed; - } } else phba->debug_dumpHBASlim = NULL; @@ -5340,12 +5915,6 @@ lpfc_debugfs_initialize(struct lpfc_vport *vport) S_IFREG|S_IRUGO|S_IWUSR, phba->hba_debugfs_root, phba, &lpfc_debugfs_op_dumpHostSlim); - if (!phba->debug_dumpHostSlim) { - lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT, - "0414 Cannot create debugfs " - "dumpHostSlim\n"); - goto debug_failed; - } } else phba->debug_dumpHostSlim = NULL; @@ -5355,11 +5924,6 @@ lpfc_debugfs_initialize(struct lpfc_vport *vport) debugfs_create_file(name, S_IFREG|S_IRUGO|S_IWUSR, phba->hba_debugfs_root, phba, &lpfc_debugfs_op_dumpData); - if (!phba->debug_dumpData) { - lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT, - "0800 Cannot create debugfs dumpData\n"); - goto debug_failed; - } /* Setup dumpDif */ snprintf(name, sizeof(name), "dumpDif"); @@ -5367,11 +5931,6 @@ lpfc_debugfs_initialize(struct lpfc_vport *vport) debugfs_create_file(name, S_IFREG|S_IRUGO|S_IWUSR, phba->hba_debugfs_root, phba, &lpfc_debugfs_op_dumpDif); - if (!phba->debug_dumpDif) { - lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT, - "0801 Cannot create debugfs dumpDif\n"); - goto debug_failed; - } /* Setup DIF Error Injections */ snprintf(name, sizeof(name), "InjErrLBA"); @@ -5379,11 +5938,6 @@ lpfc_debugfs_initialize(struct lpfc_vport *vport) debugfs_create_file(name, S_IFREG|S_IRUGO|S_IWUSR, phba->hba_debugfs_root, phba, &lpfc_debugfs_op_dif_err); - if (!phba->debug_InjErrLBA) { - lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT, - "0807 Cannot create debugfs InjErrLBA\n"); - goto debug_failed; - } phba->lpfc_injerr_lba = LPFC_INJERR_LBA_OFF; snprintf(name, sizeof(name), "InjErrNPortID"); @@ -5391,88 +5945,48 @@ lpfc_debugfs_initialize(struct lpfc_vport *vport) debugfs_create_file(name, S_IFREG|S_IRUGO|S_IWUSR, phba->hba_debugfs_root, phba, &lpfc_debugfs_op_dif_err); - if (!phba->debug_InjErrNPortID) { - lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT, - "0809 Cannot create debugfs InjErrNPortID\n"); - goto debug_failed; - } snprintf(name, sizeof(name), "InjErrWWPN"); phba->debug_InjErrWWPN = debugfs_create_file(name, S_IFREG|S_IRUGO|S_IWUSR, phba->hba_debugfs_root, phba, &lpfc_debugfs_op_dif_err); - if (!phba->debug_InjErrWWPN) { - lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT, - "0810 Cannot create debugfs InjErrWWPN\n"); - goto debug_failed; - } snprintf(name, sizeof(name), "writeGuardInjErr"); phba->debug_writeGuard = debugfs_create_file(name, S_IFREG|S_IRUGO|S_IWUSR, phba->hba_debugfs_root, phba, &lpfc_debugfs_op_dif_err); - if (!phba->debug_writeGuard) { - lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT, - "0802 Cannot create debugfs writeGuard\n"); - goto debug_failed; - } snprintf(name, sizeof(name), "writeAppInjErr"); phba->debug_writeApp = debugfs_create_file(name, S_IFREG|S_IRUGO|S_IWUSR, phba->hba_debugfs_root, phba, &lpfc_debugfs_op_dif_err); - if (!phba->debug_writeApp) { - lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT, - "0803 Cannot create debugfs writeApp\n"); - goto debug_failed; - } snprintf(name, sizeof(name), "writeRefInjErr"); phba->debug_writeRef = debugfs_create_file(name, S_IFREG|S_IRUGO|S_IWUSR, phba->hba_debugfs_root, phba, &lpfc_debugfs_op_dif_err); - if (!phba->debug_writeRef) { - lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT, - "0804 Cannot create debugfs writeRef\n"); - goto debug_failed; - } snprintf(name, sizeof(name), "readGuardInjErr"); phba->debug_readGuard = debugfs_create_file(name, S_IFREG|S_IRUGO|S_IWUSR, phba->hba_debugfs_root, phba, &lpfc_debugfs_op_dif_err); - if (!phba->debug_readGuard) { - lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT, - "0808 Cannot create debugfs readGuard\n"); - goto debug_failed; - } snprintf(name, sizeof(name), "readAppInjErr"); phba->debug_readApp = debugfs_create_file(name, S_IFREG|S_IRUGO|S_IWUSR, phba->hba_debugfs_root, phba, &lpfc_debugfs_op_dif_err); - if (!phba->debug_readApp) { - lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT, - "0805 Cannot create debugfs readApp\n"); - goto debug_failed; - } snprintf(name, sizeof(name), "readRefInjErr"); phba->debug_readRef = debugfs_create_file(name, S_IFREG|S_IRUGO|S_IWUSR, phba->hba_debugfs_root, phba, &lpfc_debugfs_op_dif_err); - if (!phba->debug_readRef) { - lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT, - "0806 Cannot create debugfs readApp\n"); - goto debug_failed; - } /* Setup slow ring trace */ if (lpfc_debugfs_max_slow_ring_trc) { @@ -5496,12 +6010,6 @@ lpfc_debugfs_initialize(struct lpfc_vport *vport) debugfs_create_file(name, S_IFREG|S_IRUGO|S_IWUSR, phba->hba_debugfs_root, phba, &lpfc_debugfs_op_slow_ring_trc); - if (!phba->debug_slow_ring_trc) { - lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT, - "0415 Cannot create debugfs " - "slow_ring_trace\n"); - goto debug_failed; - } if (!phba->slow_ring_trc) { phba->slow_ring_trc = kmalloc( (sizeof(struct lpfc_debugfs_trc) * @@ -5524,11 +6032,6 @@ lpfc_debugfs_initialize(struct lpfc_vport *vport) debugfs_create_file(name, 0644, phba->hba_debugfs_root, phba, &lpfc_debugfs_op_nvmeio_trc); - if (!phba->debug_nvmeio_trc) { - lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT, - "0574 No create debugfs nvmeio_trc\n"); - goto debug_failed; - } atomic_set(&phba->nvmeio_trc_cnt, 0); if (lpfc_debugfs_max_nvmeio_trc) { @@ -5576,11 +6079,6 @@ nvmeio_off: if (!vport->vport_debugfs_root) { vport->vport_debugfs_root = debugfs_create_dir(name, phba->hba_debugfs_root); - if (!vport->vport_debugfs_root) { - lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT, - "0417 Can't create debugfs\n"); - goto debug_failed; - } atomic_inc(&phba->debugfs_vport_count); } @@ -5617,31 +6115,26 @@ nvmeio_off: debugfs_create_file(name, S_IFREG|S_IRUGO|S_IWUSR, vport->vport_debugfs_root, vport, &lpfc_debugfs_op_disc_trc); - if (!vport->debug_disc_trc) { - lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT, - "0419 Cannot create debugfs " - "discovery_trace\n"); - goto debug_failed; - } snprintf(name, sizeof(name), "nodelist"); vport->debug_nodelist = debugfs_create_file(name, S_IFREG|S_IRUGO|S_IWUSR, vport->vport_debugfs_root, vport, &lpfc_debugfs_op_nodelist); - if (!vport->debug_nodelist) { - lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT, - "2985 Can't create debugfs nodelist\n"); - goto debug_failed; - } snprintf(name, sizeof(name), "nvmestat"); vport->debug_nvmestat = debugfs_create_file(name, 0644, vport->vport_debugfs_root, vport, &lpfc_debugfs_op_nvmestat); - if (!vport->debug_nvmestat) { + + snprintf(name, sizeof(name), "scsistat"); + vport->debug_scsistat = + debugfs_create_file(name, 0644, + vport->vport_debugfs_root, + vport, &lpfc_debugfs_op_scsistat); + if (!vport->debug_scsistat) { lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT, - "0811 Cannot create debugfs nvmestat\n"); + "0914 Cannot create debugfs scsistat\n"); goto debug_failed; } @@ -5650,22 +6143,12 @@ nvmeio_off: debugfs_create_file(name, 0644, vport->vport_debugfs_root, vport, &lpfc_debugfs_op_nvmektime); - if (!vport->debug_nvmektime) { - lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT, - "0815 Cannot create debugfs nvmektime\n"); - goto debug_failed; - } snprintf(name, sizeof(name), "cpucheck"); vport->debug_cpucheck = debugfs_create_file(name, 0644, vport->vport_debugfs_root, vport, &lpfc_debugfs_op_cpucheck); - if (!vport->debug_cpucheck) { - lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT, - "0819 Cannot create debugfs cpucheck\n"); - goto debug_failed; - } /* * The following section is for additional directories/files for the @@ -5685,11 +6168,6 @@ nvmeio_off: if (!phba->idiag_root) { phba->idiag_root = debugfs_create_dir(name, phba->hba_debugfs_root); - if (!phba->idiag_root) { - lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT, - "2922 Can't create idiag debugfs\n"); - goto debug_failed; - } /* Initialize iDiag data structure */ memset(&idiag, 0, sizeof(idiag)); } @@ -5700,11 +6178,6 @@ nvmeio_off: phba->idiag_pci_cfg = debugfs_create_file(name, S_IFREG|S_IRUGO|S_IWUSR, phba->idiag_root, phba, &lpfc_idiag_op_pciCfg); - if (!phba->idiag_pci_cfg) { - lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT, - "2923 Can't create idiag debugfs\n"); - goto debug_failed; - } idiag.offset.last_rd = 0; } @@ -5714,11 +6187,6 @@ nvmeio_off: phba->idiag_bar_acc = debugfs_create_file(name, S_IFREG|S_IRUGO|S_IWUSR, phba->idiag_root, phba, &lpfc_idiag_op_barAcc); - if (!phba->idiag_bar_acc) { - lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT, - "3056 Can't create idiag debugfs\n"); - goto debug_failed; - } idiag.offset.last_rd = 0; } @@ -5728,11 +6196,6 @@ nvmeio_off: phba->idiag_que_info = debugfs_create_file(name, S_IFREG|S_IRUGO, phba->idiag_root, phba, &lpfc_idiag_op_queInfo); - if (!phba->idiag_que_info) { - lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT, - "2924 Can't create idiag debugfs\n"); - goto debug_failed; - } } /* iDiag access PCI function queue */ @@ -5741,11 +6204,6 @@ nvmeio_off: phba->idiag_que_acc = debugfs_create_file(name, S_IFREG|S_IRUGO|S_IWUSR, phba->idiag_root, phba, &lpfc_idiag_op_queAcc); - if (!phba->idiag_que_acc) { - lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT, - "2926 Can't create idiag debugfs\n"); - goto debug_failed; - } } /* iDiag access PCI function doorbell registers */ @@ -5754,11 +6212,6 @@ nvmeio_off: phba->idiag_drb_acc = debugfs_create_file(name, S_IFREG|S_IRUGO|S_IWUSR, phba->idiag_root, phba, &lpfc_idiag_op_drbAcc); - if (!phba->idiag_drb_acc) { - lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT, - "2927 Can't create idiag debugfs\n"); - goto debug_failed; - } } /* iDiag access PCI function control registers */ @@ -5767,11 +6220,6 @@ nvmeio_off: phba->idiag_ctl_acc = debugfs_create_file(name, S_IFREG|S_IRUGO|S_IWUSR, phba->idiag_root, phba, &lpfc_idiag_op_ctlAcc); - if (!phba->idiag_ctl_acc) { - lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT, - "2981 Can't create idiag debugfs\n"); - goto debug_failed; - } } /* iDiag access mbox commands */ @@ -5780,11 +6228,6 @@ nvmeio_off: phba->idiag_mbx_acc = debugfs_create_file(name, S_IFREG|S_IRUGO|S_IWUSR, phba->idiag_root, phba, &lpfc_idiag_op_mbxAcc); - if (!phba->idiag_mbx_acc) { - lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT, - "2980 Can't create idiag debugfs\n"); - goto debug_failed; - } } /* iDiag extents access commands */ @@ -5796,12 +6239,6 @@ nvmeio_off: S_IFREG|S_IRUGO|S_IWUSR, phba->idiag_root, phba, &lpfc_idiag_op_extAcc); - if (!phba->idiag_ext_acc) { - lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT, - "2986 Cant create " - "idiag debugfs\n"); - goto debug_failed; - } } } @@ -5839,6 +6276,9 @@ lpfc_debugfs_terminate(struct lpfc_vport *vport) debugfs_remove(vport->debug_nvmestat); /* nvmestat */ vport->debug_nvmestat = NULL; + debugfs_remove(vport->debug_scsistat); /* scsistat */ + vport->debug_scsistat = NULL; + debugfs_remove(vport->debug_nvmektime); /* nvmektime */ vport->debug_nvmektime = NULL; @@ -5853,9 +6293,16 @@ lpfc_debugfs_terminate(struct lpfc_vport *vport) if (atomic_read(&phba->debugfs_vport_count) == 0) { + debugfs_remove(phba->debug_multixri_pools); /* multixripools*/ + phba->debug_multixri_pools = NULL; + debugfs_remove(phba->debug_hbqinfo); /* hbqinfo */ phba->debug_hbqinfo = NULL; +#ifdef LPFC_HDWQ_LOCK_STAT + debugfs_remove(phba->debug_lockstat); /* lockstat */ + phba->debug_lockstat = NULL; +#endif debugfs_remove(phba->debug_dumpHBASlim); /* HBASlim */ phba->debug_dumpHBASlim = NULL; @@ -5988,11 +6435,13 @@ lpfc_debug_dump_all_queues(struct lpfc_hba *phba) lpfc_debug_dump_wq(phba, DUMP_ELS, 0); lpfc_debug_dump_wq(phba, DUMP_NVMELS, 0); - for (idx = 0; idx < phba->cfg_fcp_io_channel; idx++) + for (idx = 0; idx < phba->cfg_hdw_queue; idx++) lpfc_debug_dump_wq(phba, DUMP_FCP, idx); - for (idx = 0; idx < phba->cfg_nvme_io_channel; idx++) - lpfc_debug_dump_wq(phba, DUMP_NVME, idx); + if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { + for (idx = 0; idx < phba->cfg_hdw_queue; idx++) + lpfc_debug_dump_wq(phba, DUMP_NVME, idx); + } lpfc_debug_dump_hdr_rq(phba); lpfc_debug_dump_dat_rq(phba); @@ -6003,15 +6452,17 @@ lpfc_debug_dump_all_queues(struct lpfc_hba *phba) lpfc_debug_dump_cq(phba, DUMP_ELS, 0); lpfc_debug_dump_cq(phba, DUMP_NVMELS, 0); - for (idx = 0; idx < phba->cfg_fcp_io_channel; idx++) + for (idx = 0; idx < phba->cfg_hdw_queue; idx++) lpfc_debug_dump_cq(phba, DUMP_FCP, idx); - for (idx = 0; idx < phba->cfg_nvme_io_channel; idx++) - lpfc_debug_dump_cq(phba, DUMP_NVME, idx); + if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { + for (idx = 0; idx < phba->cfg_hdw_queue; idx++) + lpfc_debug_dump_cq(phba, DUMP_NVME, idx); + } /* * Dump Event Queues (EQs) */ - for (idx = 0; idx < phba->io_channel_irqs; idx++) + for (idx = 0; idx < phba->cfg_hdw_queue; idx++) lpfc_debug_dump_hba_eq(phba, idx); } diff --git a/drivers/scsi/lpfc/lpfc_debugfs.h b/drivers/scsi/lpfc/lpfc_debugfs.h index 30efc7bf91bd..93ab7dfb8ee0 100644 --- a/drivers/scsi/lpfc/lpfc_debugfs.h +++ b/drivers/scsi/lpfc/lpfc_debugfs.h @@ -1,7 +1,7 @@ /******************************************************************* * This file is part of the Emulex Linux Device Driver for * * Fibre Channel Host Bus Adapters. * - * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term * + * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term * * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * * Copyright (C) 2007-2011 Emulex. All rights reserved. * * EMULEX and SLI are trademarks of Emulex. * @@ -50,6 +50,9 @@ #define LPFC_CPUCHECK_SIZE 8192 #define LPFC_NVMEIO_TRC_SIZE 8192 +/* scsistat output buffer size */ +#define LPFC_SCSISTAT_SIZE 8192 + #define LPFC_DEBUG_OUT_LINE_SZ 80 /* @@ -284,6 +287,9 @@ struct lpfc_idiag { #endif +/* multixripool output buffer size */ +#define LPFC_DUMP_MULTIXRIPOOL_SIZE 8192 + enum { DUMP_FCP, DUMP_NVME, @@ -410,10 +416,10 @@ lpfc_debug_dump_wq(struct lpfc_hba *phba, int qtype, int wqidx) char *qtypestr; if (qtype == DUMP_FCP) { - wq = phba->sli4_hba.fcp_wq[wqidx]; + wq = phba->sli4_hba.hdwq[wqidx].fcp_wq; qtypestr = "FCP"; } else if (qtype == DUMP_NVME) { - wq = phba->sli4_hba.nvme_wq[wqidx]; + wq = phba->sli4_hba.hdwq[wqidx].nvme_wq; qtypestr = "NVME"; } else if (qtype == DUMP_MBX) { wq = phba->sli4_hba.mbx_wq; @@ -454,14 +460,15 @@ lpfc_debug_dump_cq(struct lpfc_hba *phba, int qtype, int wqidx) int eqidx; /* fcp/nvme wq and cq are 1:1, thus same indexes */ + eq = NULL; if (qtype == DUMP_FCP) { - wq = phba->sli4_hba.fcp_wq[wqidx]; - cq = phba->sli4_hba.fcp_cq[wqidx]; + wq = phba->sli4_hba.hdwq[wqidx].fcp_wq; + cq = phba->sli4_hba.hdwq[wqidx].fcp_cq; qtypestr = "FCP"; } else if (qtype == DUMP_NVME) { - wq = phba->sli4_hba.nvme_wq[wqidx]; - cq = phba->sli4_hba.nvme_cq[wqidx]; + wq = phba->sli4_hba.hdwq[wqidx].nvme_wq; + cq = phba->sli4_hba.hdwq[wqidx].nvme_cq; qtypestr = "NVME"; } else if (qtype == DUMP_MBX) { wq = phba->sli4_hba.mbx_wq; @@ -478,17 +485,17 @@ lpfc_debug_dump_cq(struct lpfc_hba *phba, int qtype, int wqidx) } else return; - for (eqidx = 0; eqidx < phba->io_channel_irqs; eqidx++) { - if (cq->assoc_qid == phba->sli4_hba.hba_eq[eqidx]->queue_id) + for (eqidx = 0; eqidx < phba->cfg_hdw_queue; eqidx++) { + eq = phba->sli4_hba.hdwq[eqidx].hba_eq; + if (cq->assoc_qid == eq->queue_id) break; } - if (eqidx == phba->io_channel_irqs) { + if (eqidx == phba->cfg_hdw_queue) { pr_err("Couldn't find EQ for CQ. Using EQ[0]\n"); eqidx = 0; + eq = phba->sli4_hba.hdwq[0].hba_eq; } - eq = phba->sli4_hba.hba_eq[eqidx]; - if (qtype == DUMP_FCP || qtype == DUMP_NVME) pr_err("%s CQ: WQ[Idx:%d|Qid%d]->CQ[Idx%d|Qid%d]" "->EQ[Idx:%d|Qid:%d]:\n", @@ -516,7 +523,7 @@ lpfc_debug_dump_hba_eq(struct lpfc_hba *phba, int qidx) { struct lpfc_queue *qp; - qp = phba->sli4_hba.hba_eq[qidx]; + qp = phba->sli4_hba.hdwq[qidx].hba_eq; pr_err("EQ[Idx:%d|Qid:%d]\n", qidx, qp->queue_id); @@ -564,21 +571,21 @@ lpfc_debug_dump_wq_by_id(struct lpfc_hba *phba, int qid) { int wq_idx; - for (wq_idx = 0; wq_idx < phba->cfg_fcp_io_channel; wq_idx++) - if (phba->sli4_hba.fcp_wq[wq_idx]->queue_id == qid) + for (wq_idx = 0; wq_idx < phba->cfg_hdw_queue; wq_idx++) + if (phba->sli4_hba.hdwq[wq_idx].fcp_wq->queue_id == qid) break; - if (wq_idx < phba->cfg_fcp_io_channel) { + if (wq_idx < phba->cfg_hdw_queue) { pr_err("FCP WQ[Idx:%d|Qid:%d]\n", wq_idx, qid); - lpfc_debug_dump_q(phba->sli4_hba.fcp_wq[wq_idx]); + lpfc_debug_dump_q(phba->sli4_hba.hdwq[wq_idx].fcp_wq); return; } - for (wq_idx = 0; wq_idx < phba->cfg_nvme_io_channel; wq_idx++) - if (phba->sli4_hba.nvme_wq[wq_idx]->queue_id == qid) + for (wq_idx = 0; wq_idx < phba->cfg_hdw_queue; wq_idx++) + if (phba->sli4_hba.hdwq[wq_idx].nvme_wq->queue_id == qid) break; - if (wq_idx < phba->cfg_nvme_io_channel) { + if (wq_idx < phba->cfg_hdw_queue) { pr_err("NVME WQ[Idx:%d|Qid:%d]\n", wq_idx, qid); - lpfc_debug_dump_q(phba->sli4_hba.nvme_wq[wq_idx]); + lpfc_debug_dump_q(phba->sli4_hba.hdwq[wq_idx].nvme_wq); return; } @@ -646,23 +653,23 @@ lpfc_debug_dump_cq_by_id(struct lpfc_hba *phba, int qid) { int cq_idx; - for (cq_idx = 0; cq_idx < phba->cfg_fcp_io_channel; cq_idx++) - if (phba->sli4_hba.fcp_cq[cq_idx]->queue_id == qid) + for (cq_idx = 0; cq_idx < phba->cfg_hdw_queue; cq_idx++) + if (phba->sli4_hba.hdwq[cq_idx].fcp_cq->queue_id == qid) break; - if (cq_idx < phba->cfg_fcp_io_channel) { + if (cq_idx < phba->cfg_hdw_queue) { pr_err("FCP CQ[Idx:%d|Qid:%d]\n", cq_idx, qid); - lpfc_debug_dump_q(phba->sli4_hba.fcp_cq[cq_idx]); + lpfc_debug_dump_q(phba->sli4_hba.hdwq[cq_idx].fcp_cq); return; } - for (cq_idx = 0; cq_idx < phba->cfg_nvme_io_channel; cq_idx++) - if (phba->sli4_hba.nvme_cq[cq_idx]->queue_id == qid) + for (cq_idx = 0; cq_idx < phba->cfg_hdw_queue; cq_idx++) + if (phba->sli4_hba.hdwq[cq_idx].nvme_cq->queue_id == qid) break; - if (cq_idx < phba->cfg_nvme_io_channel) { + if (cq_idx < phba->cfg_hdw_queue) { pr_err("NVME CQ[Idx:%d|Qid:%d]\n", cq_idx, qid); - lpfc_debug_dump_q(phba->sli4_hba.nvme_cq[cq_idx]); + lpfc_debug_dump_q(phba->sli4_hba.hdwq[cq_idx].nvme_cq); return; } @@ -697,13 +704,13 @@ lpfc_debug_dump_eq_by_id(struct lpfc_hba *phba, int qid) { int eq_idx; - for (eq_idx = 0; eq_idx < phba->io_channel_irqs; eq_idx++) - if (phba->sli4_hba.hba_eq[eq_idx]->queue_id == qid) + for (eq_idx = 0; eq_idx < phba->cfg_hdw_queue; eq_idx++) + if (phba->sli4_hba.hdwq[eq_idx].hba_eq->queue_id == qid) break; - if (eq_idx < phba->io_channel_irqs) { + if (eq_idx < phba->cfg_hdw_queue) { printk(KERN_ERR "FCP EQ[Idx:%d|Qid:%d]\n", eq_idx, qid); - lpfc_debug_dump_q(phba->sli4_hba.hba_eq[eq_idx]); + lpfc_debug_dump_q(phba->sli4_hba.hdwq[eq_idx].hba_eq); return; } } diff --git a/drivers/scsi/lpfc/lpfc_els.c b/drivers/scsi/lpfc/lpfc_els.c index b3a4789468c3..fc077cb87900 100644 --- a/drivers/scsi/lpfc/lpfc_els.c +++ b/drivers/scsi/lpfc/lpfc_els.c @@ -1,7 +1,7 @@ /******************************************************************* * This file is part of the Emulex Linux Device Driver for * * Fibre Channel Host Bus Adapters. * - * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term * + * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term * * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * * Copyright (C) 2004-2016 Emulex. All rights reserved. * * EMULEX and SLI are trademarks of Emulex. * @@ -2827,8 +2827,8 @@ out: !(vport->fc_flag & FC_PT2PT_PLOGI)) { phba->pport->fc_myDID = 0; - if ((phba->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) || - (phba->cfg_enable_fc4_type == LPFC_ENABLE_NVME)) { + if ((vport->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) || + (vport->cfg_enable_fc4_type == LPFC_ENABLE_NVME)) { if (phba->nvmet_support) lpfc_nvmet_update_targetport(phba); else diff --git a/drivers/scsi/lpfc/lpfc_hbadisc.c b/drivers/scsi/lpfc/lpfc_hbadisc.c index b183b882d506..aa4961a2caf8 100644 --- a/drivers/scsi/lpfc/lpfc_hbadisc.c +++ b/drivers/scsi/lpfc/lpfc_hbadisc.c @@ -1,7 +1,7 @@ /******************************************************************* * This file is part of the Emulex Linux Device Driver for * * Fibre Channel Host Bus Adapters. * - * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term * + * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term * * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * * Copyright (C) 2004-2016 Emulex. All rights reserved. * * EMULEX and SLI are trademarks of Emulex. * @@ -638,8 +638,6 @@ lpfc_work_done(struct lpfc_hba *phba) if (phba->pci_dev_grp == LPFC_PCI_DEV_OC) { if (phba->hba_flag & HBA_RRQ_ACTIVE) lpfc_handle_rrq_active(phba); - if (phba->hba_flag & FCP_XRI_ABORT_EVENT) - lpfc_sli4_fcp_xri_abort_event_proc(phba); if (phba->hba_flag & ELS_XRI_ABORT_EVENT) lpfc_sli4_els_xri_abort_event_proc(phba); if (phba->hba_flag & ASYNC_EVENT) @@ -859,10 +857,9 @@ lpfc_port_link_failure(struct lpfc_vport *vport) void lpfc_linkdown_port(struct lpfc_vport *vport) { - struct lpfc_hba *phba = vport->phba; struct Scsi_Host *shost = lpfc_shost_from_vport(vport); - if (phba->cfg_enable_fc4_type != LPFC_ENABLE_NVME) + if (vport->cfg_enable_fc4_type != LPFC_ENABLE_NVME) fc_host_post_event(shost, fc_get_event_number(), FCH_EVT_LINKDOWN, 0); @@ -925,8 +922,8 @@ lpfc_linkdown(struct lpfc_hba *phba) vports[i]->fc_myDID = 0; - if ((phba->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) || - (phba->cfg_enable_fc4_type == LPFC_ENABLE_NVME)) { + if ((vport->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) || + (vport->cfg_enable_fc4_type == LPFC_ENABLE_NVME)) { if (phba->nvmet_support) lpfc_nvmet_update_targetport(phba); else @@ -1012,7 +1009,7 @@ lpfc_linkup_port(struct lpfc_vport *vport) (vport != phba->pport)) return; - if (phba->cfg_enable_fc4_type != LPFC_ENABLE_NVME) + if (vport->cfg_enable_fc4_type != LPFC_ENABLE_NVME) fc_host_post_event(shost, fc_get_event_number(), FCH_EVT_LINKUP, 0); @@ -3660,8 +3657,8 @@ lpfc_mbx_cmpl_reg_vpi(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb) spin_unlock_irq(shost->host_lock); vport->fc_myDID = 0; - if ((phba->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) || - (phba->cfg_enable_fc4_type == LPFC_ENABLE_NVME)) { + if ((vport->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) || + (vport->cfg_enable_fc4_type == LPFC_ENABLE_NVME)) { if (phba->nvmet_support) lpfc_nvmet_update_targetport(phba); else @@ -3923,11 +3920,9 @@ lpfc_mbx_cmpl_fabric_reg_login(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb) int lpfc_issue_gidft(struct lpfc_vport *vport) { - struct lpfc_hba *phba = vport->phba; - /* Good status, issue CT Request to NameServer */ - if ((phba->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) || - (phba->cfg_enable_fc4_type == LPFC_ENABLE_FCP)) { + if ((vport->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) || + (vport->cfg_enable_fc4_type == LPFC_ENABLE_FCP)) { if (lpfc_ns_cmd(vport, SLI_CTNS_GID_FT, 0, SLI_CTPT_FCP)) { /* Cannot issue NameServer FCP Query, so finish up * discovery @@ -3942,8 +3937,8 @@ lpfc_issue_gidft(struct lpfc_vport *vport) vport->gidft_inp++; } - if ((phba->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) || - (phba->cfg_enable_fc4_type == LPFC_ENABLE_NVME)) { + if ((vport->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) || + (vport->cfg_enable_fc4_type == LPFC_ENABLE_NVME)) { if (lpfc_ns_cmd(vport, SLI_CTNS_GID_FT, 0, SLI_CTPT_NVME)) { /* Cannot issue NameServer NVME Query, so finish up * discovery @@ -4059,12 +4054,12 @@ out: lpfc_ns_cmd(vport, SLI_CTNS_RSPN_ID, 0, 0); lpfc_ns_cmd(vport, SLI_CTNS_RFT_ID, 0, 0); - if ((phba->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) || - (phba->cfg_enable_fc4_type == LPFC_ENABLE_FCP)) + if ((vport->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) || + (vport->cfg_enable_fc4_type == LPFC_ENABLE_FCP)) lpfc_ns_cmd(vport, SLI_CTNS_RFF_ID, 0, FC_TYPE_FCP); - if ((phba->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) || - (phba->cfg_enable_fc4_type == LPFC_ENABLE_NVME)) + if ((vport->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) || + (vport->cfg_enable_fc4_type == LPFC_ENABLE_NVME)) lpfc_ns_cmd(vport, SLI_CTNS_RFF_ID, 0, FC_TYPE_NVME); @@ -4100,7 +4095,7 @@ lpfc_register_remote_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp) struct fc_rport_identifiers rport_ids; struct lpfc_hba *phba = vport->phba; - if (phba->cfg_enable_fc4_type == LPFC_ENABLE_NVME) + if (vport->cfg_enable_fc4_type == LPFC_ENABLE_NVME) return; /* Remote port has reappeared. Re-register w/ FC transport */ @@ -4175,9 +4170,8 @@ lpfc_unregister_remote_port(struct lpfc_nodelist *ndlp) { struct fc_rport *rport = ndlp->rport; struct lpfc_vport *vport = ndlp->vport; - struct lpfc_hba *phba = vport->phba; - if (phba->cfg_enable_fc4_type == LPFC_ENABLE_NVME) + if (vport->cfg_enable_fc4_type == LPFC_ENABLE_NVME) return; lpfc_debugfs_disc_trc(vport, LPFC_DISC_TRC_RPORT, diff --git a/drivers/scsi/lpfc/lpfc_hw4.h b/drivers/scsi/lpfc/lpfc_hw4.h index c15b9b6fb840..ff875b833192 100644 --- a/drivers/scsi/lpfc/lpfc_hw4.h +++ b/drivers/scsi/lpfc/lpfc_hw4.h @@ -1,7 +1,7 @@ /******************************************************************* * This file is part of the Emulex Linux Device Driver for * * Fibre Channel Host Bus Adapters. * - * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term * + * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term * * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * * Copyright (C) 2009-2016 Emulex. All rights reserved. * * EMULEX and SLI are trademarks of Emulex. * @@ -194,7 +194,7 @@ struct lpfc_sli_intf { #define LPFC_ACT_INTR_CNT 4 /* Algrithmns for scheduling FCP commands to WQs */ -#define LPFC_FCP_SCHED_ROUND_ROBIN 0 +#define LPFC_FCP_SCHED_BY_HDWQ 0 #define LPFC_FCP_SCHED_BY_CPU 1 /* Algrithmns for NameServer Query after RSCN */ @@ -208,12 +208,18 @@ struct lpfc_sli_intf { /* Configuration of Interrupts / sec for entire HBA port */ #define LPFC_MIN_IMAX 5000 #define LPFC_MAX_IMAX 5000000 -#define LPFC_DEF_IMAX 150000 +#define LPFC_DEF_IMAX 0 + +#define LPFC_IMAX_THRESHOLD 1000 +#define LPFC_MAX_AUTO_EQ_DELAY 120 +#define LPFC_EQ_DELAY_STEP 15 +#define LPFC_EQD_ISR_TRIGGER 20000 +/* 1s intervals */ +#define LPFC_EQ_DELAY_MSECS 1000 #define LPFC_MIN_CPU_MAP 0 -#define LPFC_MAX_CPU_MAP 2 +#define LPFC_MAX_CPU_MAP 1 #define LPFC_HBA_CPU_MAP 1 -#define LPFC_DRIVER_CPU_MAP 2 /* Default */ /* PORT_CAPABILITIES constants. */ #define LPFC_MAX_SUPPORTED_PAGES 8 diff --git a/drivers/scsi/lpfc/lpfc_init.c b/drivers/scsi/lpfc/lpfc_init.c index e1129260ed18..3b5873f6751e 100644 --- a/drivers/scsi/lpfc/lpfc_init.c +++ b/drivers/scsi/lpfc/lpfc_init.c @@ -1,7 +1,7 @@ /******************************************************************* * This file is part of the Emulex Linux Device Driver for * * Fibre Channel Host Bus Adapters. * - * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term * + * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term * * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * * Copyright (C) 2004-2016 Emulex. All rights reserved. * * EMULEX and SLI are trademarks of Emulex. * @@ -37,6 +37,7 @@ #include <linux/miscdevice.h> #include <linux/percpu.h> #include <linux/msi.h> +#include <linux/irq.h> #include <linux/bitops.h> #include <scsi/scsi.h> @@ -71,7 +72,6 @@ unsigned long _dump_buf_dif_order; spinlock_t _dump_buf_lock; /* Used when mapping IRQ vectors in a driver centric manner */ -uint16_t *lpfc_used_cpu; uint32_t lpfc_present_cpu; static void lpfc_get_hba_model_desc(struct lpfc_hba *, uint8_t *, uint8_t *); @@ -93,6 +93,8 @@ static void lpfc_sli4_cq_event_release_all(struct lpfc_hba *); static void lpfc_sli4_disable_intr(struct lpfc_hba *); static uint32_t lpfc_sli4_enable_intr(struct lpfc_hba *, uint32_t); static void lpfc_sli4_oas_verify(struct lpfc_hba *phba); +static uint16_t lpfc_find_eq_handle(struct lpfc_hba *, uint16_t); +static uint16_t lpfc_find_cpu_handle(struct lpfc_hba *, uint16_t, int); static struct scsi_transport_template *lpfc_transport_template = NULL; static struct scsi_transport_template *lpfc_vport_transport_template = NULL; @@ -1037,14 +1039,14 @@ lpfc_hba_down_post_s3(struct lpfc_hba *phba) static int lpfc_hba_down_post_s4(struct lpfc_hba *phba) { - struct lpfc_scsi_buf *psb, *psb_next; + struct lpfc_io_buf *psb, *psb_next; struct lpfc_nvmet_rcv_ctx *ctxp, *ctxp_next; + struct lpfc_sli4_hdw_queue *qp; LIST_HEAD(aborts); LIST_HEAD(nvme_aborts); LIST_HEAD(nvmet_aborts); - unsigned long iflag = 0; struct lpfc_sglq *sglq_entry = NULL; - int cnt; + int cnt, idx; lpfc_sli_hbqbuf_free_all(phba); @@ -1071,55 +1073,65 @@ lpfc_hba_down_post_s4(struct lpfc_hba *phba) spin_unlock(&phba->sli4_hba.sgl_list_lock); - /* abts_scsi_buf_list_lock required because worker thread uses this + + /* abts_xxxx_buf_list_lock required because worker thread uses this * list. */ - if (phba->cfg_enable_fc4_type & LPFC_ENABLE_FCP) { - spin_lock(&phba->sli4_hba.abts_scsi_buf_list_lock); - list_splice_init(&phba->sli4_hba.lpfc_abts_scsi_buf_list, - &aborts); - spin_unlock(&phba->sli4_hba.abts_scsi_buf_list_lock); - } - - if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { - spin_lock(&phba->sli4_hba.abts_nvme_buf_list_lock); - list_splice_init(&phba->sli4_hba.lpfc_abts_nvme_buf_list, - &nvme_aborts); - list_splice_init(&phba->sli4_hba.lpfc_abts_nvmet_ctx_list, - &nvmet_aborts); - spin_unlock(&phba->sli4_hba.abts_nvme_buf_list_lock); - } + cnt = 0; + for (idx = 0; idx < phba->cfg_hdw_queue; idx++) { + qp = &phba->sli4_hba.hdwq[idx]; - spin_unlock_irq(&phba->hbalock); - - list_for_each_entry_safe(psb, psb_next, &aborts, list) { - psb->pCmd = NULL; - psb->status = IOSTAT_SUCCESS; - } - spin_lock_irqsave(&phba->scsi_buf_list_put_lock, iflag); - list_splice(&aborts, &phba->lpfc_scsi_buf_list_put); - spin_unlock_irqrestore(&phba->scsi_buf_list_put_lock, iflag); + spin_lock(&qp->abts_scsi_buf_list_lock); + list_splice_init(&qp->lpfc_abts_scsi_buf_list, + &aborts); - if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { - cnt = 0; - list_for_each_entry_safe(psb, psb_next, &nvme_aborts, list) { + list_for_each_entry_safe(psb, psb_next, &aborts, list) { psb->pCmd = NULL; psb->status = IOSTAT_SUCCESS; cnt++; } - spin_lock_irqsave(&phba->nvme_buf_list_put_lock, iflag); - phba->put_nvme_bufs += cnt; - list_splice(&nvme_aborts, &phba->lpfc_nvme_buf_list_put); - spin_unlock_irqrestore(&phba->nvme_buf_list_put_lock, iflag); + spin_lock(&qp->io_buf_list_put_lock); + list_splice_init(&aborts, &qp->lpfc_io_buf_list_put); + qp->put_io_bufs += qp->abts_scsi_io_bufs; + qp->abts_scsi_io_bufs = 0; + spin_unlock(&qp->io_buf_list_put_lock); + spin_unlock(&qp->abts_scsi_buf_list_lock); + + if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { + spin_lock(&qp->abts_nvme_buf_list_lock); + list_splice_init(&qp->lpfc_abts_nvme_buf_list, + &nvme_aborts); + list_for_each_entry_safe(psb, psb_next, &nvme_aborts, + list) { + psb->pCmd = NULL; + psb->status = IOSTAT_SUCCESS; + cnt++; + } + spin_lock(&qp->io_buf_list_put_lock); + qp->put_io_bufs += qp->abts_nvme_io_bufs; + qp->abts_nvme_io_bufs = 0; + list_splice_init(&nvme_aborts, + &qp->lpfc_io_buf_list_put); + spin_unlock(&qp->io_buf_list_put_lock); + spin_unlock(&qp->abts_nvme_buf_list_lock); + + } + } + if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { + spin_lock(&phba->sli4_hba.abts_nvmet_buf_list_lock); + list_splice_init(&phba->sli4_hba.lpfc_abts_nvmet_ctx_list, + &nvmet_aborts); + spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock); list_for_each_entry_safe(ctxp, ctxp_next, &nvmet_aborts, list) { ctxp->flag &= ~(LPFC_NVMET_XBUSY | LPFC_NVMET_ABORT_OP); lpfc_nvmet_ctxbuf_post(phba, ctxp->ctxbuf); } } + spin_unlock_irq(&phba->hbalock); lpfc_sli4_free_sp_events(phba); - return 0; + return cnt; } /** @@ -1239,6 +1251,96 @@ lpfc_hb_mbox_cmpl(struct lpfc_hba * phba, LPFC_MBOXQ_t * pmboxq) return; } +static void +lpfc_hb_eq_delay_work(struct work_struct *work) +{ + struct lpfc_hba *phba = container_of(to_delayed_work(work), + struct lpfc_hba, eq_delay_work); + struct lpfc_eq_intr_info *eqi, *eqi_new; + struct lpfc_queue *eq, *eq_next; + unsigned char *eqcnt = NULL; + uint32_t usdelay; + int i; + + if (!phba->cfg_auto_imax || phba->pport->load_flag & FC_UNLOADING) + return; + + if (phba->link_state == LPFC_HBA_ERROR || + phba->pport->fc_flag & FC_OFFLINE_MODE) + goto requeue; + + eqcnt = kcalloc(num_possible_cpus(), sizeof(unsigned char), + GFP_KERNEL); + if (!eqcnt) + goto requeue; + + for (i = 0; i < phba->cfg_irq_chann; i++) { + eq = phba->sli4_hba.hdwq[i].hba_eq; + if (eq && eqcnt[eq->last_cpu] < 2) + eqcnt[eq->last_cpu]++; + continue; + } + + for_each_present_cpu(i) { + if (phba->cfg_irq_chann > 1 && eqcnt[i] < 2) + continue; + + eqi = per_cpu_ptr(phba->sli4_hba.eq_info, i); + + usdelay = (eqi->icnt / LPFC_IMAX_THRESHOLD) * + LPFC_EQ_DELAY_STEP; + if (usdelay > LPFC_MAX_AUTO_EQ_DELAY) + usdelay = LPFC_MAX_AUTO_EQ_DELAY; + + eqi->icnt = 0; + + list_for_each_entry_safe(eq, eq_next, &eqi->list, cpu_list) { + if (eq->last_cpu != i) { + eqi_new = per_cpu_ptr(phba->sli4_hba.eq_info, + eq->last_cpu); + list_move_tail(&eq->cpu_list, &eqi_new->list); + continue; + } + if (usdelay != eq->q_mode) + lpfc_modify_hba_eq_delay(phba, eq->hdwq, 1, + usdelay); + } + } + + kfree(eqcnt); + +requeue: + queue_delayed_work(phba->wq, &phba->eq_delay_work, + msecs_to_jiffies(LPFC_EQ_DELAY_MSECS)); +} + +/** + * lpfc_hb_mxp_handler - Multi-XRI pools handler to adjust XRI distribution + * @phba: pointer to lpfc hba data structure. + * + * For each heartbeat, this routine does some heuristic methods to adjust + * XRI distribution. The goal is to fully utilize free XRIs. + **/ +static void lpfc_hb_mxp_handler(struct lpfc_hba *phba) +{ + u32 i; + u32 hwq_count; + + hwq_count = phba->cfg_hdw_queue; + for (i = 0; i < hwq_count; i++) { + /* Adjust XRIs in private pool */ + lpfc_adjust_pvt_pool_count(phba, i); + + /* Adjust high watermark */ + lpfc_adjust_high_watermark(phba, i); + +#ifdef LPFC_MXP_STAT + /* Snapshot pbl, pvt and busy count */ + lpfc_snapshot_mxp(phba, i); +#endif + } +} + /** * lpfc_hb_timeout_handler - The HBA-timer timeout handler * @phba: pointer to lpfc hba data structure. @@ -1264,16 +1366,11 @@ lpfc_hb_timeout_handler(struct lpfc_hba *phba) int retval, i; struct lpfc_sli *psli = &phba->sli; LIST_HEAD(completions); - struct lpfc_queue *qp; - unsigned long time_elapsed; - uint32_t tick_cqe, max_cqe, val; - uint64_t tot, data1, data2, data3; - struct lpfc_nvmet_tgtport *tgtp; - struct lpfc_register reg_data; - struct nvme_fc_local_port *localport; - struct lpfc_nvme_lport *lport; - struct lpfc_nvme_ctrl_stat *cstat; - void __iomem *eqdreg = phba->sli4_hba.u.if_type2.EQDregaddr; + + if (phba->cfg_xri_rebalancing) { + /* Multi-XRI pools handler */ + lpfc_hb_mxp_handler(phba); + } vports = lpfc_create_vport_work_array(phba); if (vports != NULL) @@ -1288,107 +1385,6 @@ lpfc_hb_timeout_handler(struct lpfc_hba *phba) (phba->pport->fc_flag & FC_OFFLINE_MODE)) return; - if (phba->cfg_auto_imax) { - if (!phba->last_eqdelay_time) { - phba->last_eqdelay_time = jiffies; - goto skip_eqdelay; - } - time_elapsed = jiffies - phba->last_eqdelay_time; - phba->last_eqdelay_time = jiffies; - - tot = 0xffff; - /* Check outstanding IO count */ - if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { - if (phba->nvmet_support) { - tgtp = phba->targetport->private; - /* Calculate outstanding IOs */ - tot = atomic_read(&tgtp->rcv_fcp_cmd_drop); - tot += atomic_read(&tgtp->xmt_fcp_release); - tot = atomic_read(&tgtp->rcv_fcp_cmd_in) - tot; - } else { - localport = phba->pport->localport; - if (!localport || !localport->private) - goto skip_eqdelay; - lport = (struct lpfc_nvme_lport *) - localport->private; - tot = 0; - for (i = 0; - i < phba->cfg_nvme_io_channel; i++) { - cstat = &lport->cstat[i]; - data1 = atomic_read( - &cstat->fc4NvmeInputRequests); - data2 = atomic_read( - &cstat->fc4NvmeOutputRequests); - data3 = atomic_read( - &cstat->fc4NvmeControlRequests); - tot += (data1 + data2 + data3); - tot -= atomic_read( - &cstat->fc4NvmeIoCmpls); - } - } - } - - /* Interrupts per sec per EQ */ - val = phba->cfg_fcp_imax / phba->io_channel_irqs; - tick_cqe = val / CONFIG_HZ; /* Per tick per EQ */ - - /* Assume 1 CQE/ISR, calc max CQEs allowed for time duration */ - max_cqe = time_elapsed * tick_cqe; - - for (i = 0; i < phba->io_channel_irqs; i++) { - /* Fast-path EQ */ - qp = phba->sli4_hba.hba_eq[i]; - if (!qp) - continue; - - /* Use no EQ delay if we don't have many outstanding - * IOs, or if we are only processing 1 CQE/ISR or less. - * Otherwise, assume we can process up to lpfc_fcp_imax - * interrupts per HBA. - */ - if (tot < LPFC_NODELAY_MAX_IO || - qp->EQ_cqe_cnt <= max_cqe) - val = 0; - else - val = phba->cfg_fcp_imax; - - if (phba->sli.sli_flag & LPFC_SLI_USE_EQDR) { - /* Use EQ Delay Register method */ - - /* Convert for EQ Delay register */ - if (val) { - /* First, interrupts per sec per EQ */ - val = phba->cfg_fcp_imax / - phba->io_channel_irqs; - - /* us delay between each interrupt */ - val = LPFC_SEC_TO_USEC / val; - } - if (val != qp->q_mode) { - reg_data.word0 = 0; - bf_set(lpfc_sliport_eqdelay_id, - ®_data, qp->queue_id); - bf_set(lpfc_sliport_eqdelay_delay, - ®_data, val); - writel(reg_data.word0, eqdreg); - } - } else { - /* Use mbox command method */ - if (val != qp->q_mode) - lpfc_modify_hba_eq_delay(phba, i, - 1, val); - } - - /* - * val is cfg_fcp_imax or 0 for mbox delay or us delay - * between interrupts for EQDR. - */ - qp->q_mode = val; - qp->EQ_cqe_cnt = 0; - } - } - -skip_eqdelay: spin_lock_irq(&phba->pport->work_port_lock); if (time_after(phba->last_completion_time + @@ -2943,7 +2939,9 @@ lpfc_sli4_stop_fcf_redisc_wait_timer(struct lpfc_hba *phba) void lpfc_stop_hba_timers(struct lpfc_hba *phba) { - lpfc_stop_vport_timers(phba->pport); + if (phba->pport) + lpfc_stop_vport_timers(phba->pport); + cancel_delayed_work_sync(&phba->eq_delay_work); del_timer_sync(&phba->sli.mbox_tmo); del_timer_sync(&phba->fabric_block_timer); del_timer_sync(&phba->eratt_poll); @@ -3071,6 +3069,242 @@ lpfc_sli4_node_prep(struct lpfc_hba *phba) } /** + * lpfc_create_expedite_pool - create expedite pool + * @phba: pointer to lpfc hba data structure. + * + * This routine moves a batch of XRIs from lpfc_io_buf_list_put of HWQ 0 + * to expedite pool. Mark them as expedite. + **/ +void lpfc_create_expedite_pool(struct lpfc_hba *phba) +{ + struct lpfc_sli4_hdw_queue *qp; + struct lpfc_io_buf *lpfc_ncmd; + struct lpfc_io_buf *lpfc_ncmd_next; + struct lpfc_epd_pool *epd_pool; + unsigned long iflag; + + epd_pool = &phba->epd_pool; + qp = &phba->sli4_hba.hdwq[0]; + + spin_lock_init(&epd_pool->lock); + spin_lock_irqsave(&qp->io_buf_list_put_lock, iflag); + spin_lock(&epd_pool->lock); + INIT_LIST_HEAD(&epd_pool->list); + list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next, + &qp->lpfc_io_buf_list_put, list) { + list_move_tail(&lpfc_ncmd->list, &epd_pool->list); + lpfc_ncmd->expedite = true; + qp->put_io_bufs--; + epd_pool->count++; + if (epd_pool->count >= XRI_BATCH) + break; + } + spin_unlock(&epd_pool->lock); + spin_unlock_irqrestore(&qp->io_buf_list_put_lock, iflag); +} + +/** + * lpfc_destroy_expedite_pool - destroy expedite pool + * @phba: pointer to lpfc hba data structure. + * + * This routine returns XRIs from expedite pool to lpfc_io_buf_list_put + * of HWQ 0. Clear the mark. + **/ +void lpfc_destroy_expedite_pool(struct lpfc_hba *phba) +{ + struct lpfc_sli4_hdw_queue *qp; + struct lpfc_io_buf *lpfc_ncmd; + struct lpfc_io_buf *lpfc_ncmd_next; + struct lpfc_epd_pool *epd_pool; + unsigned long iflag; + + epd_pool = &phba->epd_pool; + qp = &phba->sli4_hba.hdwq[0]; + + spin_lock_irqsave(&qp->io_buf_list_put_lock, iflag); + spin_lock(&epd_pool->lock); + list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next, + &epd_pool->list, list) { + list_move_tail(&lpfc_ncmd->list, + &qp->lpfc_io_buf_list_put); + lpfc_ncmd->flags = false; + qp->put_io_bufs++; + epd_pool->count--; + } + spin_unlock(&epd_pool->lock); + spin_unlock_irqrestore(&qp->io_buf_list_put_lock, iflag); +} + +/** + * lpfc_create_multixri_pools - create multi-XRI pools + * @phba: pointer to lpfc hba data structure. + * + * This routine initialize public, private per HWQ. Then, move XRIs from + * lpfc_io_buf_list_put to public pool. High and low watermark are also + * Initialized. + **/ +void lpfc_create_multixri_pools(struct lpfc_hba *phba) +{ + u32 i, j; + u32 hwq_count; + u32 count_per_hwq; + struct lpfc_io_buf *lpfc_ncmd; + struct lpfc_io_buf *lpfc_ncmd_next; + unsigned long iflag; + struct lpfc_sli4_hdw_queue *qp; + struct lpfc_multixri_pool *multixri_pool; + struct lpfc_pbl_pool *pbl_pool; + struct lpfc_pvt_pool *pvt_pool; + + lpfc_printf_log(phba, KERN_INFO, LOG_INIT, + "1234 num_hdw_queue=%d num_present_cpu=%d common_xri_cnt=%d\n", + phba->cfg_hdw_queue, phba->sli4_hba.num_present_cpu, + phba->sli4_hba.io_xri_cnt); + + if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) + lpfc_create_expedite_pool(phba); + + hwq_count = phba->cfg_hdw_queue; + count_per_hwq = phba->sli4_hba.io_xri_cnt / hwq_count; + + for (i = 0; i < hwq_count; i++) { + multixri_pool = kzalloc(sizeof(*multixri_pool), GFP_KERNEL); + + if (!multixri_pool) { + lpfc_printf_log(phba, KERN_INFO, LOG_INIT, + "1238 Failed to allocate memory for " + "multixri_pool\n"); + + if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) + lpfc_destroy_expedite_pool(phba); + + j = 0; + while (j < i) { + qp = &phba->sli4_hba.hdwq[j]; + kfree(qp->p_multixri_pool); + j++; + } + phba->cfg_xri_rebalancing = 0; + return; + } + + qp = &phba->sli4_hba.hdwq[i]; + qp->p_multixri_pool = multixri_pool; + + multixri_pool->xri_limit = count_per_hwq; + multixri_pool->rrb_next_hwqid = i; + + /* Deal with public free xri pool */ + pbl_pool = &multixri_pool->pbl_pool; + spin_lock_init(&pbl_pool->lock); + spin_lock_irqsave(&qp->io_buf_list_put_lock, iflag); + spin_lock(&pbl_pool->lock); + INIT_LIST_HEAD(&pbl_pool->list); + list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next, + &qp->lpfc_io_buf_list_put, list) { + list_move_tail(&lpfc_ncmd->list, &pbl_pool->list); + qp->put_io_bufs--; + pbl_pool->count++; + } + lpfc_printf_log(phba, KERN_INFO, LOG_INIT, + "1235 Moved %d buffers from PUT list over to pbl_pool[%d]\n", + pbl_pool->count, i); + spin_unlock(&pbl_pool->lock); + spin_unlock_irqrestore(&qp->io_buf_list_put_lock, iflag); + + /* Deal with private free xri pool */ + pvt_pool = &multixri_pool->pvt_pool; + pvt_pool->high_watermark = multixri_pool->xri_limit / 2; + pvt_pool->low_watermark = XRI_BATCH; + spin_lock_init(&pvt_pool->lock); + spin_lock_irqsave(&pvt_pool->lock, iflag); + INIT_LIST_HEAD(&pvt_pool->list); + pvt_pool->count = 0; + spin_unlock_irqrestore(&pvt_pool->lock, iflag); + } +} + +/** + * lpfc_destroy_multixri_pools - destroy multi-XRI pools + * @phba: pointer to lpfc hba data structure. + * + * This routine returns XRIs from public/private to lpfc_io_buf_list_put. + **/ +void lpfc_destroy_multixri_pools(struct lpfc_hba *phba) +{ + u32 i; + u32 hwq_count; + struct lpfc_io_buf *lpfc_ncmd; + struct lpfc_io_buf *lpfc_ncmd_next; + unsigned long iflag; + struct lpfc_sli4_hdw_queue *qp; + struct lpfc_multixri_pool *multixri_pool; + struct lpfc_pbl_pool *pbl_pool; + struct lpfc_pvt_pool *pvt_pool; + + if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) + lpfc_destroy_expedite_pool(phba); + + hwq_count = phba->cfg_hdw_queue; + + for (i = 0; i < hwq_count; i++) { + qp = &phba->sli4_hba.hdwq[i]; + multixri_pool = qp->p_multixri_pool; + if (!multixri_pool) + continue; + + qp->p_multixri_pool = NULL; + + spin_lock_irqsave(&qp->io_buf_list_put_lock, iflag); + + /* Deal with public free xri pool */ + pbl_pool = &multixri_pool->pbl_pool; + spin_lock(&pbl_pool->lock); + + lpfc_printf_log(phba, KERN_INFO, LOG_INIT, + "1236 Moving %d buffers from pbl_pool[%d] TO PUT list\n", + pbl_pool->count, i); + + list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next, + &pbl_pool->list, list) { + list_move_tail(&lpfc_ncmd->list, + &qp->lpfc_io_buf_list_put); + qp->put_io_bufs++; + pbl_pool->count--; + } + + INIT_LIST_HEAD(&pbl_pool->list); + pbl_pool->count = 0; + + spin_unlock(&pbl_pool->lock); + + /* Deal with private free xri pool */ + pvt_pool = &multixri_pool->pvt_pool; + spin_lock(&pvt_pool->lock); + + lpfc_printf_log(phba, KERN_INFO, LOG_INIT, + "1237 Moving %d buffers from pvt_pool[%d] TO PUT list\n", + pvt_pool->count, i); + + list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next, + &pvt_pool->list, list) { + list_move_tail(&lpfc_ncmd->list, + &qp->lpfc_io_buf_list_put); + qp->put_io_bufs++; + pvt_pool->count--; + } + + INIT_LIST_HEAD(&pvt_pool->list); + pvt_pool->count = 0; + + spin_unlock(&pvt_pool->lock); + spin_unlock_irqrestore(&qp->io_buf_list_put_lock, iflag); + + kfree(multixri_pool); + } +} + +/** * lpfc_online - Initialize and bring a HBA online * @phba: pointer to lpfc hba data structure. * @@ -3152,6 +3386,9 @@ lpfc_online(struct lpfc_hba *phba) } lpfc_destroy_vport_work_array(phba, vports); + if (phba->cfg_xri_rebalancing) + lpfc_create_multixri_pools(phba); + lpfc_unblock_mgmt_io(phba); return 0; } @@ -3310,6 +3547,9 @@ lpfc_offline(struct lpfc_hba *phba) spin_unlock_irq(shost->host_lock); } lpfc_destroy_vport_work_array(phba, vports); + + if (phba->cfg_xri_rebalancing) + lpfc_destroy_multixri_pools(phba); } /** @@ -3323,7 +3563,7 @@ lpfc_offline(struct lpfc_hba *phba) static void lpfc_scsi_free(struct lpfc_hba *phba) { - struct lpfc_scsi_buf *sb, *sb_next; + struct lpfc_io_buf *sb, *sb_next; if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_FCP)) return; @@ -3355,50 +3595,57 @@ lpfc_scsi_free(struct lpfc_hba *phba) spin_unlock(&phba->scsi_buf_list_get_lock); spin_unlock_irq(&phba->hbalock); } + /** - * lpfc_nvme_free - Free all the NVME buffers and IOCBs from driver lists + * lpfc_io_free - Free all the IO buffers and IOCBs from driver lists * @phba: pointer to lpfc hba data structure. * - * This routine is to free all the NVME buffers and IOCBs from the driver + * This routine is to free all the IO buffers and IOCBs from the driver * list back to kernel. It is called from lpfc_pci_remove_one to free * the internal resources before the device is removed from the system. **/ -static void -lpfc_nvme_free(struct lpfc_hba *phba) +void +lpfc_io_free(struct lpfc_hba *phba) { - struct lpfc_nvme_buf *lpfc_ncmd, *lpfc_ncmd_next; - - if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME)) - return; + struct lpfc_io_buf *lpfc_ncmd, *lpfc_ncmd_next; + struct lpfc_sli4_hdw_queue *qp; + int idx; spin_lock_irq(&phba->hbalock); - /* Release all the lpfc_nvme_bufs maintained by this host. */ - spin_lock(&phba->nvme_buf_list_put_lock); - list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next, - &phba->lpfc_nvme_buf_list_put, list) { - list_del(&lpfc_ncmd->list); - phba->put_nvme_bufs--; - dma_pool_free(phba->lpfc_sg_dma_buf_pool, lpfc_ncmd->data, - lpfc_ncmd->dma_handle); - kfree(lpfc_ncmd); - phba->total_nvme_bufs--; + for (idx = 0; idx < phba->cfg_hdw_queue; idx++) { + qp = &phba->sli4_hba.hdwq[idx]; + /* Release all the lpfc_nvme_bufs maintained by this host. */ + spin_lock(&qp->io_buf_list_put_lock); + list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next, + &qp->lpfc_io_buf_list_put, + list) { + list_del(&lpfc_ncmd->list); + qp->put_io_bufs--; + dma_pool_free(phba->lpfc_sg_dma_buf_pool, + lpfc_ncmd->data, lpfc_ncmd->dma_handle); + kfree(lpfc_ncmd); + qp->total_io_bufs--; + } + spin_unlock(&qp->io_buf_list_put_lock); + + spin_lock(&qp->io_buf_list_get_lock); + list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next, + &qp->lpfc_io_buf_list_get, + list) { + list_del(&lpfc_ncmd->list); + qp->get_io_bufs--; + dma_pool_free(phba->lpfc_sg_dma_buf_pool, + lpfc_ncmd->data, lpfc_ncmd->dma_handle); + kfree(lpfc_ncmd); + qp->total_io_bufs--; + } + spin_unlock(&qp->io_buf_list_get_lock); } - spin_unlock(&phba->nvme_buf_list_put_lock); - spin_lock(&phba->nvme_buf_list_get_lock); - list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next, - &phba->lpfc_nvme_buf_list_get, list) { - list_del(&lpfc_ncmd->list); - phba->get_nvme_bufs--; - dma_pool_free(phba->lpfc_sg_dma_buf_pool, lpfc_ncmd->data, - lpfc_ncmd->dma_handle); - kfree(lpfc_ncmd); - phba->total_nvme_bufs--; - } - spin_unlock(&phba->nvme_buf_list_get_lock); spin_unlock_irq(&phba->hbalock); } + /** * lpfc_sli4_els_sgl_update - update ELS xri-sgl sizing and mapping * @phba: pointer to lpfc hba data structure. @@ -3640,8 +3887,102 @@ out_free_mem: return rc; } +int +lpfc_io_buf_flush(struct lpfc_hba *phba, struct list_head *cbuf) +{ + LIST_HEAD(blist); + struct lpfc_sli4_hdw_queue *qp; + struct lpfc_io_buf *lpfc_cmd; + struct lpfc_io_buf *iobufp, *prev_iobufp; + int idx, cnt, xri, inserted; + + cnt = 0; + for (idx = 0; idx < phba->cfg_hdw_queue; idx++) { + qp = &phba->sli4_hba.hdwq[idx]; + spin_lock_irq(&qp->io_buf_list_get_lock); + spin_lock(&qp->io_buf_list_put_lock); + + /* Take everything off the get and put lists */ + list_splice_init(&qp->lpfc_io_buf_list_get, &blist); + list_splice(&qp->lpfc_io_buf_list_put, &blist); + INIT_LIST_HEAD(&qp->lpfc_io_buf_list_get); + INIT_LIST_HEAD(&qp->lpfc_io_buf_list_put); + cnt += qp->get_io_bufs + qp->put_io_bufs; + qp->get_io_bufs = 0; + qp->put_io_bufs = 0; + qp->total_io_bufs = 0; + spin_unlock(&qp->io_buf_list_put_lock); + spin_unlock_irq(&qp->io_buf_list_get_lock); + } + + /* + * Take IO buffers off blist and put on cbuf sorted by XRI. + * This is because POST_SGL takes a sequential range of XRIs + * to post to the firmware. + */ + for (idx = 0; idx < cnt; idx++) { + list_remove_head(&blist, lpfc_cmd, struct lpfc_io_buf, list); + if (!lpfc_cmd) + return cnt; + if (idx == 0) { + list_add_tail(&lpfc_cmd->list, cbuf); + continue; + } + xri = lpfc_cmd->cur_iocbq.sli4_xritag; + inserted = 0; + prev_iobufp = NULL; + list_for_each_entry(iobufp, cbuf, list) { + if (xri < iobufp->cur_iocbq.sli4_xritag) { + if (prev_iobufp) + list_add(&lpfc_cmd->list, + &prev_iobufp->list); + else + list_add(&lpfc_cmd->list, cbuf); + inserted = 1; + break; + } + prev_iobufp = iobufp; + } + if (!inserted) + list_add_tail(&lpfc_cmd->list, cbuf); + } + return cnt; +} + +int +lpfc_io_buf_replenish(struct lpfc_hba *phba, struct list_head *cbuf) +{ + struct lpfc_sli4_hdw_queue *qp; + struct lpfc_io_buf *lpfc_cmd; + int idx, cnt; + + qp = phba->sli4_hba.hdwq; + cnt = 0; + while (!list_empty(cbuf)) { + for (idx = 0; idx < phba->cfg_hdw_queue; idx++) { + list_remove_head(cbuf, lpfc_cmd, + struct lpfc_io_buf, list); + if (!lpfc_cmd) + return cnt; + cnt++; + qp = &phba->sli4_hba.hdwq[idx]; + lpfc_cmd->hdwq_no = idx; + lpfc_cmd->hdwq = qp; + lpfc_cmd->cur_iocbq.wqe_cmpl = NULL; + lpfc_cmd->cur_iocbq.iocb_cmpl = NULL; + spin_lock(&qp->io_buf_list_put_lock); + list_add_tail(&lpfc_cmd->list, + &qp->lpfc_io_buf_list_put); + qp->put_io_bufs++; + qp->total_io_bufs++; + spin_unlock(&qp->io_buf_list_put_lock); + } + } + return cnt; +} + /** - * lpfc_sli4_scsi_sgl_update - update xri-sgl sizing and mapping + * lpfc_sli4_io_sgl_update - update xri-sgl sizing and mapping * @phba: pointer to lpfc hba data structure. * * This routine first calculates the sizes of the current els and allocated @@ -3653,94 +3994,192 @@ out_free_mem: * 0 - successful (for now, it always returns 0) **/ int -lpfc_sli4_scsi_sgl_update(struct lpfc_hba *phba) +lpfc_sli4_io_sgl_update(struct lpfc_hba *phba) { - struct lpfc_scsi_buf *psb, *psb_next; - uint16_t i, lxri, els_xri_cnt, scsi_xri_cnt; - LIST_HEAD(scsi_sgl_list); - int rc; - - /* - * update on pci function's els xri-sgl list - */ - els_xri_cnt = lpfc_sli4_get_els_iocb_cnt(phba); - phba->total_scsi_bufs = 0; + struct lpfc_io_buf *lpfc_ncmd = NULL, *lpfc_ncmd_next = NULL; + uint16_t i, lxri, els_xri_cnt; + uint16_t io_xri_cnt, io_xri_max; + LIST_HEAD(io_sgl_list); + int rc, cnt; /* - * update on pci function's allocated scsi xri-sgl list + * update on pci function's allocated nvme xri-sgl list */ - /* maximum number of xris available for scsi buffers */ - phba->sli4_hba.scsi_xri_max = phba->sli4_hba.max_cfg_param.max_xri - - els_xri_cnt; - if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_FCP)) - return 0; + /* maximum number of xris available for nvme buffers */ + els_xri_cnt = lpfc_sli4_get_els_iocb_cnt(phba); + io_xri_max = phba->sli4_hba.max_cfg_param.max_xri - els_xri_cnt; + phba->sli4_hba.io_xri_max = io_xri_max; - if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) - phba->sli4_hba.scsi_xri_max = /* Split them up */ - (phba->sli4_hba.scsi_xri_max * - phba->cfg_xri_split) / 100; + lpfc_printf_log(phba, KERN_INFO, LOG_SLI, + "6074 Current allocated XRI sgl count:%d, " + "maximum XRI count:%d\n", + phba->sli4_hba.io_xri_cnt, + phba->sli4_hba.io_xri_max); - spin_lock_irq(&phba->scsi_buf_list_get_lock); - spin_lock(&phba->scsi_buf_list_put_lock); - list_splice_init(&phba->lpfc_scsi_buf_list_get, &scsi_sgl_list); - list_splice(&phba->lpfc_scsi_buf_list_put, &scsi_sgl_list); - spin_unlock(&phba->scsi_buf_list_put_lock); - spin_unlock_irq(&phba->scsi_buf_list_get_lock); + cnt = lpfc_io_buf_flush(phba, &io_sgl_list); - lpfc_printf_log(phba, KERN_INFO, LOG_SLI, - "6060 Current allocated SCSI xri-sgl count:%d, " - "maximum SCSI xri count:%d (split:%d)\n", - phba->sli4_hba.scsi_xri_cnt, - phba->sli4_hba.scsi_xri_max, phba->cfg_xri_split); - - if (phba->sli4_hba.scsi_xri_cnt > phba->sli4_hba.scsi_xri_max) { - /* max scsi xri shrinked below the allocated scsi buffers */ - scsi_xri_cnt = phba->sli4_hba.scsi_xri_cnt - - phba->sli4_hba.scsi_xri_max; - /* release the extra allocated scsi buffers */ - for (i = 0; i < scsi_xri_cnt; i++) { - list_remove_head(&scsi_sgl_list, psb, - struct lpfc_scsi_buf, list); - if (psb) { + if (phba->sli4_hba.io_xri_cnt > phba->sli4_hba.io_xri_max) { + /* max nvme xri shrunk below the allocated nvme buffers */ + io_xri_cnt = phba->sli4_hba.io_xri_cnt - + phba->sli4_hba.io_xri_max; + /* release the extra allocated nvme buffers */ + for (i = 0; i < io_xri_cnt; i++) { + list_remove_head(&io_sgl_list, lpfc_ncmd, + struct lpfc_io_buf, list); + if (lpfc_ncmd) { dma_pool_free(phba->lpfc_sg_dma_buf_pool, - psb->data, psb->dma_handle); - kfree(psb); + lpfc_ncmd->data, + lpfc_ncmd->dma_handle); + kfree(lpfc_ncmd); } } - spin_lock_irq(&phba->scsi_buf_list_get_lock); - phba->sli4_hba.scsi_xri_cnt -= scsi_xri_cnt; - spin_unlock_irq(&phba->scsi_buf_list_get_lock); + phba->sli4_hba.io_xri_cnt -= io_xri_cnt; } - /* update xris associated to remaining allocated scsi buffers */ - psb = NULL; - psb_next = NULL; - list_for_each_entry_safe(psb, psb_next, &scsi_sgl_list, list) { + /* update xris associated to remaining allocated nvme buffers */ + lpfc_ncmd = NULL; + lpfc_ncmd_next = NULL; + phba->sli4_hba.io_xri_cnt = cnt; + list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next, + &io_sgl_list, list) { lxri = lpfc_sli4_next_xritag(phba); if (lxri == NO_XRI) { lpfc_printf_log(phba, KERN_ERR, LOG_SLI, - "2560 Failed to allocate xri for " - "scsi buffer\n"); + "6075 Failed to allocate xri for " + "nvme buffer\n"); rc = -ENOMEM; goto out_free_mem; } - psb->cur_iocbq.sli4_lxritag = lxri; - psb->cur_iocbq.sli4_xritag = phba->sli4_hba.xri_ids[lxri]; + lpfc_ncmd->cur_iocbq.sli4_lxritag = lxri; + lpfc_ncmd->cur_iocbq.sli4_xritag = phba->sli4_hba.xri_ids[lxri]; } - spin_lock_irq(&phba->scsi_buf_list_get_lock); - spin_lock(&phba->scsi_buf_list_put_lock); - list_splice_init(&scsi_sgl_list, &phba->lpfc_scsi_buf_list_get); - INIT_LIST_HEAD(&phba->lpfc_scsi_buf_list_put); - spin_unlock(&phba->scsi_buf_list_put_lock); - spin_unlock_irq(&phba->scsi_buf_list_get_lock); + cnt = lpfc_io_buf_replenish(phba, &io_sgl_list); return 0; out_free_mem: - lpfc_scsi_free(phba); + lpfc_io_free(phba); return rc; } +/** + * lpfc_new_io_buf - IO buffer allocator for HBA with SLI4 IF spec + * @vport: The virtual port for which this call being executed. + * @num_to_allocate: The requested number of buffers to allocate. + * + * This routine allocates nvme buffers for device with SLI-4 interface spec, + * the nvme buffer contains all the necessary information needed to initiate + * an I/O. After allocating up to @num_to_allocate IO buffers and put + * them on a list, it post them to the port by using SGL block post. + * + * Return codes: + * int - number of IO buffers that were allocated and posted. + * 0 = failure, less than num_to_alloc is a partial failure. + **/ +int +lpfc_new_io_buf(struct lpfc_hba *phba, int num_to_alloc) +{ + struct lpfc_io_buf *lpfc_ncmd; + struct lpfc_iocbq *pwqeq; + uint16_t iotag, lxri = 0; + int bcnt, num_posted; + LIST_HEAD(prep_nblist); + LIST_HEAD(post_nblist); + LIST_HEAD(nvme_nblist); + + /* Sanity check to ensure our sizing is right for both SCSI and NVME */ + if (sizeof(struct lpfc_io_buf) > LPFC_COMMON_IO_BUF_SZ) { + lpfc_printf_log(phba, KERN_ERR, LOG_FCP, + "6426 Common buffer size %ld exceeds %d\n", + sizeof(struct lpfc_io_buf), + LPFC_COMMON_IO_BUF_SZ); + return 0; + } + + phba->sli4_hba.io_xri_cnt = 0; + for (bcnt = 0; bcnt < num_to_alloc; bcnt++) { + lpfc_ncmd = kzalloc(LPFC_COMMON_IO_BUF_SZ, GFP_KERNEL); + if (!lpfc_ncmd) + break; + /* + * Get memory from the pci pool to map the virt space to + * pci bus space for an I/O. The DMA buffer includes the + * number of SGE's necessary to support the sg_tablesize. + */ + lpfc_ncmd->data = dma_pool_alloc(phba->lpfc_sg_dma_buf_pool, + GFP_KERNEL, + &lpfc_ncmd->dma_handle); + if (!lpfc_ncmd->data) { + kfree(lpfc_ncmd); + break; + } + memset(lpfc_ncmd->data, 0, phba->cfg_sg_dma_buf_size); + + /* + * 4K Page alignment is CRITICAL to BlockGuard, double check + * to be sure. + */ + if ((phba->sli3_options & LPFC_SLI3_BG_ENABLED) && + (((unsigned long)(lpfc_ncmd->data) & + (unsigned long)(SLI4_PAGE_SIZE - 1)) != 0)) { + lpfc_printf_log(phba, KERN_ERR, LOG_FCP, + "3369 Memory alignment err: addr=%lx\n", + (unsigned long)lpfc_ncmd->data); + dma_pool_free(phba->lpfc_sg_dma_buf_pool, + lpfc_ncmd->data, lpfc_ncmd->dma_handle); + kfree(lpfc_ncmd); + break; + } + + lxri = lpfc_sli4_next_xritag(phba); + if (lxri == NO_XRI) { + dma_pool_free(phba->lpfc_sg_dma_buf_pool, + lpfc_ncmd->data, lpfc_ncmd->dma_handle); + kfree(lpfc_ncmd); + break; + } + pwqeq = &lpfc_ncmd->cur_iocbq; + + /* Allocate iotag for lpfc_ncmd->cur_iocbq. */ + iotag = lpfc_sli_next_iotag(phba, pwqeq); + if (iotag == 0) { + dma_pool_free(phba->lpfc_sg_dma_buf_pool, + lpfc_ncmd->data, lpfc_ncmd->dma_handle); + kfree(lpfc_ncmd); + lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, + "6121 Failed to allocate IOTAG for" + " XRI:0x%x\n", lxri); + lpfc_sli4_free_xri(phba, lxri); + break; + } + pwqeq->sli4_lxritag = lxri; + pwqeq->sli4_xritag = phba->sli4_hba.xri_ids[lxri]; + pwqeq->context1 = lpfc_ncmd; + + /* Initialize local short-hand pointers. */ + lpfc_ncmd->dma_sgl = lpfc_ncmd->data; + lpfc_ncmd->dma_phys_sgl = lpfc_ncmd->dma_handle; + lpfc_ncmd->cur_iocbq.context1 = lpfc_ncmd; + spin_lock_init(&lpfc_ncmd->buf_lock); + + /* add the nvme buffer to a post list */ + list_add_tail(&lpfc_ncmd->list, &post_nblist); + phba->sli4_hba.io_xri_cnt++; + } + lpfc_printf_log(phba, KERN_INFO, LOG_NVME, + "6114 Allocate %d out of %d requested new NVME " + "buffers\n", bcnt, num_to_alloc); + + /* post the list of nvme buffer sgls to port if available */ + if (!list_empty(&post_nblist)) + num_posted = lpfc_sli4_post_io_sgl_list( + phba, &post_nblist, bcnt); + else + num_posted = 0; + + return num_posted; +} + static uint64_t lpfc_get_wwpn(struct lpfc_hba *phba) { @@ -3777,111 +4216,6 @@ lpfc_get_wwpn(struct lpfc_hba *phba) } /** - * lpfc_sli4_nvme_sgl_update - update xri-sgl sizing and mapping - * @phba: pointer to lpfc hba data structure. - * - * This routine first calculates the sizes of the current els and allocated - * scsi sgl lists, and then goes through all sgls to updates the physical - * XRIs assigned due to port function reset. During port initialization, the - * current els and allocated scsi sgl lists are 0s. - * - * Return codes - * 0 - successful (for now, it always returns 0) - **/ -int -lpfc_sli4_nvme_sgl_update(struct lpfc_hba *phba) -{ - struct lpfc_nvme_buf *lpfc_ncmd = NULL, *lpfc_ncmd_next = NULL; - uint16_t i, lxri, els_xri_cnt; - uint16_t nvme_xri_cnt, nvme_xri_max; - LIST_HEAD(nvme_sgl_list); - int rc, cnt; - - phba->total_nvme_bufs = 0; - phba->get_nvme_bufs = 0; - phba->put_nvme_bufs = 0; - - if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME)) - return 0; - /* - * update on pci function's allocated nvme xri-sgl list - */ - - /* maximum number of xris available for nvme buffers */ - els_xri_cnt = lpfc_sli4_get_els_iocb_cnt(phba); - nvme_xri_max = phba->sli4_hba.max_cfg_param.max_xri - els_xri_cnt; - phba->sli4_hba.nvme_xri_max = nvme_xri_max; - phba->sli4_hba.nvme_xri_max -= phba->sli4_hba.scsi_xri_max; - - lpfc_printf_log(phba, KERN_INFO, LOG_SLI, - "6074 Current allocated NVME xri-sgl count:%d, " - "maximum NVME xri count:%d\n", - phba->sli4_hba.nvme_xri_cnt, - phba->sli4_hba.nvme_xri_max); - - spin_lock_irq(&phba->nvme_buf_list_get_lock); - spin_lock(&phba->nvme_buf_list_put_lock); - list_splice_init(&phba->lpfc_nvme_buf_list_get, &nvme_sgl_list); - list_splice(&phba->lpfc_nvme_buf_list_put, &nvme_sgl_list); - cnt = phba->get_nvme_bufs + phba->put_nvme_bufs; - phba->get_nvme_bufs = 0; - phba->put_nvme_bufs = 0; - spin_unlock(&phba->nvme_buf_list_put_lock); - spin_unlock_irq(&phba->nvme_buf_list_get_lock); - - if (phba->sli4_hba.nvme_xri_cnt > phba->sli4_hba.nvme_xri_max) { - /* max nvme xri shrunk below the allocated nvme buffers */ - spin_lock_irq(&phba->nvme_buf_list_get_lock); - nvme_xri_cnt = phba->sli4_hba.nvme_xri_cnt - - phba->sli4_hba.nvme_xri_max; - spin_unlock_irq(&phba->nvme_buf_list_get_lock); - /* release the extra allocated nvme buffers */ - for (i = 0; i < nvme_xri_cnt; i++) { - list_remove_head(&nvme_sgl_list, lpfc_ncmd, - struct lpfc_nvme_buf, list); - if (lpfc_ncmd) { - dma_pool_free(phba->lpfc_sg_dma_buf_pool, - lpfc_ncmd->data, - lpfc_ncmd->dma_handle); - kfree(lpfc_ncmd); - } - } - spin_lock_irq(&phba->nvme_buf_list_get_lock); - phba->sli4_hba.nvme_xri_cnt -= nvme_xri_cnt; - spin_unlock_irq(&phba->nvme_buf_list_get_lock); - } - - /* update xris associated to remaining allocated nvme buffers */ - lpfc_ncmd = NULL; - lpfc_ncmd_next = NULL; - list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next, - &nvme_sgl_list, list) { - lxri = lpfc_sli4_next_xritag(phba); - if (lxri == NO_XRI) { - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, - "6075 Failed to allocate xri for " - "nvme buffer\n"); - rc = -ENOMEM; - goto out_free_mem; - } - lpfc_ncmd->cur_iocbq.sli4_lxritag = lxri; - lpfc_ncmd->cur_iocbq.sli4_xritag = phba->sli4_hba.xri_ids[lxri]; - } - spin_lock_irq(&phba->nvme_buf_list_get_lock); - spin_lock(&phba->nvme_buf_list_put_lock); - list_splice_init(&nvme_sgl_list, &phba->lpfc_nvme_buf_list_get); - phba->get_nvme_bufs = cnt; - INIT_LIST_HEAD(&phba->lpfc_nvme_buf_list_put); - spin_unlock(&phba->nvme_buf_list_put_lock); - spin_unlock_irq(&phba->nvme_buf_list_get_lock); - return 0; - -out_free_mem: - lpfc_nvme_free(phba); - return rc; -} - -/** * lpfc_create_port - Create an FC port * @phba: pointer to lpfc hba data structure. * @instance: a unique integer ID to this FC port. @@ -3956,17 +4290,29 @@ lpfc_create_port(struct lpfc_hba *phba, int instance, struct device *dev) vport->fc_rscn_flush = 0; lpfc_get_vport_cfgparam(vport); + /* Adjust value in vport */ + vport->cfg_enable_fc4_type = phba->cfg_enable_fc4_type; + shost->unique_id = instance; shost->max_id = LPFC_MAX_TARGET; shost->max_lun = vport->cfg_max_luns; shost->this_id = -1; shost->max_cmd_len = 16; - shost->nr_hw_queues = phba->cfg_fcp_io_channel; + if (phba->sli_rev == LPFC_SLI_REV4) { + if (phba->cfg_fcp_io_sched == LPFC_FCP_SCHED_BY_HDWQ) + shost->nr_hw_queues = phba->cfg_hdw_queue; + else + shost->nr_hw_queues = phba->sli4_hba.num_present_cpu; + shost->dma_boundary = phba->sli4_hba.pc_sli4_params.sge_supp_len-1; shost->sg_tablesize = phba->cfg_scsi_seg_cnt; - } + } else + /* SLI-3 has a limited number of hardware queues (3), + * thus there is only one for FCP processing. + */ + shost->nr_hw_queues = 1; /* * Set initial can_queue value since 0 is no longer supported and @@ -4220,7 +4566,8 @@ lpfc_stop_port_s4(struct lpfc_hba *phba) { /* Reset some HBA SLI4 setup states */ lpfc_stop_hba_timers(phba); - phba->pport->work_port_events = 0; + if (phba->pport) + phba->pport->work_port_events = 0; phba->sli4_hba.intr_enable = 0; } @@ -5819,24 +6166,11 @@ lpfc_setup_driver_resource_phase1(struct lpfc_hba *phba) "NVME" : " "), (phba->nvmet_support ? "NVMET" : " ")); - if (phba->cfg_enable_fc4_type & LPFC_ENABLE_FCP) { - /* Initialize the scsi buffer list used by driver for scsi IO */ - spin_lock_init(&phba->scsi_buf_list_get_lock); - INIT_LIST_HEAD(&phba->lpfc_scsi_buf_list_get); - spin_lock_init(&phba->scsi_buf_list_put_lock); - INIT_LIST_HEAD(&phba->lpfc_scsi_buf_list_put); - } - - if ((phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) && - (phba->nvmet_support == 0)) { - /* Initialize the NVME buffer list used by driver for NVME IO */ - spin_lock_init(&phba->nvme_buf_list_get_lock); - INIT_LIST_HEAD(&phba->lpfc_nvme_buf_list_get); - phba->get_nvme_bufs = 0; - spin_lock_init(&phba->nvme_buf_list_put_lock); - INIT_LIST_HEAD(&phba->lpfc_nvme_buf_list_put); - phba->put_nvme_bufs = 0; - } + /* Initialize the IO buffer list used by driver for SLI3 SCSI */ + spin_lock_init(&phba->scsi_buf_list_get_lock); + INIT_LIST_HEAD(&phba->lpfc_scsi_buf_list_get); + spin_lock_init(&phba->scsi_buf_list_put_lock); + INIT_LIST_HEAD(&phba->lpfc_scsi_buf_list_put); /* Initialize the fabric iocb list */ INIT_LIST_HEAD(&phba->fabric_iocb_list); @@ -5860,6 +6194,8 @@ lpfc_setup_driver_resource_phase1(struct lpfc_hba *phba) /* Heartbeat timer */ timer_setup(&phba->hb_tmofunc, lpfc_hb_timeout, 0); + INIT_DELAYED_WORK(&phba->eq_delay_work, lpfc_hb_eq_delay_work); + return 0; } @@ -5877,7 +6213,7 @@ lpfc_setup_driver_resource_phase1(struct lpfc_hba *phba) static int lpfc_sli_driver_resource_setup(struct lpfc_hba *phba) { - int rc; + int rc, entry_sz; /* * Initialize timers used by driver @@ -5922,6 +6258,11 @@ lpfc_sli_driver_resource_setup(struct lpfc_hba *phba) lpfc_template_no_hr.sg_tablesize = phba->cfg_sg_seg_cnt; lpfc_template.sg_tablesize = phba->cfg_sg_seg_cnt; + if (phba->sli_rev == LPFC_SLI_REV4) + entry_sz = sizeof(struct sli4_sge); + else + entry_sz = sizeof(struct ulp_bde64); + /* There are going to be 2 reserved BDEs: 1 FCP cmnd + 1 FCP rsp */ if (phba->cfg_enable_bg) { /* @@ -5935,7 +6276,7 @@ lpfc_sli_driver_resource_setup(struct lpfc_hba *phba) */ phba->cfg_sg_dma_buf_size = sizeof(struct fcp_cmnd) + sizeof(struct fcp_rsp) + - (LPFC_MAX_SG_SEG_CNT * sizeof(struct ulp_bde64)); + (LPFC_MAX_SG_SEG_CNT * entry_sz); if (phba->cfg_sg_seg_cnt > LPFC_MAX_SG_SEG_CNT_DIF) phba->cfg_sg_seg_cnt = LPFC_MAX_SG_SEG_CNT_DIF; @@ -5950,7 +6291,7 @@ lpfc_sli_driver_resource_setup(struct lpfc_hba *phba) */ phba->cfg_sg_dma_buf_size = sizeof(struct fcp_cmnd) + sizeof(struct fcp_rsp) + - ((phba->cfg_sg_seg_cnt + 2) * sizeof(struct ulp_bde64)); + ((phba->cfg_sg_seg_cnt + 2) * entry_sz); /* Total BDEs in BPL for scsi_sg_list */ phba->cfg_total_seg_cnt = phba->cfg_sg_seg_cnt + 2; @@ -6031,14 +6372,13 @@ lpfc_sli4_driver_resource_setup(struct lpfc_hba *phba) uint8_t pn_page[LPFC_MAX_SUPPORTED_PAGES] = {0}; struct lpfc_mqe *mqe; int longs; - int fof_vectors = 0; int extra; uint64_t wwn; u32 if_type; u32 if_fam; - phba->sli4_hba.num_online_cpu = num_online_cpus(); phba->sli4_hba.num_present_cpu = lpfc_present_cpu; + phba->sli4_hba.num_possible_cpu = num_possible_cpus(); phba->sli4_hba.curr_disp_cpu = 0; /* Get all the module params for configuring this host */ @@ -6200,8 +6540,7 @@ lpfc_sli4_driver_resource_setup(struct lpfc_hba *phba) if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { /* Initialize the Abort nvme buffer list used by driver */ - spin_lock_init(&phba->sli4_hba.abts_nvme_buf_list_lock); - INIT_LIST_HEAD(&phba->sli4_hba.lpfc_abts_nvme_buf_list); + spin_lock_init(&phba->sli4_hba.abts_nvmet_buf_list_lock); INIT_LIST_HEAD(&phba->sli4_hba.lpfc_abts_nvmet_ctx_list); INIT_LIST_HEAD(&phba->sli4_hba.lpfc_nvmet_io_wait_list); } @@ -6337,6 +6676,8 @@ lpfc_sli4_driver_resource_setup(struct lpfc_hba *phba) " NVME_TARGET_FC infrastructure" " is not in kernel\n"); #endif + /* Not supported for NVMET */ + phba->cfg_xri_rebalancing = 0; break; } } @@ -6405,8 +6746,6 @@ lpfc_sli4_driver_resource_setup(struct lpfc_hba *phba) /* Verify OAS is supported */ lpfc_sli4_oas_verify(phba); - if (phba->cfg_fof) - fof_vectors = 1; /* Verify RAS support on adapter */ lpfc_sli4_ras_init(phba); @@ -6450,9 +6789,9 @@ lpfc_sli4_driver_resource_setup(struct lpfc_hba *phba) goto out_remove_rpi_hdrs; } - phba->sli4_hba.hba_eq_hdl = kcalloc(fof_vectors + phba->io_channel_irqs, - sizeof(struct lpfc_hba_eq_hdl), - GFP_KERNEL); + phba->sli4_hba.hba_eq_hdl = kcalloc(phba->cfg_irq_chann, + sizeof(struct lpfc_hba_eq_hdl), + GFP_KERNEL); if (!phba->sli4_hba.hba_eq_hdl) { lpfc_printf_log(phba, KERN_ERR, LOG_INIT, "2572 Failed allocate memory for " @@ -6461,7 +6800,7 @@ lpfc_sli4_driver_resource_setup(struct lpfc_hba *phba) goto out_free_fcf_rr_bmask; } - phba->sli4_hba.cpu_map = kcalloc(phba->sli4_hba.num_present_cpu, + phba->sli4_hba.cpu_map = kcalloc(phba->sli4_hba.num_possible_cpu, sizeof(struct lpfc_vector_map_info), GFP_KERNEL); if (!phba->sli4_hba.cpu_map) { @@ -6471,21 +6810,14 @@ lpfc_sli4_driver_resource_setup(struct lpfc_hba *phba) rc = -ENOMEM; goto out_free_hba_eq_hdl; } - if (lpfc_used_cpu == NULL) { - lpfc_used_cpu = kcalloc(lpfc_present_cpu, sizeof(uint16_t), - GFP_KERNEL); - if (!lpfc_used_cpu) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, - "3335 Failed allocate memory for msi-x " - "interrupt vector mapping\n"); - kfree(phba->sli4_hba.cpu_map); - rc = -ENOMEM; - goto out_free_hba_eq_hdl; - } - for (i = 0; i < lpfc_present_cpu; i++) - lpfc_used_cpu[i] = LPFC_VECTOR_MAP_EMPTY; - } + phba->sli4_hba.eq_info = alloc_percpu(struct lpfc_eq_intr_info); + if (!phba->sli4_hba.eq_info) { + lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + "3321 Failed allocation for per_cpu stats\n"); + rc = -ENOMEM; + goto out_free_hba_cpu_map; + } /* * Enable sr-iov virtual functions if supported and configured * through the module parameter. @@ -6505,6 +6837,8 @@ lpfc_sli4_driver_resource_setup(struct lpfc_hba *phba) return 0; +out_free_hba_cpu_map: + kfree(phba->sli4_hba.cpu_map); out_free_hba_eq_hdl: kfree(phba->sli4_hba.hba_eq_hdl); out_free_fcf_rr_bmask: @@ -6534,10 +6868,12 @@ lpfc_sli4_driver_resource_unset(struct lpfc_hba *phba) { struct lpfc_fcf_conn_entry *conn_entry, *next_conn_entry; + free_percpu(phba->sli4_hba.eq_info); + /* Free memory allocated for msi-x interrupt vector to CPU mapping */ kfree(phba->sli4_hba.cpu_map); + phba->sli4_hba.num_possible_cpu = 0; phba->sli4_hba.num_present_cpu = 0; - phba->sli4_hba.num_online_cpu = 0; phba->sli4_hba.curr_disp_cpu = 0; /* Free memory allocated for fast-path work queue handles */ @@ -6875,11 +7211,8 @@ lpfc_init_sgl_list(struct lpfc_hba *phba) /* els xri-sgl book keeping */ phba->sli4_hba.els_xri_cnt = 0; - /* scsi xri-buffer book keeping */ - phba->sli4_hba.scsi_xri_cnt = 0; - /* nvme xri-buffer book keeping */ - phba->sli4_hba.nvme_xri_cnt = 0; + phba->sli4_hba.io_xri_cnt = 0; } /** @@ -7093,6 +7426,9 @@ lpfc_hba_alloc(struct pci_dev *pdev) static void lpfc_hba_free(struct lpfc_hba *phba) { + if (phba->sli_rev == LPFC_SLI_REV4) + kfree(phba->sli4_hba.hdwq); + /* Release the driver assigned board number */ idr_remove(&lpfc_hba_index, phba->brd_no); @@ -7128,10 +7464,6 @@ lpfc_create_shost(struct lpfc_hba *phba) phba->fc_arbtov = FF_DEF_ARBTOV; atomic_set(&phba->sdev_cnt, 0); - atomic_set(&phba->fc4ScsiInputRequests, 0); - atomic_set(&phba->fc4ScsiOutputRequests, 0); - atomic_set(&phba->fc4ScsiControlRequests, 0); - atomic_set(&phba->fc4ScsiIoCmpls, 0); vport = lpfc_create_port(phba, phba->brd_no, &phba->pcidev->dev); if (!vport) return -ENODEV; @@ -7909,7 +8241,7 @@ lpfc_sli4_read_config(struct lpfc_hba *phba) struct lpfc_rsrc_desc_fcfcoe *desc; char *pdesc_0; uint16_t forced_link_speed; - uint32_t if_type; + uint32_t if_type, qmin; int length, i, rc = 0, rc2; pmb = (LPFC_MBOXQ_t *) mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL); @@ -8014,38 +8346,44 @@ lpfc_sli4_read_config(struct lpfc_hba *phba) phba->sli4_hba.max_cfg_param.max_rq); /* - * Calculate NVME queue resources based on how - * many WQ/CQs are available. + * Calculate queue resources based on how + * many WQ/CQ/EQs are available. */ - if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { - length = phba->sli4_hba.max_cfg_param.max_wq; - if (phba->sli4_hba.max_cfg_param.max_cq < - phba->sli4_hba.max_cfg_param.max_wq) - length = phba->sli4_hba.max_cfg_param.max_cq; + qmin = phba->sli4_hba.max_cfg_param.max_wq; + if (phba->sli4_hba.max_cfg_param.max_cq < qmin) + qmin = phba->sli4_hba.max_cfg_param.max_cq; + if (phba->sli4_hba.max_cfg_param.max_eq < qmin) + qmin = phba->sli4_hba.max_cfg_param.max_eq; + /* + * Whats left after this can go toward NVME / FCP. + * The minus 4 accounts for ELS, NVME LS, MBOX + * plus one extra. When configured for + * NVMET, FCP io channel WQs are not created. + */ + qmin -= 4; - /* - * Whats left after this can go toward NVME. - * The minus 6 accounts for ELS, NVME LS, MBOX - * fof plus a couple extra. When configured for - * NVMET, FCP io channel WQs are not created. - */ - length -= 6; - if (!phba->nvmet_support) - length -= phba->cfg_fcp_io_channel; - - if (phba->cfg_nvme_io_channel > length) { - lpfc_printf_log( - phba, KERN_ERR, LOG_SLI, - "2005 Reducing NVME IO channel to %d: " - "WQ %d CQ %d NVMEIO %d FCPIO %d\n", - length, + /* If NVME is configured, double the number of CQ/WQs needed */ + if ((phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) && + !phba->nvmet_support) + qmin /= 2; + + /* Check to see if there is enough for NVME */ + if ((phba->cfg_irq_chann > qmin) || + (phba->cfg_hdw_queue > qmin)) { + lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + "2005 Reducing Queues: " + "WQ %d CQ %d EQ %d: min %d: " + "IRQ %d HDWQ %d\n", phba->sli4_hba.max_cfg_param.max_wq, phba->sli4_hba.max_cfg_param.max_cq, - phba->cfg_nvme_io_channel, - phba->cfg_fcp_io_channel); + phba->sli4_hba.max_cfg_param.max_eq, + qmin, phba->cfg_irq_chann, + phba->cfg_hdw_queue); - phba->cfg_nvme_io_channel = length; - } + if (phba->cfg_irq_chann > qmin) + phba->cfg_irq_chann = qmin; + if (phba->cfg_hdw_queue > qmin) + phba->cfg_hdw_queue = qmin; } } @@ -8257,53 +8595,22 @@ lpfc_setup_endian_order(struct lpfc_hba *phba) static int lpfc_sli4_queue_verify(struct lpfc_hba *phba) { - int io_channel; - int fof_vectors = phba->cfg_fof ? 1 : 0; - /* * Sanity check for configured queue parameters against the run-time * device parameters */ - /* Sanity check on HBA EQ parameters */ - io_channel = phba->io_channel_irqs; - - if (phba->sli4_hba.num_online_cpu < io_channel) { - lpfc_printf_log(phba, - KERN_ERR, LOG_INIT, - "3188 Reducing IO channels to match number of " - "online CPUs: from %d to %d\n", - io_channel, phba->sli4_hba.num_online_cpu); - io_channel = phba->sli4_hba.num_online_cpu; - } - - if (io_channel + fof_vectors > phba->sli4_hba.max_cfg_param.max_eq) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, - "2575 Reducing IO channels to match number of " - "available EQs: from %d to %d\n", - io_channel, - phba->sli4_hba.max_cfg_param.max_eq); - io_channel = phba->sli4_hba.max_cfg_param.max_eq - fof_vectors; - } - - /* The actual number of FCP / NVME event queues adopted */ - if (io_channel != phba->io_channel_irqs) - phba->io_channel_irqs = io_channel; - if (phba->cfg_fcp_io_channel > io_channel) - phba->cfg_fcp_io_channel = io_channel; - if (phba->cfg_nvme_io_channel > io_channel) - phba->cfg_nvme_io_channel = io_channel; if (phba->nvmet_support) { - if (phba->cfg_nvme_io_channel < phba->cfg_nvmet_mrq) - phba->cfg_nvmet_mrq = phba->cfg_nvme_io_channel; + if (phba->cfg_irq_chann < phba->cfg_nvmet_mrq) + phba->cfg_nvmet_mrq = phba->cfg_irq_chann; } if (phba->cfg_nvmet_mrq > LPFC_NVMET_MRQ_MAX) phba->cfg_nvmet_mrq = LPFC_NVMET_MRQ_MAX; lpfc_printf_log(phba, KERN_ERR, LOG_INIT, - "2574 IO channels: irqs %d fcp %d nvme %d MRQ: %d\n", - phba->io_channel_irqs, phba->cfg_fcp_io_channel, - phba->cfg_nvme_io_channel, phba->cfg_nvmet_mrq); + "2574 IO channels: hdwQ %d IRQ %d MRQ: %d\n", + phba->cfg_hdw_queue, phba->cfg_irq_chann, + phba->cfg_nvmet_mrq); /* Get EQ depth from module parameter, fake the default for now */ phba->sli4_hba.eq_esize = LPFC_EQE_SIZE_4B; @@ -8330,7 +8637,9 @@ lpfc_alloc_nvme_wq_cq(struct lpfc_hba *phba, int wqidx) return 1; } qdesc->qe_valid = 1; - phba->sli4_hba.nvme_cq[wqidx] = qdesc; + qdesc->hdwq = wqidx; + qdesc->chann = lpfc_find_cpu_handle(phba, wqidx, LPFC_FIND_BY_HDWQ); + phba->sli4_hba.hdwq[wqidx].nvme_cq = qdesc; qdesc = lpfc_sli4_queue_alloc(phba, LPFC_EXPANDED_PAGE_SIZE, LPFC_WQE128_SIZE, LPFC_WQE_EXP_COUNT); @@ -8340,7 +8649,9 @@ lpfc_alloc_nvme_wq_cq(struct lpfc_hba *phba, int wqidx) wqidx); return 1; } - phba->sli4_hba.nvme_wq[wqidx] = qdesc; + qdesc->hdwq = wqidx; + qdesc->chann = wqidx; + phba->sli4_hba.hdwq[wqidx].nvme_wq = qdesc; list_add_tail(&qdesc->wq_list, &phba->sli4_hba.lpfc_wq_list); return 0; } @@ -8368,7 +8679,9 @@ lpfc_alloc_fcp_wq_cq(struct lpfc_hba *phba, int wqidx) return 1; } qdesc->qe_valid = 1; - phba->sli4_hba.fcp_cq[wqidx] = qdesc; + qdesc->hdwq = wqidx; + qdesc->chann = lpfc_find_cpu_handle(phba, wqidx, LPFC_FIND_BY_HDWQ); + phba->sli4_hba.hdwq[wqidx].fcp_cq = qdesc; /* Create Fast Path FCP WQs */ if (phba->enab_exp_wqcq_pages) { @@ -8389,7 +8702,9 @@ lpfc_alloc_fcp_wq_cq(struct lpfc_hba *phba, int wqidx) wqidx); return 1; } - phba->sli4_hba.fcp_wq[wqidx] = qdesc; + qdesc->hdwq = wqidx; + qdesc->chann = wqidx; + phba->sli4_hba.hdwq[wqidx].fcp_wq = qdesc; list_add_tail(&qdesc->wq_list, &phba->sli4_hba.lpfc_wq_list); return 0; } @@ -8412,16 +8727,14 @@ int lpfc_sli4_queue_create(struct lpfc_hba *phba) { struct lpfc_queue *qdesc; - int idx, io_channel; + int idx, eqidx; + struct lpfc_sli4_hdw_queue *qp; + struct lpfc_eq_intr_info *eqi; /* * Create HBA Record arrays. * Both NVME and FCP will share that same vectors / EQs */ - io_channel = phba->io_channel_irqs; - if (!io_channel) - return -ERANGE; - phba->sli4_hba.mq_esize = LPFC_MQE_SIZE; phba->sli4_hba.mq_ecount = LPFC_MQE_DEF_COUNT; phba->sli4_hba.wq_esize = LPFC_WQE_SIZE; @@ -8433,87 +8746,36 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba) phba->sli4_hba.cq_esize = LPFC_CQE_SIZE; phba->sli4_hba.cq_ecount = LPFC_CQE_DEF_COUNT; - phba->sli4_hba.hba_eq = kcalloc(io_channel, - sizeof(struct lpfc_queue *), - GFP_KERNEL); - if (!phba->sli4_hba.hba_eq) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, - "2576 Failed allocate memory for " - "fast-path EQ record array\n"); - goto out_error; - } - - if (phba->cfg_fcp_io_channel) { - phba->sli4_hba.fcp_cq = kcalloc(phba->cfg_fcp_io_channel, - sizeof(struct lpfc_queue *), - GFP_KERNEL); - if (!phba->sli4_hba.fcp_cq) { + if (!phba->sli4_hba.hdwq) { + phba->sli4_hba.hdwq = kcalloc( + phba->cfg_hdw_queue, sizeof(struct lpfc_sli4_hdw_queue), + GFP_KERNEL); + if (!phba->sli4_hba.hdwq) { lpfc_printf_log(phba, KERN_ERR, LOG_INIT, - "2577 Failed allocate memory for " - "fast-path CQ record array\n"); + "6427 Failed allocate memory for " + "fast-path Hardware Queue array\n"); goto out_error; } - phba->sli4_hba.fcp_wq = kcalloc(phba->cfg_fcp_io_channel, - sizeof(struct lpfc_queue *), - GFP_KERNEL); - if (!phba->sli4_hba.fcp_wq) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, - "2578 Failed allocate memory for " - "fast-path FCP WQ record array\n"); - goto out_error; - } - /* - * Since the first EQ can have multiple CQs associated with it, - * this array is used to quickly see if we have a FCP fast-path - * CQ match. - */ - phba->sli4_hba.fcp_cq_map = kcalloc(phba->cfg_fcp_io_channel, - sizeof(uint16_t), - GFP_KERNEL); - if (!phba->sli4_hba.fcp_cq_map) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, - "2545 Failed allocate memory for " - "fast-path CQ map\n"); - goto out_error; + /* Prepare hardware queues to take IO buffers */ + for (idx = 0; idx < phba->cfg_hdw_queue; idx++) { + qp = &phba->sli4_hba.hdwq[idx]; + spin_lock_init(&qp->io_buf_list_get_lock); + spin_lock_init(&qp->io_buf_list_put_lock); + INIT_LIST_HEAD(&qp->lpfc_io_buf_list_get); + INIT_LIST_HEAD(&qp->lpfc_io_buf_list_put); + qp->get_io_bufs = 0; + qp->put_io_bufs = 0; + qp->total_io_bufs = 0; + spin_lock_init(&qp->abts_scsi_buf_list_lock); + INIT_LIST_HEAD(&qp->lpfc_abts_scsi_buf_list); + qp->abts_scsi_io_bufs = 0; + spin_lock_init(&qp->abts_nvme_buf_list_lock); + INIT_LIST_HEAD(&qp->lpfc_abts_nvme_buf_list); + qp->abts_nvme_io_bufs = 0; } } - if (phba->cfg_nvme_io_channel) { - phba->sli4_hba.nvme_cq = kcalloc(phba->cfg_nvme_io_channel, - sizeof(struct lpfc_queue *), - GFP_KERNEL); - if (!phba->sli4_hba.nvme_cq) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, - "6077 Failed allocate memory for " - "fast-path CQ record array\n"); - goto out_error; - } - - phba->sli4_hba.nvme_wq = kcalloc(phba->cfg_nvme_io_channel, - sizeof(struct lpfc_queue *), - GFP_KERNEL); - if (!phba->sli4_hba.nvme_wq) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, - "2581 Failed allocate memory for " - "fast-path NVME WQ record array\n"); - goto out_error; - } - - /* - * Since the first EQ can have multiple CQs associated with it, - * this array is used to quickly see if we have a NVME fast-path - * CQ match. - */ - phba->sli4_hba.nvme_cq_map = kcalloc(phba->cfg_nvme_io_channel, - sizeof(uint16_t), - GFP_KERNEL); - if (!phba->sli4_hba.nvme_cq_map) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, - "6078 Failed allocate memory for " - "fast-path CQ map\n"); - goto out_error; - } - + if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { if (phba->nvmet_support) { phba->sli4_hba.nvmet_cqset = kcalloc( phba->cfg_nvmet_mrq, @@ -8551,8 +8813,19 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba) INIT_LIST_HEAD(&phba->sli4_hba.lpfc_wq_list); /* Create HBA Event Queues (EQs) */ - for (idx = 0; idx < io_channel; idx++) { - /* Create EQs */ + for (idx = 0; idx < phba->cfg_hdw_queue; idx++) { + /* + * If there are more Hardware Queues than available + * CQs, multiple Hardware Queues may share a common EQ. + */ + if (idx >= phba->cfg_irq_chann) { + /* Share an existing EQ */ + eqidx = lpfc_find_eq_handle(phba, idx); + phba->sli4_hba.hdwq[idx].hba_eq = + phba->sli4_hba.hdwq[eqidx].hba_eq; + continue; + } + /* Create an EQ */ qdesc = lpfc_sli4_queue_alloc(phba, LPFC_DEFAULT_PAGE_SIZE, phba->sli4_hba.eq_esize, phba->sli4_hba.eq_ecount); @@ -8562,33 +8835,51 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba) goto out_error; } qdesc->qe_valid = 1; - phba->sli4_hba.hba_eq[idx] = qdesc; + qdesc->hdwq = idx; + + /* Save the CPU this EQ is affinitised to */ + eqidx = lpfc_find_eq_handle(phba, idx); + qdesc->chann = lpfc_find_cpu_handle(phba, eqidx, + LPFC_FIND_BY_EQ); + phba->sli4_hba.hdwq[idx].hba_eq = qdesc; + qdesc->last_cpu = qdesc->chann; + eqi = per_cpu_ptr(phba->sli4_hba.eq_info, qdesc->last_cpu); + list_add(&qdesc->cpu_list, &eqi->list); } - /* FCP and NVME io channels are not required to be balanced */ - for (idx = 0; idx < phba->cfg_fcp_io_channel; idx++) + /* Allocate SCSI SLI4 CQ/WQs */ + for (idx = 0; idx < phba->cfg_hdw_queue; idx++) { if (lpfc_alloc_fcp_wq_cq(phba, idx)) goto out_error; + } - for (idx = 0; idx < phba->cfg_nvme_io_channel; idx++) - if (lpfc_alloc_nvme_wq_cq(phba, idx)) - goto out_error; + /* Allocate NVME SLI4 CQ/WQs */ + if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { + for (idx = 0; idx < phba->cfg_hdw_queue; idx++) { + if (lpfc_alloc_nvme_wq_cq(phba, idx)) + goto out_error; + } - if (phba->nvmet_support) { - for (idx = 0; idx < phba->cfg_nvmet_mrq; idx++) { - qdesc = lpfc_sli4_queue_alloc(phba, + if (phba->nvmet_support) { + for (idx = 0; idx < phba->cfg_nvmet_mrq; idx++) { + qdesc = lpfc_sli4_queue_alloc( + phba, LPFC_DEFAULT_PAGE_SIZE, phba->sli4_hba.cq_esize, phba->sli4_hba.cq_ecount); - if (!qdesc) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, - "3142 Failed allocate NVME " - "CQ Set (%d)\n", idx); - goto out_error; + if (!qdesc) { + lpfc_printf_log( + phba, KERN_ERR, LOG_INIT, + "3142 Failed allocate NVME " + "CQ Set (%d)\n", idx); + goto out_error; + } + qdesc->qe_valid = 1; + qdesc->hdwq = idx; + qdesc->chann = idx; + phba->sli4_hba.nvmet_cqset[idx] = qdesc; } - qdesc->qe_valid = 1; - phba->sli4_hba.nvmet_cqset[idx] = qdesc; } } @@ -8618,6 +8909,7 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba) goto out_error; } qdesc->qe_valid = 1; + qdesc->chann = 0; phba->sli4_hba.els_cq = qdesc; @@ -8635,6 +8927,7 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba) "0505 Failed allocate slow-path MQ\n"); goto out_error; } + qdesc->chann = 0; phba->sli4_hba.mbx_wq = qdesc; /* @@ -8650,6 +8943,7 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba) "0504 Failed allocate slow-path ELS WQ\n"); goto out_error; } + qdesc->chann = 0; phba->sli4_hba.els_wq = qdesc; list_add_tail(&qdesc->wq_list, &phba->sli4_hba.lpfc_wq_list); @@ -8663,6 +8957,7 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba) "6079 Failed allocate NVME LS CQ\n"); goto out_error; } + qdesc->chann = 0; qdesc->qe_valid = 1; phba->sli4_hba.nvmels_cq = qdesc; @@ -8675,6 +8970,7 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba) "6080 Failed allocate NVME LS WQ\n"); goto out_error; } + qdesc->chann = 0; phba->sli4_hba.nvmels_wq = qdesc; list_add_tail(&qdesc->wq_list, &phba->sli4_hba.lpfc_wq_list); } @@ -8705,7 +9001,8 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba) } phba->sli4_hba.dat_rq = qdesc; - if (phba->nvmet_support) { + if ((phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) && + phba->nvmet_support) { for (idx = 0; idx < phba->cfg_nvmet_mrq; idx++) { /* Create NVMET Receive Queue for header */ qdesc = lpfc_sli4_queue_alloc(phba, @@ -8718,6 +9015,7 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba) "receive HRQ\n"); goto out_error; } + qdesc->hdwq = idx; phba->sli4_hba.nvmet_mrq_hdr[idx] = qdesc; /* Only needed for header of RQ pair */ @@ -8744,13 +9042,29 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba) "receive DRQ\n"); goto out_error; } + qdesc->hdwq = idx; phba->sli4_hba.nvmet_mrq_data[idx] = qdesc; } } - /* Create the Queues needed for Flash Optimized Fabric operations */ - if (phba->cfg_fof) - lpfc_fof_queue_create(phba); +#if defined(BUILD_NVME) + /* Clear NVME stats */ + if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { + for (idx = 0; idx < phba->cfg_hdw_queue; idx++) { + memset(&phba->sli4_hba.hdwq[idx].nvme_cstat, 0, + sizeof(phba->sli4_hba.hdwq[idx].nvme_cstat)); + } + } +#endif + + /* Clear SCSI stats */ + if (phba->cfg_enable_fc4_type & LPFC_ENABLE_FCP) { + for (idx = 0; idx < phba->cfg_hdw_queue; idx++) { + memset(&phba->sli4_hba.hdwq[idx].scsi_cstat, 0, + sizeof(phba->sli4_hba.hdwq[idx].scsi_cstat)); + } + } + return 0; out_error: @@ -8783,11 +9097,25 @@ lpfc_sli4_release_queues(struct lpfc_queue ***qs, int max) } static inline void -lpfc_sli4_release_queue_map(uint16_t **qmap) +lpfc_sli4_release_hdwq(struct lpfc_hba *phba) { - if (*qmap != NULL) { - kfree(*qmap); - *qmap = NULL; + struct lpfc_sli4_hdw_queue *hdwq; + uint32_t idx; + + hdwq = phba->sli4_hba.hdwq; + for (idx = 0; idx < phba->cfg_hdw_queue; idx++) { + if (idx < phba->cfg_irq_chann) + lpfc_sli4_queue_free(hdwq[idx].hba_eq); + hdwq[idx].hba_eq = NULL; + + lpfc_sli4_queue_free(hdwq[idx].fcp_cq); + lpfc_sli4_queue_free(hdwq[idx].nvme_cq); + lpfc_sli4_queue_free(hdwq[idx].fcp_wq); + lpfc_sli4_queue_free(hdwq[idx].nvme_wq); + hdwq[idx].fcp_cq = NULL; + hdwq[idx].nvme_cq = NULL; + hdwq[idx].fcp_wq = NULL; + hdwq[idx].nvme_wq = NULL; } } @@ -8806,33 +9134,9 @@ lpfc_sli4_release_queue_map(uint16_t **qmap) void lpfc_sli4_queue_destroy(struct lpfc_hba *phba) { - if (phba->cfg_fof) - lpfc_fof_queue_destroy(phba); - /* Release HBA eqs */ - lpfc_sli4_release_queues(&phba->sli4_hba.hba_eq, phba->io_channel_irqs); - - /* Release FCP cqs */ - lpfc_sli4_release_queues(&phba->sli4_hba.fcp_cq, - phba->cfg_fcp_io_channel); - - /* Release FCP wqs */ - lpfc_sli4_release_queues(&phba->sli4_hba.fcp_wq, - phba->cfg_fcp_io_channel); - - /* Release FCP CQ mapping array */ - lpfc_sli4_release_queue_map(&phba->sli4_hba.fcp_cq_map); - - /* Release NVME cqs */ - lpfc_sli4_release_queues(&phba->sli4_hba.nvme_cq, - phba->cfg_nvme_io_channel); - - /* Release NVME wqs */ - lpfc_sli4_release_queues(&phba->sli4_hba.nvme_wq, - phba->cfg_nvme_io_channel); - - /* Release NVME CQ mapping array */ - lpfc_sli4_release_queue_map(&phba->sli4_hba.nvme_cq_map); + if (phba->sli4_hba.hdwq) + lpfc_sli4_release_hdwq(phba); if (phba->nvmet_support) { lpfc_sli4_release_queues(&phba->sli4_hba.nvmet_cqset, @@ -8913,10 +9217,9 @@ lpfc_create_wq_cq(struct lpfc_hba *phba, struct lpfc_queue *eq, qidx, (uint32_t)rc); return rc; } - cq->chann = qidx; if (qtype != LPFC_MBOX) { - /* Setup nvme_cq_map for fast lookup */ + /* Setup cq_map for fast lookup */ if (cq_map) *cq_map = cq->queue_id; @@ -8933,7 +9236,6 @@ lpfc_create_wq_cq(struct lpfc_hba *phba, struct lpfc_queue *eq, /* no need to tear down cq - caller will do so */ return rc; } - wq->chann = qidx; /* Bind this CQ/WQ to the NVME ring */ pring = wq->pring; @@ -8963,6 +9265,38 @@ lpfc_create_wq_cq(struct lpfc_hba *phba, struct lpfc_queue *eq, } /** + * lpfc_setup_cq_lookup - Setup the CQ lookup table + * @phba: pointer to lpfc hba data structure. + * + * This routine will populate the cq_lookup table by all + * available CQ queue_id's. + **/ +void +lpfc_setup_cq_lookup(struct lpfc_hba *phba) +{ + struct lpfc_queue *eq, *childq; + struct lpfc_sli4_hdw_queue *qp; + int qidx; + + qp = phba->sli4_hba.hdwq; + memset(phba->sli4_hba.cq_lookup, 0, + (sizeof(struct lpfc_queue *) * (phba->sli4_hba.cq_max + 1))); + for (qidx = 0; qidx < phba->cfg_irq_chann; qidx++) { + eq = qp[qidx].hba_eq; + if (!eq) + continue; + list_for_each_entry(childq, &eq->child_list, list) { + if (childq->queue_id > phba->sli4_hba.cq_max) + continue; + if ((childq->subtype == LPFC_FCP) || + (childq->subtype == LPFC_NVME)) + phba->sli4_hba.cq_lookup[childq->queue_id] = + childq; + } + } +} + +/** * lpfc_sli4_queue_setup - Set up all the SLI4 queues * @phba: pointer to lpfc hba data structure. * @@ -8979,9 +9313,10 @@ lpfc_sli4_queue_setup(struct lpfc_hba *phba) { uint32_t shdr_status, shdr_add_status; union lpfc_sli4_cfg_shdr *shdr; + struct lpfc_sli4_hdw_queue *qp; LPFC_MBOXQ_t *mboxq; int qidx; - uint32_t length, io_channel; + uint32_t length, usdelay; int rc = -ENOMEM; /* Check for dual-ULP support */ @@ -9032,25 +9367,25 @@ lpfc_sli4_queue_setup(struct lpfc_hba *phba) /* * Set up HBA Event Queues (EQs) */ - io_channel = phba->io_channel_irqs; + qp = phba->sli4_hba.hdwq; /* Set up HBA event queue */ - if (io_channel && !phba->sli4_hba.hba_eq) { + if (!qp) { lpfc_printf_log(phba, KERN_ERR, LOG_INIT, "3147 Fast-path EQs not allocated\n"); rc = -ENOMEM; goto out_error; } - for (qidx = 0; qidx < io_channel; qidx++) { - if (!phba->sli4_hba.hba_eq[qidx]) { + for (qidx = 0; qidx < phba->cfg_irq_chann; qidx++) { + if (!qp[qidx].hba_eq) { lpfc_printf_log(phba, KERN_ERR, LOG_INIT, "0522 Fast-path EQ (%d) not " "allocated\n", qidx); rc = -ENOMEM; goto out_destroy; } - rc = lpfc_eq_create(phba, phba->sli4_hba.hba_eq[qidx], - phba->cfg_fcp_imax); + rc = lpfc_eq_create(phba, qp[qidx].hba_eq, + phba->cfg_fcp_imax); if (rc) { lpfc_printf_log(phba, KERN_ERR, LOG_INIT, "0523 Failed setup of fast-path EQ " @@ -9059,26 +9394,17 @@ lpfc_sli4_queue_setup(struct lpfc_hba *phba) goto out_destroy; } lpfc_printf_log(phba, KERN_INFO, LOG_INIT, - "2584 HBA EQ setup: queue[%d]-id=%d\n", - qidx, phba->sli4_hba.hba_eq[qidx]->queue_id); + "2584 HBA EQ setup: queue[%d]-id=%d\n", qidx, + qp[qidx].hba_eq->queue_id); } - if (phba->cfg_nvme_io_channel) { - if (!phba->sli4_hba.nvme_cq || !phba->sli4_hba.nvme_wq) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, - "6084 Fast-path NVME %s array not allocated\n", - (phba->sli4_hba.nvme_cq) ? "CQ" : "WQ"); - rc = -ENOMEM; - goto out_destroy; - } - - for (qidx = 0; qidx < phba->cfg_nvme_io_channel; qidx++) { + if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { + for (qidx = 0; qidx < phba->cfg_hdw_queue; qidx++) { rc = lpfc_create_wq_cq(phba, - phba->sli4_hba.hba_eq[ - qidx % io_channel], - phba->sli4_hba.nvme_cq[qidx], - phba->sli4_hba.nvme_wq[qidx], - &phba->sli4_hba.nvme_cq_map[qidx], + qp[qidx].hba_eq, + qp[qidx].nvme_cq, + qp[qidx].nvme_wq, + &phba->sli4_hba.hdwq[qidx].nvme_cq_map, qidx, LPFC_NVME); if (rc) { lpfc_printf_log(phba, KERN_ERR, LOG_INIT, @@ -9090,31 +9416,19 @@ lpfc_sli4_queue_setup(struct lpfc_hba *phba) } } - if (phba->cfg_fcp_io_channel) { - /* Set up fast-path FCP Response Complete Queue */ - if (!phba->sli4_hba.fcp_cq || !phba->sli4_hba.fcp_wq) { + for (qidx = 0; qidx < phba->cfg_hdw_queue; qidx++) { + rc = lpfc_create_wq_cq(phba, + qp[qidx].hba_eq, + qp[qidx].fcp_cq, + qp[qidx].fcp_wq, + &phba->sli4_hba.hdwq[qidx].fcp_cq_map, + qidx, LPFC_FCP); + if (rc) { lpfc_printf_log(phba, KERN_ERR, LOG_INIT, - "3148 Fast-path FCP %s array not allocated\n", - phba->sli4_hba.fcp_cq ? "WQ" : "CQ"); - rc = -ENOMEM; - goto out_destroy; - } - - for (qidx = 0; qidx < phba->cfg_fcp_io_channel; qidx++) { - rc = lpfc_create_wq_cq(phba, - phba->sli4_hba.hba_eq[ - qidx % io_channel], - phba->sli4_hba.fcp_cq[qidx], - phba->sli4_hba.fcp_wq[qidx], - &phba->sli4_hba.fcp_cq_map[qidx], - qidx, LPFC_FCP); - if (rc) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, "0535 Failed to setup fastpath " "FCP WQ/CQ (%d), rc = 0x%x\n", qidx, (uint32_t)rc); - goto out_destroy; - } + goto out_destroy; } } @@ -9133,7 +9447,7 @@ lpfc_sli4_queue_setup(struct lpfc_hba *phba) goto out_destroy; } - rc = lpfc_create_wq_cq(phba, phba->sli4_hba.hba_eq[0], + rc = lpfc_create_wq_cq(phba, qp[0].hba_eq, phba->sli4_hba.mbx_cq, phba->sli4_hba.mbx_wq, NULL, 0, LPFC_MBOX); @@ -9154,7 +9468,7 @@ lpfc_sli4_queue_setup(struct lpfc_hba *phba) if (phba->cfg_nvmet_mrq > 1) { rc = lpfc_cq_create_set(phba, phba->sli4_hba.nvmet_cqset, - phba->sli4_hba.hba_eq, + qp, LPFC_WCQ, LPFC_NVMET); if (rc) { lpfc_printf_log(phba, KERN_ERR, LOG_INIT, @@ -9166,7 +9480,7 @@ lpfc_sli4_queue_setup(struct lpfc_hba *phba) } else { /* Set up NVMET Receive Complete Queue */ rc = lpfc_cq_create(phba, phba->sli4_hba.nvmet_cqset[0], - phba->sli4_hba.hba_eq[0], + qp[0].hba_eq, LPFC_WCQ, LPFC_NVMET); if (rc) { lpfc_printf_log(phba, KERN_ERR, LOG_INIT, @@ -9180,7 +9494,7 @@ lpfc_sli4_queue_setup(struct lpfc_hba *phba) "6090 NVMET CQ setup: cq-id=%d, " "parent eq-id=%d\n", phba->sli4_hba.nvmet_cqset[0]->queue_id, - phba->sli4_hba.hba_eq[0]->queue_id); + qp[0].hba_eq->queue_id); } } @@ -9192,14 +9506,14 @@ lpfc_sli4_queue_setup(struct lpfc_hba *phba) rc = -ENOMEM; goto out_destroy; } - rc = lpfc_create_wq_cq(phba, phba->sli4_hba.hba_eq[0], - phba->sli4_hba.els_cq, - phba->sli4_hba.els_wq, - NULL, 0, LPFC_ELS); + rc = lpfc_create_wq_cq(phba, qp[0].hba_eq, + phba->sli4_hba.els_cq, + phba->sli4_hba.els_wq, + NULL, 0, LPFC_ELS); if (rc) { lpfc_printf_log(phba, KERN_ERR, LOG_INIT, - "0529 Failed setup of ELS WQ/CQ: rc = 0x%x\n", - (uint32_t)rc); + "0525 Failed setup of ELS WQ/CQ: rc = 0x%x\n", + (uint32_t)rc); goto out_destroy; } lpfc_printf_log(phba, KERN_INFO, LOG_INIT, @@ -9207,7 +9521,7 @@ lpfc_sli4_queue_setup(struct lpfc_hba *phba) phba->sli4_hba.els_wq->queue_id, phba->sli4_hba.els_cq->queue_id); - if (phba->cfg_nvme_io_channel) { + if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { /* Set up NVME LS Complete Queue */ if (!phba->sli4_hba.nvmels_cq || !phba->sli4_hba.nvmels_wq) { lpfc_printf_log(phba, KERN_ERR, LOG_INIT, @@ -9216,14 +9530,14 @@ lpfc_sli4_queue_setup(struct lpfc_hba *phba) rc = -ENOMEM; goto out_destroy; } - rc = lpfc_create_wq_cq(phba, phba->sli4_hba.hba_eq[0], - phba->sli4_hba.nvmels_cq, - phba->sli4_hba.nvmels_wq, - NULL, 0, LPFC_NVME_LS); + rc = lpfc_create_wq_cq(phba, qp[0].hba_eq, + phba->sli4_hba.nvmels_cq, + phba->sli4_hba.nvmels_wq, + NULL, 0, LPFC_NVME_LS); if (rc) { lpfc_printf_log(phba, KERN_ERR, LOG_INIT, - "0529 Failed setup of NVVME LS WQ/CQ: " - "rc = 0x%x\n", (uint32_t)rc); + "0526 Failed setup of NVVME LS WQ/CQ: " + "rc = 0x%x\n", (uint32_t)rc); goto out_destroy; } @@ -9309,20 +9623,29 @@ lpfc_sli4_queue_setup(struct lpfc_hba *phba) phba->sli4_hba.dat_rq->queue_id, phba->sli4_hba.els_cq->queue_id); - if (phba->cfg_fof) { - rc = lpfc_fof_queue_setup(phba); - if (rc) { + if (phba->cfg_fcp_imax) + usdelay = LPFC_SEC_TO_USEC / phba->cfg_fcp_imax; + else + usdelay = 0; + + for (qidx = 0; qidx < phba->cfg_irq_chann; + qidx += LPFC_MAX_EQ_DELAY_EQID_CNT) + lpfc_modify_hba_eq_delay(phba, qidx, LPFC_MAX_EQ_DELAY_EQID_CNT, + usdelay); + + if (phba->sli4_hba.cq_max) { + kfree(phba->sli4_hba.cq_lookup); + phba->sli4_hba.cq_lookup = kcalloc((phba->sli4_hba.cq_max + 1), + sizeof(struct lpfc_queue *), GFP_KERNEL); + if (!phba->sli4_hba.cq_lookup) { lpfc_printf_log(phba, KERN_ERR, LOG_INIT, - "0549 Failed setup of FOF Queues: " - "rc = 0x%x\n", rc); + "0549 Failed setup of CQ Lookup table: " + "size 0x%x\n", phba->sli4_hba.cq_max); + rc = -ENOMEM; goto out_destroy; } + lpfc_setup_cq_lookup(phba); } - - for (qidx = 0; qidx < io_channel; qidx += LPFC_MAX_EQ_DELAY_EQID_CNT) - lpfc_modify_hba_eq_delay(phba, qidx, LPFC_MAX_EQ_DELAY_EQID_CNT, - phba->cfg_fcp_imax); - return 0; out_destroy: @@ -9346,12 +9669,9 @@ out_error: void lpfc_sli4_queue_unset(struct lpfc_hba *phba) { + struct lpfc_sli4_hdw_queue *qp; int qidx; - /* Unset the queues created for Flash Optimized Fabric operations */ - if (phba->cfg_fof) - lpfc_fof_queue_destroy(phba); - /* Unset mailbox command work queue */ if (phba->sli4_hba.mbx_wq) lpfc_mq_destroy(phba, phba->sli4_hba.mbx_wq); @@ -9369,17 +9689,6 @@ lpfc_sli4_queue_unset(struct lpfc_hba *phba) lpfc_rq_destroy(phba, phba->sli4_hba.hdr_rq, phba->sli4_hba.dat_rq); - /* Unset FCP work queue */ - if (phba->sli4_hba.fcp_wq) - for (qidx = 0; qidx < phba->cfg_fcp_io_channel; qidx++) - lpfc_wq_destroy(phba, phba->sli4_hba.fcp_wq[qidx]); - - /* Unset NVME work queue */ - if (phba->sli4_hba.nvme_wq) { - for (qidx = 0; qidx < phba->cfg_nvme_io_channel; qidx++) - lpfc_wq_destroy(phba, phba->sli4_hba.nvme_wq[qidx]); - } - /* Unset mailbox command complete queue */ if (phba->sli4_hba.mbx_cq) lpfc_cq_destroy(phba, phba->sli4_hba.mbx_cq); @@ -9392,11 +9701,6 @@ lpfc_sli4_queue_unset(struct lpfc_hba *phba) if (phba->sli4_hba.nvmels_cq) lpfc_cq_destroy(phba, phba->sli4_hba.nvmels_cq); - /* Unset NVME response complete queue */ - if (phba->sli4_hba.nvme_cq) - for (qidx = 0; qidx < phba->cfg_nvme_io_channel; qidx++) - lpfc_cq_destroy(phba, phba->sli4_hba.nvme_cq[qidx]); - if (phba->nvmet_support) { /* Unset NVMET MRQ queue */ if (phba->sli4_hba.nvmet_mrq_hdr) { @@ -9415,15 +9719,22 @@ lpfc_sli4_queue_unset(struct lpfc_hba *phba) } } - /* Unset FCP response complete queue */ - if (phba->sli4_hba.fcp_cq) - for (qidx = 0; qidx < phba->cfg_fcp_io_channel; qidx++) - lpfc_cq_destroy(phba, phba->sli4_hba.fcp_cq[qidx]); + /* Unset fast-path SLI4 queues */ + if (phba->sli4_hba.hdwq) { + for (qidx = 0; qidx < phba->cfg_hdw_queue; qidx++) { + qp = &phba->sli4_hba.hdwq[qidx]; + lpfc_wq_destroy(phba, qp->fcp_wq); + lpfc_wq_destroy(phba, qp->nvme_wq); + lpfc_cq_destroy(phba, qp->fcp_cq); + lpfc_cq_destroy(phba, qp->nvme_cq); + if (qidx < phba->cfg_irq_chann) + lpfc_eq_destroy(phba, qp->hba_eq); + } + } - /* Unset fast-path event queue */ - if (phba->sli4_hba.hba_eq) - for (qidx = 0; qidx < phba->io_channel_irqs; qidx++) - lpfc_eq_destroy(phba, phba->sli4_hba.hba_eq[qidx]); + kfree(phba->sli4_hba.cq_lookup); + phba->sli4_hba.cq_lookup = NULL; + phba->sli4_hba.cq_max = 0; } /** @@ -9813,7 +10124,7 @@ lpfc_sli4_pci_mem_setup(struct lpfc_hba *phba) dev_printk(KERN_ERR, &pdev->dev, "ioremap failed for SLI4 PCI config " "registers.\n"); - goto out; + goto out; } lpfc_sli4_bar0_register_memmap(phba, if_type); } @@ -9918,13 +10229,13 @@ lpfc_sli4_pci_mem_setup(struct lpfc_hba *phba) case LPFC_SLI_INTF_IF_TYPE_0: case LPFC_SLI_INTF_IF_TYPE_2: phba->sli4_hba.sli4_eq_clr_intr = lpfc_sli4_eq_clr_intr; - phba->sli4_hba.sli4_eq_release = lpfc_sli4_eq_release; - phba->sli4_hba.sli4_cq_release = lpfc_sli4_cq_release; + phba->sli4_hba.sli4_write_eq_db = lpfc_sli4_write_eq_db; + phba->sli4_hba.sli4_write_cq_db = lpfc_sli4_write_cq_db; break; case LPFC_SLI_INTF_IF_TYPE_6: phba->sli4_hba.sli4_eq_clr_intr = lpfc_sli4_if6_eq_clr_intr; - phba->sli4_hba.sli4_eq_release = lpfc_sli4_if6_eq_release; - phba->sli4_hba.sli4_cq_release = lpfc_sli4_if6_cq_release; + phba->sli4_hba.sli4_write_eq_db = lpfc_sli4_if6_write_eq_db; + phba->sli4_hba.sli4_write_cq_db = lpfc_sli4_if6_write_cq_db; break; default: break; @@ -10205,25 +10516,97 @@ lpfc_sli_disable_intr(struct lpfc_hba *phba) } /** + * lpfc_find_cpu_handle - Find the CPU that corresponds to the specified EQ + * @phba: pointer to lpfc hba data structure. + * @id: EQ vector index or Hardware Queue index + * @match: LPFC_FIND_BY_EQ = match by EQ + * LPFC_FIND_BY_HDWQ = match by Hardware Queue + */ +static uint16_t +lpfc_find_cpu_handle(struct lpfc_hba *phba, uint16_t id, int match) +{ + struct lpfc_vector_map_info *cpup; + int cpu; + + /* Find the desired phys_id for the specified EQ */ + for_each_present_cpu(cpu) { + cpup = &phba->sli4_hba.cpu_map[cpu]; + if ((match == LPFC_FIND_BY_EQ) && + (cpup->irq != LPFC_VECTOR_MAP_EMPTY) && + (cpup->eq == id)) + return cpu; + if ((match == LPFC_FIND_BY_HDWQ) && (cpup->hdwq == id)) + return cpu; + } + return 0; +} + +/** + * lpfc_find_eq_handle - Find the EQ that corresponds to the specified + * Hardware Queue + * @phba: pointer to lpfc hba data structure. + * @hdwq: Hardware Queue index + */ +static uint16_t +lpfc_find_eq_handle(struct lpfc_hba *phba, uint16_t hdwq) +{ + struct lpfc_vector_map_info *cpup; + int cpu; + + /* Find the desired phys_id for the specified EQ */ + for_each_present_cpu(cpu) { + cpup = &phba->sli4_hba.cpu_map[cpu]; + if (cpup->hdwq == hdwq) + return cpup->eq; + } + return 0; +} + +#ifdef CONFIG_X86 +/** + * lpfc_find_hyper - Determine if the CPU map entry is hyper-threaded + * @phba: pointer to lpfc hba data structure. + * @cpu: CPU map index + * @phys_id: CPU package physical id + * @core_id: CPU core id + */ +static int +lpfc_find_hyper(struct lpfc_hba *phba, int cpu, + uint16_t phys_id, uint16_t core_id) +{ + struct lpfc_vector_map_info *cpup; + int idx; + + for_each_present_cpu(idx) { + cpup = &phba->sli4_hba.cpu_map[idx]; + /* Does the cpup match the one we are looking for */ + if ((cpup->phys_id == phys_id) && + (cpup->core_id == core_id) && + (cpu != idx)) + return 1; + } + return 0; +} +#endif + +/** * lpfc_cpu_affinity_check - Check vector CPU affinity mappings * @phba: pointer to lpfc hba data structure. * @vectors: number of msix vectors allocated. * * The routine will figure out the CPU affinity assignment for every - * MSI-X vector allocated for the HBA. The hba_eq_hdl will be updated - * with a pointer to the CPU mask that defines ALL the CPUs this vector - * can be associated with. If the vector can be unquely associated with - * a single CPU, that CPU will be recorded in hba_eq_hdl[index].cpu. + * MSI-X vector allocated for the HBA. * In addition, the CPU to IO channel mapping will be calculated * and the phba->sli4_hba.cpu_map array will reflect this. */ static void lpfc_cpu_affinity_check(struct lpfc_hba *phba, int vectors) { + int i, cpu, idx; + int max_phys_id, min_phys_id; + int max_core_id, min_core_id; struct lpfc_vector_map_info *cpup; - int index = 0; - int vec = 0; - int cpu; + const struct cpumask *maskp; #ifdef CONFIG_X86 struct cpuinfo_x86 *cpuinfo; #endif @@ -10231,32 +10614,71 @@ lpfc_cpu_affinity_check(struct lpfc_hba *phba, int vectors) /* Init cpu_map array */ memset(phba->sli4_hba.cpu_map, 0xff, (sizeof(struct lpfc_vector_map_info) * - phba->sli4_hba.num_present_cpu)); + phba->sli4_hba.num_possible_cpu)); + + max_phys_id = 0; + min_phys_id = 0xffff; + max_core_id = 0; + min_core_id = 0xffff; /* Update CPU map with physical id and core id of each CPU */ - cpup = phba->sli4_hba.cpu_map; - for (cpu = 0; cpu < phba->sli4_hba.num_present_cpu; cpu++) { + for_each_present_cpu(cpu) { + cpup = &phba->sli4_hba.cpu_map[cpu]; #ifdef CONFIG_X86 cpuinfo = &cpu_data(cpu); cpup->phys_id = cpuinfo->phys_proc_id; cpup->core_id = cpuinfo->cpu_core_id; + cpup->hyper = lpfc_find_hyper(phba, cpu, + cpup->phys_id, cpup->core_id); #else /* No distinction between CPUs for other platforms */ cpup->phys_id = 0; - cpup->core_id = 0; + cpup->core_id = cpu; + cpup->hyper = 0; #endif - cpup->channel_id = index; /* For now round robin */ - cpup->irq = pci_irq_vector(phba->pcidev, vec); - vec++; - if (vec >= vectors) - vec = 0; - index++; - if (index >= phba->cfg_fcp_io_channel) - index = 0; - cpup++; + + lpfc_printf_log(phba, KERN_INFO, LOG_INIT, + "3328 CPU physid %d coreid %d\n", + cpup->phys_id, cpup->core_id); + + if (cpup->phys_id > max_phys_id) + max_phys_id = cpup->phys_id; + if (cpup->phys_id < min_phys_id) + min_phys_id = cpup->phys_id; + + if (cpup->core_id > max_core_id) + max_core_id = cpup->core_id; + if (cpup->core_id < min_core_id) + min_core_id = cpup->core_id; } -} + for_each_possible_cpu(i) { + struct lpfc_eq_intr_info *eqi = + per_cpu_ptr(phba->sli4_hba.eq_info, i); + + INIT_LIST_HEAD(&eqi->list); + eqi->icnt = 0; + } + + for (idx = 0; idx < phba->cfg_irq_chann; idx++) { + maskp = pci_irq_get_affinity(phba->pcidev, idx); + if (!maskp) + continue; + + for_each_cpu_and(cpu, maskp, cpu_present_mask) { + cpup = &phba->sli4_hba.cpu_map[cpu]; + cpup->eq = idx; + cpup->hdwq = idx; + cpup->irq = pci_irq_vector(phba->pcidev, idx); + + lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + "3336 Set Affinity: CPU %d " + "hdwq %d irq %d\n", + cpu, cpup->hdwq, cpup->irq); + } + } + return; +} /** * lpfc_sli4_enable_msix - Enable MSI-X interrupt mode to SLI-4 device @@ -10276,12 +10698,10 @@ lpfc_sli4_enable_msix(struct lpfc_hba *phba) char *name; /* Set up MSI-X multi-message vectors */ - vectors = phba->io_channel_irqs; - if (phba->cfg_fof) - vectors++; + vectors = phba->cfg_irq_chann; rc = pci_alloc_irq_vectors(phba->pcidev, - (phba->nvmet_support) ? 1 : 2, + 1, vectors, PCI_IRQ_MSIX | PCI_IRQ_AFFINITY); if (rc < 0) { lpfc_printf_log(phba, KERN_INFO, LOG_INIT, @@ -10299,17 +10719,10 @@ lpfc_sli4_enable_msix(struct lpfc_hba *phba) phba->sli4_hba.hba_eq_hdl[index].idx = index; phba->sli4_hba.hba_eq_hdl[index].phba = phba; - atomic_set(&phba->sli4_hba.hba_eq_hdl[index].hba_eq_in_use, 1); - if (phba->cfg_fof && (index == (vectors - 1))) - rc = request_irq(pci_irq_vector(phba->pcidev, index), - &lpfc_sli4_fof_intr_handler, 0, - name, - &phba->sli4_hba.hba_eq_hdl[index]); - else - rc = request_irq(pci_irq_vector(phba->pcidev, index), - &lpfc_sli4_hba_intr_handler, 0, - name, - &phba->sli4_hba.hba_eq_hdl[index]); + rc = request_irq(pci_irq_vector(phba->pcidev, index), + &lpfc_sli4_hba_intr_handler, 0, + name, + &phba->sli4_hba.hba_eq_hdl[index]); if (rc) { lpfc_printf_log(phba, KERN_WARNING, LOG_INIT, "0486 MSI-X fast-path (%d) " @@ -10318,24 +10731,16 @@ lpfc_sli4_enable_msix(struct lpfc_hba *phba) } } - if (phba->cfg_fof) - vectors--; - - if (vectors != phba->io_channel_irqs) { + if (vectors != phba->cfg_irq_chann) { lpfc_printf_log(phba, KERN_ERR, LOG_INIT, "3238 Reducing IO channels to match number of " "MSI-X vectors, requested %d got %d\n", - phba->io_channel_irqs, vectors); - if (phba->cfg_fcp_io_channel > vectors) - phba->cfg_fcp_io_channel = vectors; - if (phba->cfg_nvme_io_channel > vectors) - phba->cfg_nvme_io_channel = vectors; - if (phba->cfg_fcp_io_channel > phba->cfg_nvme_io_channel) - phba->io_channel_irqs = phba->cfg_fcp_io_channel; - else - phba->io_channel_irqs = phba->cfg_nvme_io_channel; + phba->cfg_irq_chann, vectors); + if (phba->cfg_irq_chann > vectors) + phba->cfg_irq_chann = vectors; + if (phba->cfg_nvmet_mrq > vectors) + phba->cfg_nvmet_mrq = vectors; } - lpfc_cpu_affinity_check(phba, vectors); return rc; @@ -10390,15 +10795,11 @@ lpfc_sli4_enable_msi(struct lpfc_hba *phba) return rc; } - for (index = 0; index < phba->io_channel_irqs; index++) { + for (index = 0; index < phba->cfg_irq_chann; index++) { phba->sli4_hba.hba_eq_hdl[index].idx = index; phba->sli4_hba.hba_eq_hdl[index].phba = phba; } - if (phba->cfg_fof) { - phba->sli4_hba.hba_eq_hdl[index].idx = index; - phba->sli4_hba.hba_eq_hdl[index].phba = phba; - } return 0; } @@ -10459,17 +10860,10 @@ lpfc_sli4_enable_intr(struct lpfc_hba *phba, uint32_t cfg_mode) phba->intr_type = INTx; intr_mode = 0; - for (idx = 0; idx < phba->io_channel_irqs; idx++) { - eqhdl = &phba->sli4_hba.hba_eq_hdl[idx]; - eqhdl->idx = idx; - eqhdl->phba = phba; - atomic_set(&eqhdl->hba_eq_in_use, 1); - } - if (phba->cfg_fof) { + for (idx = 0; idx < phba->cfg_irq_chann; idx++) { eqhdl = &phba->sli4_hba.hba_eq_hdl[idx]; eqhdl->idx = idx; eqhdl->phba = phba; - atomic_set(&eqhdl->hba_eq_in_use, 1); } } } @@ -10493,13 +10887,13 @@ lpfc_sli4_disable_intr(struct lpfc_hba *phba) int index; /* Free up MSI-X multi-message vectors */ - for (index = 0; index < phba->io_channel_irqs; index++) - free_irq(pci_irq_vector(phba->pcidev, index), - &phba->sli4_hba.hba_eq_hdl[index]); - - if (phba->cfg_fof) + for (index = 0; index < phba->cfg_irq_chann; index++) { + irq_set_affinity_hint( + pci_irq_vector(phba->pcidev, index), + NULL); free_irq(pci_irq_vector(phba->pcidev, index), &phba->sli4_hba.hba_eq_hdl[index]); + } } else { free_irq(phba->pcidev->irq, phba); } @@ -10560,8 +10954,10 @@ lpfc_unset_hba(struct lpfc_hba *phba) static void lpfc_sli4_xri_exchange_busy_wait(struct lpfc_hba *phba) { + struct lpfc_sli4_hdw_queue *qp; + int idx, ccnt, fcnt; int wait_time = 0; - int nvme_xri_cmpl = 1; + int io_xri_cmpl = 1; int nvmet_xri_cmpl = 1; int fcp_xri_cmpl = 1; int els_xri_cmpl = list_empty(&phba->sli4_hba.lpfc_abts_els_sgl_list); @@ -10576,17 +10972,32 @@ lpfc_sli4_xri_exchange_busy_wait(struct lpfc_hba *phba) if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) lpfc_nvme_wait_for_io_drain(phba); - if (phba->cfg_enable_fc4_type & LPFC_ENABLE_FCP) - fcp_xri_cmpl = - list_empty(&phba->sli4_hba.lpfc_abts_scsi_buf_list); + ccnt = 0; + fcnt = 0; + for (idx = 0; idx < phba->cfg_hdw_queue; idx++) { + qp = &phba->sli4_hba.hdwq[idx]; + fcp_xri_cmpl = list_empty( + &qp->lpfc_abts_scsi_buf_list); + if (!fcp_xri_cmpl) /* if list is NOT empty */ + fcnt++; + if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { + io_xri_cmpl = list_empty( + &qp->lpfc_abts_nvme_buf_list); + if (!io_xri_cmpl) /* if list is NOT empty */ + ccnt++; + } + } + if (ccnt) + io_xri_cmpl = 0; + if (fcnt) + fcp_xri_cmpl = 0; + if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { - nvme_xri_cmpl = - list_empty(&phba->sli4_hba.lpfc_abts_nvme_buf_list); nvmet_xri_cmpl = list_empty(&phba->sli4_hba.lpfc_abts_nvmet_ctx_list); } - while (!fcp_xri_cmpl || !els_xri_cmpl || !nvme_xri_cmpl || + while (!fcp_xri_cmpl || !els_xri_cmpl || !io_xri_cmpl || !nvmet_xri_cmpl) { if (wait_time > LPFC_XRI_EXCH_BUSY_WAIT_TMO) { if (!nvmet_xri_cmpl) @@ -10594,7 +11005,7 @@ lpfc_sli4_xri_exchange_busy_wait(struct lpfc_hba *phba) "6424 NVMET XRI exchange busy " "wait time: %d seconds.\n", wait_time/1000); - if (!nvme_xri_cmpl) + if (!io_xri_cmpl) lpfc_printf_log(phba, KERN_ERR, LOG_INIT, "6100 NVME XRI exchange busy " "wait time: %d seconds.\n", @@ -10615,17 +11026,31 @@ lpfc_sli4_xri_exchange_busy_wait(struct lpfc_hba *phba) msleep(LPFC_XRI_EXCH_BUSY_WAIT_T1); wait_time += LPFC_XRI_EXCH_BUSY_WAIT_T1; } + + ccnt = 0; + fcnt = 0; + for (idx = 0; idx < phba->cfg_hdw_queue; idx++) { + qp = &phba->sli4_hba.hdwq[idx]; + fcp_xri_cmpl = list_empty( + &qp->lpfc_abts_scsi_buf_list); + if (!fcp_xri_cmpl) /* if list is NOT empty */ + fcnt++; + if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { + io_xri_cmpl = list_empty( + &qp->lpfc_abts_nvme_buf_list); + if (!io_xri_cmpl) /* if list is NOT empty */ + ccnt++; + } + } + if (ccnt) + io_xri_cmpl = 0; + if (fcnt) + fcp_xri_cmpl = 0; + if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { - nvme_xri_cmpl = list_empty( - &phba->sli4_hba.lpfc_abts_nvme_buf_list); nvmet_xri_cmpl = list_empty( &phba->sli4_hba.lpfc_abts_nvmet_ctx_list); } - - if (phba->cfg_enable_fc4_type & LPFC_ENABLE_FCP) - fcp_xri_cmpl = list_empty( - &phba->sli4_hba.lpfc_abts_scsi_buf_list); - els_xri_cmpl = list_empty(&phba->sli4_hba.lpfc_abts_els_sgl_list); @@ -10650,7 +11075,8 @@ lpfc_sli4_hba_unset(struct lpfc_hba *phba) struct pci_dev *pdev = phba->pcidev; lpfc_stop_hba_timers(phba); - phba->sli4_hba.intr_enable = 0; + if (phba->pport) + phba->sli4_hba.intr_enable = 0; /* * Gracefully wait out the potential current outstanding asynchronous @@ -10869,8 +11295,6 @@ lpfc_get_sli4_parameters(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq) phba->nvme_support = 0; phba->nvmet_support = 0; phba->cfg_nvmet_mrq = LPFC_NVMET_MRQ_OFF; - phba->cfg_nvme_io_channel = 0; - phba->io_channel_irqs = phba->cfg_fcp_io_channel; lpfc_printf_log(phba, KERN_ERR, LOG_INIT | LOG_NVME, "6101 Disabling NVME support: " "Not supported by firmware: %d %d\n", @@ -11195,6 +11619,8 @@ lpfc_pci_remove_one_s3(struct pci_dev *pdev) * corresponding pools here. */ lpfc_scsi_free(phba); + lpfc_free_iocb_list(phba); + lpfc_mem_free_all(phba); dma_free_coherent(&pdev->dev, lpfc_sli_hbq_size(), @@ -11820,28 +12246,11 @@ lpfc_pci_probe_one_s4(struct pci_dev *pdev, const struct pci_device_id *pid) /* Get the default values for Model Name and Description */ lpfc_get_hba_model_desc(phba, phba->ModelName, phba->ModelDesc); - /* Create SCSI host to the physical port */ - error = lpfc_create_shost(phba); - if (error) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, - "1415 Failed to create scsi host.\n"); - goto out_unset_driver_resource; - } - - /* Configure sysfs attributes */ - vport = phba->pport; - error = lpfc_alloc_sysfs_attr(vport); - if (error) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, - "1416 Failed to allocate sysfs attr\n"); - goto out_destroy_shost; - } - - shost = lpfc_shost_from_vport(vport); /* save shost for error cleanup */ /* Now, trying to enable interrupt and bring up the device */ cfg_mode = phba->cfg_use_msi; /* Put device to a known state before enabling interrupt */ + phba->pport = NULL; lpfc_stop_port(phba); /* Configure and enable interrupt */ @@ -11850,18 +12259,34 @@ lpfc_pci_probe_one_s4(struct pci_dev *pdev, const struct pci_device_id *pid) lpfc_printf_log(phba, KERN_ERR, LOG_INIT, "0426 Failed to enable interrupt.\n"); error = -ENODEV; - goto out_free_sysfs_attr; + goto out_unset_driver_resource; } /* Default to single EQ for non-MSI-X */ if (phba->intr_type != MSIX) { - if (phba->cfg_enable_fc4_type & LPFC_ENABLE_FCP) - phba->cfg_fcp_io_channel = 1; + phba->cfg_irq_chann = 1; if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { - phba->cfg_nvme_io_channel = 1; if (phba->nvmet_support) phba->cfg_nvmet_mrq = 1; } - phba->io_channel_irqs = 1; + } + lpfc_cpu_affinity_check(phba, phba->cfg_irq_chann); + + /* Create SCSI host to the physical port */ + error = lpfc_create_shost(phba); + if (error) { + lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + "1415 Failed to create scsi host.\n"); + goto out_disable_intr; + } + vport = phba->pport; + shost = lpfc_shost_from_vport(vport); /* save shost for error cleanup */ + + /* Configure sysfs attributes */ + error = lpfc_alloc_sysfs_attr(vport); + if (error) { + lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + "1416 Failed to allocate sysfs attr\n"); + goto out_destroy_shost; } /* Set up SLI-4 HBA */ @@ -11869,7 +12294,7 @@ lpfc_pci_probe_one_s4(struct pci_dev *pdev, const struct pci_device_id *pid) lpfc_printf_log(phba, KERN_ERR, LOG_INIT, "1421 Failed to set up hba\n"); error = -ENODEV; - goto out_disable_intr; + goto out_free_sysfs_attr; } /* Log the current active interrupt mode */ @@ -11882,19 +12307,20 @@ lpfc_pci_probe_one_s4(struct pci_dev *pdev, const struct pci_device_id *pid) /* NVME support in FW earlier in the driver load corrects the * FC4 type making a check for nvme_support unnecessary. */ - if ((phba->nvmet_support == 0) && - (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME)) { - /* Create NVME binding with nvme_fc_transport. This - * ensures the vport is initialized. If the localport - * create fails, it should not unload the driver to - * support field issues. - */ - error = lpfc_nvme_create_localport(vport); - if (error) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, - "6004 NVME registration failed, " - "error x%x\n", - error); + if (phba->nvmet_support == 0) { + if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { + /* Create NVME binding with nvme_fc_transport. This + * ensures the vport is initialized. If the localport + * create fails, it should not unload the driver to + * support field issues. + */ + error = lpfc_nvme_create_localport(vport); + if (error) { + lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + "6004 NVME registration " + "failed, error x%x\n", + error); + } } } @@ -11910,12 +12336,12 @@ lpfc_pci_probe_one_s4(struct pci_dev *pdev, const struct pci_device_id *pid) return 0; -out_disable_intr: - lpfc_sli4_disable_intr(phba); out_free_sysfs_attr: lpfc_free_sysfs_attr(vport); out_destroy_shost: lpfc_destroy_shost(phba); +out_disable_intr: + lpfc_sli4_disable_intr(phba); out_unset_driver_resource: lpfc_unset_driver_resource_phase2(phba); out_unset_driver_resource_s4: @@ -11978,13 +12404,16 @@ lpfc_pci_remove_one_s4(struct pci_dev *pdev) lpfc_nvmet_destroy_targetport(phba); lpfc_nvme_destroy_localport(vport); + /* De-allocate multi-XRI pools */ + if (phba->cfg_xri_rebalancing) + lpfc_destroy_multixri_pools(phba); + /* * Bring down the SLI Layer. This step disables all interrupts, * clears the rings, discards all mailbox commands, and resets * the HBA FCoE function. */ lpfc_debugfs_terminate(vport); - lpfc_sli4_hba_unset(phba); lpfc_stop_hba_timers(phba); spin_lock_irq(&phba->port_list_lock); @@ -11994,9 +12423,9 @@ lpfc_pci_remove_one_s4(struct pci_dev *pdev) /* Perform scsi free before driver resource_unset since scsi * buffers are released to their corresponding pools here. */ - lpfc_scsi_free(phba); - lpfc_nvme_free(phba); + lpfc_io_free(phba); lpfc_free_iocb_list(phba); + lpfc_sli4_hba_unset(phba); lpfc_unset_driver_resource_phase2(phba); lpfc_sli4_driver_resource_unset(phba); @@ -12658,165 +13087,6 @@ lpfc_sli4_ras_init(struct lpfc_hba *phba) } } -/** - * lpfc_fof_queue_setup - Set up all the fof queues - * @phba: pointer to lpfc hba data structure. - * - * This routine is invoked to set up all the fof queues for the FC HBA - * operation. - * - * Return codes - * 0 - successful - * -ENOMEM - No available memory - **/ -int -lpfc_fof_queue_setup(struct lpfc_hba *phba) -{ - struct lpfc_sli_ring *pring; - int rc; - - rc = lpfc_eq_create(phba, phba->sli4_hba.fof_eq, LPFC_MAX_IMAX); - if (rc) - return -ENOMEM; - - if (phba->cfg_fof) { - - rc = lpfc_cq_create(phba, phba->sli4_hba.oas_cq, - phba->sli4_hba.fof_eq, LPFC_WCQ, LPFC_FCP); - if (rc) - goto out_oas_cq; - - rc = lpfc_wq_create(phba, phba->sli4_hba.oas_wq, - phba->sli4_hba.oas_cq, LPFC_FCP); - if (rc) - goto out_oas_wq; - - /* Bind this CQ/WQ to the NVME ring */ - pring = phba->sli4_hba.oas_wq->pring; - pring->sli.sli4.wqp = - (void *)phba->sli4_hba.oas_wq; - phba->sli4_hba.oas_cq->pring = pring; - } - - return 0; - -out_oas_wq: - lpfc_cq_destroy(phba, phba->sli4_hba.oas_cq); -out_oas_cq: - lpfc_eq_destroy(phba, phba->sli4_hba.fof_eq); - return rc; - -} - -/** - * lpfc_fof_queue_create - Create all the fof queues - * @phba: pointer to lpfc hba data structure. - * - * This routine is invoked to allocate all the fof queues for the FC HBA - * operation. For each SLI4 queue type, the parameters such as queue entry - * count (queue depth) shall be taken from the module parameter. For now, - * we just use some constant number as place holder. - * - * Return codes - * 0 - successful - * -ENOMEM - No availble memory - * -EIO - The mailbox failed to complete successfully. - **/ -int -lpfc_fof_queue_create(struct lpfc_hba *phba) -{ - struct lpfc_queue *qdesc; - uint32_t wqesize; - - /* Create FOF EQ */ - qdesc = lpfc_sli4_queue_alloc(phba, LPFC_DEFAULT_PAGE_SIZE, - phba->sli4_hba.eq_esize, - phba->sli4_hba.eq_ecount); - if (!qdesc) - goto out_error; - - qdesc->qe_valid = 1; - phba->sli4_hba.fof_eq = qdesc; - - if (phba->cfg_fof) { - - /* Create OAS CQ */ - if (phba->enab_exp_wqcq_pages) - qdesc = lpfc_sli4_queue_alloc(phba, - LPFC_EXPANDED_PAGE_SIZE, - phba->sli4_hba.cq_esize, - LPFC_CQE_EXP_COUNT); - else - qdesc = lpfc_sli4_queue_alloc(phba, - LPFC_DEFAULT_PAGE_SIZE, - phba->sli4_hba.cq_esize, - phba->sli4_hba.cq_ecount); - if (!qdesc) - goto out_error; - - qdesc->qe_valid = 1; - phba->sli4_hba.oas_cq = qdesc; - - /* Create OAS WQ */ - if (phba->enab_exp_wqcq_pages) { - wqesize = (phba->fcp_embed_io) ? - LPFC_WQE128_SIZE : phba->sli4_hba.wq_esize; - qdesc = lpfc_sli4_queue_alloc(phba, - LPFC_EXPANDED_PAGE_SIZE, - wqesize, - LPFC_WQE_EXP_COUNT); - } else - qdesc = lpfc_sli4_queue_alloc(phba, - LPFC_DEFAULT_PAGE_SIZE, - phba->sli4_hba.wq_esize, - phba->sli4_hba.wq_ecount); - - if (!qdesc) - goto out_error; - - phba->sli4_hba.oas_wq = qdesc; - list_add_tail(&qdesc->wq_list, &phba->sli4_hba.lpfc_wq_list); - - } - return 0; - -out_error: - lpfc_fof_queue_destroy(phba); - return -ENOMEM; -} - -/** - * lpfc_fof_queue_destroy - Destroy all the fof queues - * @phba: pointer to lpfc hba data structure. - * - * This routine is invoked to release all the SLI4 queues with the FC HBA - * operation. - * - * Return codes - * 0 - successful - **/ -int -lpfc_fof_queue_destroy(struct lpfc_hba *phba) -{ - /* Release FOF Event queue */ - if (phba->sli4_hba.fof_eq != NULL) { - lpfc_sli4_queue_free(phba->sli4_hba.fof_eq); - phba->sli4_hba.fof_eq = NULL; - } - - /* Release OAS Completion queue */ - if (phba->sli4_hba.oas_cq != NULL) { - lpfc_sli4_queue_free(phba->sli4_hba.oas_cq); - phba->sli4_hba.oas_cq = NULL; - } - - /* Release OAS Work queue */ - if (phba->sli4_hba.oas_wq != NULL) { - lpfc_sli4_queue_free(phba->sli4_hba.oas_wq); - phba->sli4_hba.oas_wq = NULL; - } - return 0; -} MODULE_DEVICE_TABLE(pci, lpfc_id_table); @@ -12888,7 +13158,6 @@ lpfc_init(void) lpfc_nvmet_cmd_template(); /* Initialize in case vector mapping is needed */ - lpfc_used_cpu = NULL; lpfc_present_cpu = num_present_cpus(); error = pci_register_driver(&lpfc_driver); @@ -12927,7 +13196,6 @@ lpfc_exit(void) (1L << _dump_buf_dif_order), _dump_buf_dif); free_pages((unsigned long)_dump_buf_dif, _dump_buf_dif_order); } - kfree(lpfc_used_cpu); idr_destroy(&lpfc_hba_index); } diff --git a/drivers/scsi/lpfc/lpfc_mbox.c b/drivers/scsi/lpfc/lpfc_mbox.c index 4d3b94317515..8abe933bad09 100644 --- a/drivers/scsi/lpfc/lpfc_mbox.c +++ b/drivers/scsi/lpfc/lpfc_mbox.c @@ -2095,8 +2095,8 @@ lpfc_request_features(struct lpfc_hba *phba, struct lpfcMboxq *mboxq) if (phba->nvmet_support) { bf_set(lpfc_mbx_rq_ftr_rq_mrqp, &mboxq->u.mqe.un.req_ftrs, 1); /* iaab/iaar NOT set for now */ - bf_set(lpfc_mbx_rq_ftr_rq_iaab, &mboxq->u.mqe.un.req_ftrs, 0); - bf_set(lpfc_mbx_rq_ftr_rq_iaar, &mboxq->u.mqe.un.req_ftrs, 0); + bf_set(lpfc_mbx_rq_ftr_rq_iaab, &mboxq->u.mqe.un.req_ftrs, 0); + bf_set(lpfc_mbx_rq_ftr_rq_iaar, &mboxq->u.mqe.un.req_ftrs, 0); } return; } diff --git a/drivers/scsi/lpfc/lpfc_nportdisc.c b/drivers/scsi/lpfc/lpfc_nportdisc.c index 96bc3789a166..6172682a24ba 100644 --- a/drivers/scsi/lpfc/lpfc_nportdisc.c +++ b/drivers/scsi/lpfc/lpfc_nportdisc.c @@ -1,7 +1,7 @@ /******************************************************************* * This file is part of the Emulex Linux Device Driver for * * Fibre Channel Host Bus Adapters. * - * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term * + * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term * * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * * Copyright (C) 2004-2016 Emulex. All rights reserved. * * EMULEX and SLI are trademarks of Emulex. * @@ -825,7 +825,7 @@ lpfc_rcv_prli(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp, "rport rolechg: role:x%x did:x%x flg:x%x", roles, ndlp->nlp_DID, ndlp->nlp_flag); - if (phba->cfg_enable_fc4_type != LPFC_ENABLE_NVME) + if (vport->cfg_enable_fc4_type != LPFC_ENABLE_NVME) fc_remote_port_rolechg(rport, roles); } } @@ -1789,8 +1789,8 @@ lpfc_cmpl_reglogin_reglogin_issue(struct lpfc_vport *vport, * is configured try it. */ ndlp->nlp_fc4_type |= NLP_FC4_FCP; - if ((phba->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) || - (phba->cfg_enable_fc4_type == LPFC_ENABLE_NVME)) { + if ((vport->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) || + (vport->cfg_enable_fc4_type == LPFC_ENABLE_NVME)) { ndlp->nlp_fc4_type |= NLP_FC4_NVME; /* We need to update the localport also */ lpfc_nvme_update_localport(vport); @@ -1804,7 +1804,7 @@ lpfc_cmpl_reglogin_reglogin_issue(struct lpfc_vport *vport, * should just issue PRLI for FCP. Otherwise issue * GFT_ID to determine if remote port supports NVME. */ - if (phba->cfg_enable_fc4_type != LPFC_ENABLE_FCP) { + if (vport->cfg_enable_fc4_type != LPFC_ENABLE_FCP) { rc = lpfc_ns_cmd(vport, SLI_CTNS_GFT_ID, 0, ndlp->nlp_DID); return ndlp->nlp_state; diff --git a/drivers/scsi/lpfc/lpfc_nvme.c b/drivers/scsi/lpfc/lpfc_nvme.c index 8c9f79042228..55ab9d3ee4ba 100644 --- a/drivers/scsi/lpfc/lpfc_nvme.c +++ b/drivers/scsi/lpfc/lpfc_nvme.c @@ -1,7 +1,7 @@ /******************************************************************* * This file is part of the Emulex Linux Device Driver for * * Fibre Channel Host Bus Adapters. * - * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term * + * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term * * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * * Copyright (C) 2004-2016 Emulex. All rights reserved. * * EMULEX and SLI are trademarks of Emulex. * @@ -56,12 +56,12 @@ /* NVME initiator-based functions */ -static struct lpfc_nvme_buf * +static struct lpfc_io_buf * lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, - int expedite); + int idx, int expedite); static void -lpfc_release_nvme_buf(struct lpfc_hba *, struct lpfc_nvme_buf *); +lpfc_release_nvme_buf(struct lpfc_hba *, struct lpfc_io_buf *); static struct nvme_fc_port_template lpfc_nvme_template; @@ -239,7 +239,7 @@ lpfc_nvme_create_queue(struct nvme_fc_local_port *pnvme_lport, if (qidx) { str = "IO "; /* IO queue */ qhandle->index = ((qidx - 1) % - vport->phba->cfg_nvme_io_channel); + lpfc_nvme_template.max_hw_queues); } else { str = "ADM"; /* Admin queue */ qhandle->index = qidx; @@ -247,7 +247,7 @@ lpfc_nvme_create_queue(struct nvme_fc_local_port *pnvme_lport, lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME, "6073 Binding %s HdwQueue %d (cpu %d) to " - "io_channel %d qhandle %p\n", str, + "hdw_queue %d qhandle %p\n", str, qidx, qhandle->cpu_id, qhandle->index, qhandle); *handle = (void *)qhandle; return 0; @@ -529,7 +529,7 @@ lpfc_nvme_gen_req(struct lpfc_vport *vport, struct lpfc_dmabuf *bmp, lpfc_nvmeio_data(phba, "NVME LS XMIT: xri x%x iotag x%x to x%06x\n", genwqe->sli4_xritag, genwqe->iotag, ndlp->nlp_DID); - rc = lpfc_sli4_issue_wqe(phba, LPFC_ELS_RING, genwqe); + rc = lpfc_sli4_issue_wqe(phba, &phba->sli4_hba.hdwq[0], genwqe); if (rc) { lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS, "6045 Issue GEN REQ WQE to NPORT x%x " @@ -761,7 +761,7 @@ lpfc_nvme_ls_abort(struct nvme_fc_local_port *pnvme_lport, /* Fix up the existing sgls for NVME IO. */ static inline void lpfc_nvme_adj_fcp_sgls(struct lpfc_vport *vport, - struct lpfc_nvme_buf *lpfc_ncmd, + struct lpfc_io_buf *lpfc_ncmd, struct nvmefc_fcp_req *nCmd) { struct lpfc_hba *phba = vport->phba; @@ -784,7 +784,7 @@ lpfc_nvme_adj_fcp_sgls(struct lpfc_vport *vport, * rather than the virtual memory to ease the restore * operation. */ - sgl = lpfc_ncmd->nvme_sgl; + sgl = lpfc_ncmd->dma_sgl; sgl->sge_len = cpu_to_le32(nCmd->cmdlen); if (phba->cfg_nvme_embed_cmd) { sgl->addr_hi = 0; @@ -858,7 +858,7 @@ lpfc_nvme_adj_fcp_sgls(struct lpfc_vport *vport, #ifdef CONFIG_SCSI_LPFC_DEBUG_FS static void lpfc_nvme_ktime(struct lpfc_hba *phba, - struct lpfc_nvme_buf *lpfc_ncmd) + struct lpfc_io_buf *lpfc_ncmd) { uint64_t seg1, seg2, seg3, seg4; uint64_t segsum; @@ -956,57 +956,54 @@ static void lpfc_nvme_io_cmd_wqe_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn, struct lpfc_wcqe_complete *wcqe) { - struct lpfc_nvme_buf *lpfc_ncmd = - (struct lpfc_nvme_buf *)pwqeIn->context1; + struct lpfc_io_buf *lpfc_ncmd = + (struct lpfc_io_buf *)pwqeIn->context1; struct lpfc_vport *vport = pwqeIn->vport; struct nvmefc_fcp_req *nCmd; struct nvme_fc_ersp_iu *ep; struct nvme_fc_cmd_iu *cp; - struct lpfc_nvme_rport *rport; struct lpfc_nodelist *ndlp; struct lpfc_nvme_fcpreq_priv *freqpriv; struct lpfc_nvme_lport *lport; - struct lpfc_nvme_ctrl_stat *cstat; - unsigned long flags; - uint32_t code, status, idx; + uint32_t code, status, idx, cpu; uint16_t cid, sqhd, data; uint32_t *ptr; /* Sanity check on return of outstanding command */ - if (!lpfc_ncmd || !lpfc_ncmd->nvmeCmd || !lpfc_ncmd->nrport) { - if (!lpfc_ncmd) { - lpfc_printf_vlog(vport, KERN_ERR, - LOG_NODE | LOG_NVME_IOERR, - "6071 Null lpfc_ncmd pointer. No " - "release, skip completion\n"); - return; - } + if (!lpfc_ncmd) { + lpfc_printf_vlog(vport, KERN_ERR, + LOG_NODE | LOG_NVME_IOERR, + "6071 Null lpfc_ncmd pointer. No " + "release, skip completion\n"); + return; + } + + /* Guard against abort handler being called at same time */ + spin_lock(&lpfc_ncmd->buf_lock); + if (!lpfc_ncmd->nvmeCmd) { + spin_unlock(&lpfc_ncmd->buf_lock); lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE | LOG_NVME_IOERR, "6066 Missing cmpl ptrs: lpfc_ncmd %p, " - "nvmeCmd %p nrport %p\n", - lpfc_ncmd, lpfc_ncmd->nvmeCmd, - lpfc_ncmd->nrport); + "nvmeCmd %p\n", + lpfc_ncmd, lpfc_ncmd->nvmeCmd); /* Release the lpfc_ncmd regardless of the missing elements. */ lpfc_release_nvme_buf(phba, lpfc_ncmd); return; } nCmd = lpfc_ncmd->nvmeCmd; - rport = lpfc_ncmd->nrport; status = bf_get(lpfc_wcqe_c_status, wcqe); + idx = lpfc_ncmd->cur_iocbq.hba_wqidx; + phba->sli4_hba.hdwq[idx].nvme_cstat.io_cmpls++; + if (vport->localport) { lport = (struct lpfc_nvme_lport *)vport->localport->private; - if (lport) { - idx = lpfc_ncmd->cur_iocbq.hba_wqidx; - cstat = &lport->cstat[idx]; - atomic_inc(&cstat->fc4NvmeIoCmpls); - if (status) { - if (bf_get(lpfc_wcqe_c_xb, wcqe)) - atomic_inc(&lport->cmpl_fcp_xb); - atomic_inc(&lport->cmpl_fcp_err); - } + if (lport && status) { + if (bf_get(lpfc_wcqe_c_xb, wcqe)) + atomic_inc(&lport->cmpl_fcp_xb); + atomic_inc(&lport->cmpl_fcp_err); } } @@ -1017,18 +1014,11 @@ lpfc_nvme_io_cmd_wqe_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn, * Catch race where our node has transitioned, but the * transport is still transitioning. */ - ndlp = rport->ndlp; + ndlp = lpfc_ncmd->ndlp; if (!ndlp || !NLP_CHK_NODE_ACT(ndlp)) { - lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE | LOG_NVME_IOERR, - "6061 rport %p, DID x%06x node not ready.\n", - rport, rport->remoteport->port_id); - - ndlp = lpfc_findnode_did(vport, rport->remoteport->port_id); - if (!ndlp) { - lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_IOERR, - "6062 Ignoring NVME cmpl. No ndlp\n"); - goto out_err; - } + lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_IOERR, + "6062 Ignoring NVME cmpl. No ndlp\n"); + goto out_err; } code = bf_get(lpfc_wcqe_c_code, wcqe); @@ -1148,13 +1138,17 @@ out_err: lpfc_nvme_ktime(phba, lpfc_ncmd); } if (phba->cpucheck_on & LPFC_CHECK_NVME_IO) { - if (lpfc_ncmd->cpu != smp_processor_id()) - lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_IOERR, - "6701 CPU Check cmpl: " - "cpu %d expect %d\n", - smp_processor_id(), lpfc_ncmd->cpu); - if (lpfc_ncmd->cpu < LPFC_CHECK_CPU_CNT) - phba->cpucheck_cmpl_io[lpfc_ncmd->cpu]++; + idx = lpfc_ncmd->cur_iocbq.hba_wqidx; + cpu = smp_processor_id(); + if (cpu < LPFC_CHECK_CPU_CNT) { + if (lpfc_ncmd->cpu != cpu) + lpfc_printf_vlog(vport, + KERN_INFO, LOG_NVME_IOERR, + "6701 CPU Check cmpl: " + "cpu %d expect %d\n", + cpu, lpfc_ncmd->cpu); + phba->sli4_hba.hdwq[idx].cpucheck_cmpl_io[cpu]++; + } } #endif @@ -1165,13 +1159,11 @@ out_err: if (!(lpfc_ncmd->flags & LPFC_SBUF_XBUSY)) { freqpriv = nCmd->private; freqpriv->nvme_buf = NULL; - nCmd->done(nCmd); lpfc_ncmd->nvmeCmd = NULL; - } - - spin_lock_irqsave(&phba->hbalock, flags); - lpfc_ncmd->nrport = NULL; - spin_unlock_irqrestore(&phba->hbalock, flags); + spin_unlock(&lpfc_ncmd->buf_lock); + nCmd->done(nCmd); + } else + spin_unlock(&lpfc_ncmd->buf_lock); /* Call release with XB=1 to queue the IO into the abort list. */ lpfc_release_nvme_buf(phba, lpfc_ncmd); @@ -1196,9 +1188,9 @@ out_err: **/ static int lpfc_nvme_prep_io_cmd(struct lpfc_vport *vport, - struct lpfc_nvme_buf *lpfc_ncmd, + struct lpfc_io_buf *lpfc_ncmd, struct lpfc_nodelist *pnode, - struct lpfc_nvme_ctrl_stat *cstat) + struct lpfc_fc4_ctrl_stat *cstat) { struct lpfc_hba *phba = vport->phba; struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd; @@ -1206,7 +1198,7 @@ lpfc_nvme_prep_io_cmd(struct lpfc_vport *vport, union lpfc_wqe128 *wqe = &pwqeq->wqe; uint32_t req_len; - if (!pnode || !NLP_CHK_NODE_ACT(pnode)) + if (!NLP_CHK_NODE_ACT(pnode)) return -EINVAL; /* @@ -1236,7 +1228,7 @@ lpfc_nvme_prep_io_cmd(struct lpfc_vport *vport, } else { wqe->fcp_iwrite.initial_xfer_len = 0; } - atomic_inc(&cstat->fc4NvmeOutputRequests); + cstat->output_requests++; } else { /* From the iread template, initialize words 7 - 11 */ memcpy(&wqe->words[7], @@ -1249,13 +1241,13 @@ lpfc_nvme_prep_io_cmd(struct lpfc_vport *vport, /* Word 5 */ wqe->fcp_iread.rsrvd5 = 0; - atomic_inc(&cstat->fc4NvmeInputRequests); + cstat->input_requests++; } } else { /* From the icmnd template, initialize words 4 - 11 */ memcpy(&wqe->words[4], &lpfc_icmnd_cmd_template.words[4], sizeof(uint32_t) * 8); - atomic_inc(&cstat->fc4NvmeControlRequests); + cstat->control_requests++; } /* * Finish initializing those WQE fields that are independent @@ -1302,12 +1294,12 @@ lpfc_nvme_prep_io_cmd(struct lpfc_vport *vport, **/ static int lpfc_nvme_prep_io_dma(struct lpfc_vport *vport, - struct lpfc_nvme_buf *lpfc_ncmd) + struct lpfc_io_buf *lpfc_ncmd) { struct lpfc_hba *phba = vport->phba; struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd; union lpfc_wqe128 *wqe = &lpfc_ncmd->cur_iocbq.wqe; - struct sli4_sge *sgl = lpfc_ncmd->nvme_sgl; + struct sli4_sge *sgl = lpfc_ncmd->dma_sgl; struct scatterlist *data_sg; struct sli4_sge *first_data_sgl; struct ulp_bde64 *bde; @@ -1396,6 +1388,8 @@ lpfc_nvme_prep_io_dma(struct lpfc_vport *vport, } } else { + lpfc_ncmd->seg_cnt = 0; + /* For this clause to be valid, the payload_length * and sg_cnt must zero. */ @@ -1435,13 +1429,13 @@ lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport, { int ret = 0; int expedite = 0; - int idx; + int idx, cpu; struct lpfc_nvme_lport *lport; - struct lpfc_nvme_ctrl_stat *cstat; + struct lpfc_fc4_ctrl_stat *cstat; struct lpfc_vport *vport; struct lpfc_hba *phba; struct lpfc_nodelist *ndlp; - struct lpfc_nvme_buf *lpfc_ncmd; + struct lpfc_io_buf *lpfc_ncmd; struct lpfc_nvme_rport *rport; struct lpfc_nvme_qhandle *lpfc_queue_info; struct lpfc_nvme_fcpreq_priv *freqpriv; @@ -1559,7 +1553,15 @@ lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport, } } - lpfc_ncmd = lpfc_get_nvme_buf(phba, ndlp, expedite); + /* Lookup Hardware Queue index based on fcp_io_sched module parameter */ + if (phba->cfg_fcp_io_sched == LPFC_FCP_SCHED_BY_HDWQ) { + idx = lpfc_queue_info->index; + } else { + cpu = smp_processor_id(); + idx = phba->sli4_hba.cpu_map[cpu].hdwq; + } + + lpfc_ncmd = lpfc_get_nvme_buf(phba, ndlp, idx, expedite); if (lpfc_ncmd == NULL) { atomic_inc(&lport->xmt_fcp_noxri); lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, @@ -1586,9 +1588,8 @@ lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport, */ freqpriv->nvme_buf = lpfc_ncmd; lpfc_ncmd->nvmeCmd = pnvme_fcreq; - lpfc_ncmd->nrport = rport; lpfc_ncmd->ndlp = ndlp; - lpfc_ncmd->start_time = jiffies; + lpfc_ncmd->qidx = lpfc_queue_info->qidx; /* * Issue the IO on the WQ indicated by index in the hw_queue_handle. @@ -1598,9 +1599,8 @@ lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport, * index to use and that they have affinitized a CPU to this hardware * queue. A hardware queue maps to a driver MSI-X vector/EQ/CQ/WQ. */ - idx = lpfc_queue_info->index; lpfc_ncmd->cur_iocbq.hba_wqidx = idx; - cstat = &lport->cstat[idx]; + cstat = &phba->sli4_hba.hdwq[idx].nvme_cstat; lpfc_nvme_prep_io_cmd(vport, lpfc_ncmd, ndlp, cstat); ret = lpfc_nvme_prep_io_dma(vport, lpfc_ncmd); @@ -1618,7 +1618,7 @@ lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport, lpfc_ncmd->cur_iocbq.sli4_xritag, lpfc_queue_info->index, ndlp->nlp_DID); - ret = lpfc_sli4_issue_wqe(phba, LPFC_FCP_RING, &lpfc_ncmd->cur_iocbq); + ret = lpfc_sli4_issue_wqe(phba, lpfc_ncmd->hdwq, &lpfc_ncmd->cur_iocbq); if (ret) { atomic_inc(&lport->xmt_fcp_wqerr); lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, @@ -1629,26 +1629,26 @@ lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport, goto out_free_nvme_buf; } + if (phba->cfg_xri_rebalancing) + lpfc_keep_pvt_pool_above_lowwm(phba, lpfc_ncmd->hdwq_no); + #ifdef CONFIG_SCSI_LPFC_DEBUG_FS if (lpfc_ncmd->ts_cmd_start) lpfc_ncmd->ts_cmd_wqput = ktime_get_ns(); if (phba->cpucheck_on & LPFC_CHECK_NVME_IO) { - lpfc_ncmd->cpu = smp_processor_id(); - if (lpfc_ncmd->cpu != lpfc_queue_info->index) { - /* Check for admin queue */ - if (lpfc_queue_info->qidx) { + cpu = smp_processor_id(); + if (cpu < LPFC_CHECK_CPU_CNT) { + lpfc_ncmd->cpu = cpu; + if (idx != cpu) lpfc_printf_vlog(vport, - KERN_ERR, LOG_NVME_IOERR, + KERN_INFO, LOG_NVME_IOERR, "6702 CPU Check cmd: " "cpu %d wq %d\n", lpfc_ncmd->cpu, lpfc_queue_info->index); - } - lpfc_ncmd->cpu = lpfc_queue_info->index; + phba->sli4_hba.hdwq[idx].cpucheck_xmt_io[cpu]++; } - if (lpfc_ncmd->cpu < LPFC_CHECK_CPU_CNT) - phba->cpucheck_xmt_io[lpfc_ncmd->cpu]++; } #endif return 0; @@ -1656,11 +1656,11 @@ lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport, out_free_nvme_buf: if (lpfc_ncmd->nvmeCmd->sg_cnt) { if (lpfc_ncmd->nvmeCmd->io_dir == NVMEFC_FCP_WRITE) - atomic_dec(&cstat->fc4NvmeOutputRequests); + cstat->output_requests--; else - atomic_dec(&cstat->fc4NvmeInputRequests); + cstat->input_requests--; } else - atomic_dec(&cstat->fc4NvmeControlRequests); + cstat->control_requests--; lpfc_release_nvme_buf(phba, lpfc_ncmd); out_fail: return ret; @@ -1720,7 +1720,7 @@ lpfc_nvme_fcp_abort(struct nvme_fc_local_port *pnvme_lport, struct lpfc_nvme_lport *lport; struct lpfc_vport *vport; struct lpfc_hba *phba; - struct lpfc_nvme_buf *lpfc_nbuf; + struct lpfc_io_buf *lpfc_nbuf; struct lpfc_iocbq *abts_buf; struct lpfc_iocbq *nvmereq_wqe; struct lpfc_nvme_fcpreq_priv *freqpriv; @@ -1788,6 +1788,9 @@ lpfc_nvme_fcp_abort(struct nvme_fc_local_port *pnvme_lport, } nvmereq_wqe = &lpfc_nbuf->cur_iocbq; + /* Guard against IO completion being called at same time */ + spin_lock(&lpfc_nbuf->buf_lock); + /* * The lpfc_nbuf and the mapped nvme_fcreq in the driver's * state must match the nvme_fcreq passed by the nvme @@ -1796,24 +1799,22 @@ lpfc_nvme_fcp_abort(struct nvme_fc_local_port *pnvme_lport, * has not seen it yet. */ if (lpfc_nbuf->nvmeCmd != pnvme_fcreq) { - spin_unlock_irqrestore(&phba->hbalock, flags); lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, "6143 NVME req mismatch: " "lpfc_nbuf %p nvmeCmd %p, " "pnvme_fcreq %p. Skipping Abort xri x%x\n", lpfc_nbuf, lpfc_nbuf->nvmeCmd, pnvme_fcreq, nvmereq_wqe->sli4_xritag); - return; + goto out_unlock; } /* Don't abort IOs no longer on the pending queue. */ if (!(nvmereq_wqe->iocb_flag & LPFC_IO_ON_TXCMPLQ)) { - spin_unlock_irqrestore(&phba->hbalock, flags); lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, "6142 NVME IO req %p not queued - skipping " "abort req xri x%x\n", pnvme_fcreq, nvmereq_wqe->sli4_xritag); - return; + goto out_unlock; } atomic_inc(&lport->xmt_fcp_abort); @@ -1823,24 +1824,22 @@ lpfc_nvme_fcp_abort(struct nvme_fc_local_port *pnvme_lport, /* Outstanding abort is in progress */ if (nvmereq_wqe->iocb_flag & LPFC_DRIVER_ABORTED) { - spin_unlock_irqrestore(&phba->hbalock, flags); lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, "6144 Outstanding NVME I/O Abort Request " "still pending on nvme_fcreq %p, " "lpfc_ncmd %p xri x%x\n", pnvme_fcreq, lpfc_nbuf, nvmereq_wqe->sli4_xritag); - return; + goto out_unlock; } abts_buf = __lpfc_sli_get_iocbq(phba); if (!abts_buf) { - spin_unlock_irqrestore(&phba->hbalock, flags); lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, "6136 No available abort wqes. Skipping " "Abts req for nvme_fcreq %p xri x%x\n", pnvme_fcreq, nvmereq_wqe->sli4_xritag); - return; + goto out_unlock; } /* Ready - mark outstanding as aborted by driver. */ @@ -1883,7 +1882,8 @@ lpfc_nvme_fcp_abort(struct nvme_fc_local_port *pnvme_lport, abts_buf->hba_wqidx = nvmereq_wqe->hba_wqidx; abts_buf->vport = vport; abts_buf->wqe_cmpl = lpfc_nvme_abort_fcreq_cmpl; - ret_val = lpfc_sli4_issue_wqe(phba, LPFC_FCP_RING, abts_buf); + ret_val = lpfc_sli4_issue_wqe(phba, lpfc_nbuf->hdwq, abts_buf); + spin_unlock(&lpfc_nbuf->buf_lock); spin_unlock_irqrestore(&phba->hbalock, flags); if (ret_val) { lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, @@ -1899,6 +1899,12 @@ lpfc_nvme_fcp_abort(struct nvme_fc_local_port *pnvme_lport, "ox_id x%x on reqtag x%x\n", nvmereq_wqe->sli4_xritag, abts_buf->iotag); + return; + +out_unlock: + spin_unlock(&lpfc_nbuf->buf_lock); + spin_unlock_irqrestore(&phba->hbalock, flags); + return; } /* Declare and initialization an instance of the FC NVME template. */ @@ -1928,454 +1934,63 @@ static struct nvme_fc_port_template lpfc_nvme_template = { }; /** - * lpfc_sli4_post_nvme_sgl_block - post a block of nvme sgl list to firmware - * @phba: pointer to lpfc hba data structure. - * @nblist: pointer to nvme buffer list. - * @count: number of scsi buffers on the list. - * - * This routine is invoked to post a block of @count scsi sgl pages from a - * SCSI buffer list @nblist to the HBA using non-embedded mailbox command. - * No Lock is held. - * - **/ -static int -lpfc_sli4_post_nvme_sgl_block(struct lpfc_hba *phba, - struct list_head *nblist, - int count) -{ - struct lpfc_nvme_buf *lpfc_ncmd; - struct lpfc_mbx_post_uembed_sgl_page1 *sgl; - struct sgl_page_pairs *sgl_pg_pairs; - void *viraddr; - LPFC_MBOXQ_t *mbox; - uint32_t reqlen, alloclen, pg_pairs; - uint32_t mbox_tmo; - uint16_t xritag_start = 0; - int rc = 0; - uint32_t shdr_status, shdr_add_status; - dma_addr_t pdma_phys_bpl1; - union lpfc_sli4_cfg_shdr *shdr; - - /* Calculate the requested length of the dma memory */ - reqlen = count * sizeof(struct sgl_page_pairs) + - sizeof(union lpfc_sli4_cfg_shdr) + sizeof(uint32_t); - if (reqlen > SLI4_PAGE_SIZE) { - lpfc_printf_log(phba, KERN_WARNING, LOG_INIT, - "6118 Block sgl registration required DMA " - "size (%d) great than a page\n", reqlen); - return -ENOMEM; - } - mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL); - if (!mbox) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, - "6119 Failed to allocate mbox cmd memory\n"); - return -ENOMEM; - } - - /* Allocate DMA memory and set up the non-embedded mailbox command */ - alloclen = lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_FCOE, - LPFC_MBOX_OPCODE_FCOE_POST_SGL_PAGES, reqlen, - LPFC_SLI4_MBX_NEMBED); - - if (alloclen < reqlen) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, - "6120 Allocated DMA memory size (%d) is " - "less than the requested DMA memory " - "size (%d)\n", alloclen, reqlen); - lpfc_sli4_mbox_cmd_free(phba, mbox); - return -ENOMEM; - } - - /* Get the first SGE entry from the non-embedded DMA memory */ - viraddr = mbox->sge_array->addr[0]; - - /* Set up the SGL pages in the non-embedded DMA pages */ - sgl = (struct lpfc_mbx_post_uembed_sgl_page1 *)viraddr; - sgl_pg_pairs = &sgl->sgl_pg_pairs; - - pg_pairs = 0; - list_for_each_entry(lpfc_ncmd, nblist, list) { - /* Set up the sge entry */ - sgl_pg_pairs->sgl_pg0_addr_lo = - cpu_to_le32(putPaddrLow(lpfc_ncmd->dma_phys_sgl)); - sgl_pg_pairs->sgl_pg0_addr_hi = - cpu_to_le32(putPaddrHigh(lpfc_ncmd->dma_phys_sgl)); - if (phba->cfg_sg_dma_buf_size > SGL_PAGE_SIZE) - pdma_phys_bpl1 = lpfc_ncmd->dma_phys_sgl + - SGL_PAGE_SIZE; - else - pdma_phys_bpl1 = 0; - sgl_pg_pairs->sgl_pg1_addr_lo = - cpu_to_le32(putPaddrLow(pdma_phys_bpl1)); - sgl_pg_pairs->sgl_pg1_addr_hi = - cpu_to_le32(putPaddrHigh(pdma_phys_bpl1)); - /* Keep the first xritag on the list */ - if (pg_pairs == 0) - xritag_start = lpfc_ncmd->cur_iocbq.sli4_xritag; - sgl_pg_pairs++; - pg_pairs++; - } - bf_set(lpfc_post_sgl_pages_xri, sgl, xritag_start); - bf_set(lpfc_post_sgl_pages_xricnt, sgl, pg_pairs); - /* Perform endian conversion if necessary */ - sgl->word0 = cpu_to_le32(sgl->word0); - - if (!phba->sli4_hba.intr_enable) - rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL); - else { - mbox_tmo = lpfc_mbox_tmo_val(phba, mbox); - rc = lpfc_sli_issue_mbox_wait(phba, mbox, mbox_tmo); - } - shdr = (union lpfc_sli4_cfg_shdr *)&sgl->cfg_shdr; - shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response); - shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response); - if (rc != MBX_TIMEOUT) - lpfc_sli4_mbox_cmd_free(phba, mbox); - if (shdr_status || shdr_add_status || rc) { - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, - "6125 POST_SGL_BLOCK mailbox command failed " - "status x%x add_status x%x mbx status x%x\n", - shdr_status, shdr_add_status, rc); - rc = -ENXIO; - } - return rc; -} - -/** - * lpfc_post_nvme_sgl_list - Post blocks of nvme buffer sgls from a list - * @phba: pointer to lpfc hba data structure. - * @post_nblist: pointer to the nvme buffer list. - * - * This routine walks a list of nvme buffers that was passed in. It attempts - * to construct blocks of nvme buffer sgls which contains contiguous xris and - * uses the non-embedded SGL block post mailbox commands to post to the port. - * For single NVME buffer sgl with non-contiguous xri, if any, it shall use - * embedded SGL post mailbox command for posting. The @post_nblist passed in - * must be local list, thus no lock is needed when manipulate the list. - * - * Returns: 0 = failure, non-zero number of successfully posted buffers. - **/ -static int -lpfc_post_nvme_sgl_list(struct lpfc_hba *phba, - struct list_head *post_nblist, int sb_count) -{ - struct lpfc_nvme_buf *lpfc_ncmd, *lpfc_ncmd_next; - int status, sgl_size; - int post_cnt = 0, block_cnt = 0, num_posting = 0, num_posted = 0; - dma_addr_t pdma_phys_sgl1; - int last_xritag = NO_XRI; - int cur_xritag; - LIST_HEAD(prep_nblist); - LIST_HEAD(blck_nblist); - LIST_HEAD(nvme_nblist); - - /* sanity check */ - if (sb_count <= 0) - return -EINVAL; - - sgl_size = phba->cfg_sg_dma_buf_size; - - list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next, post_nblist, list) { - list_del_init(&lpfc_ncmd->list); - block_cnt++; - if ((last_xritag != NO_XRI) && - (lpfc_ncmd->cur_iocbq.sli4_xritag != last_xritag + 1)) { - /* a hole in xri block, form a sgl posting block */ - list_splice_init(&prep_nblist, &blck_nblist); - post_cnt = block_cnt - 1; - /* prepare list for next posting block */ - list_add_tail(&lpfc_ncmd->list, &prep_nblist); - block_cnt = 1; - } else { - /* prepare list for next posting block */ - list_add_tail(&lpfc_ncmd->list, &prep_nblist); - /* enough sgls for non-embed sgl mbox command */ - if (block_cnt == LPFC_NEMBED_MBOX_SGL_CNT) { - list_splice_init(&prep_nblist, &blck_nblist); - post_cnt = block_cnt; - block_cnt = 0; - } - } - num_posting++; - last_xritag = lpfc_ncmd->cur_iocbq.sli4_xritag; - - /* end of repost sgl list condition for NVME buffers */ - if (num_posting == sb_count) { - if (post_cnt == 0) { - /* last sgl posting block */ - list_splice_init(&prep_nblist, &blck_nblist); - post_cnt = block_cnt; - } else if (block_cnt == 1) { - /* last single sgl with non-contiguous xri */ - if (sgl_size > SGL_PAGE_SIZE) - pdma_phys_sgl1 = - lpfc_ncmd->dma_phys_sgl + - SGL_PAGE_SIZE; - else - pdma_phys_sgl1 = 0; - cur_xritag = lpfc_ncmd->cur_iocbq.sli4_xritag; - status = lpfc_sli4_post_sgl(phba, - lpfc_ncmd->dma_phys_sgl, - pdma_phys_sgl1, cur_xritag); - if (status) { - /* failure, put on abort nvme list */ - lpfc_ncmd->flags |= LPFC_SBUF_XBUSY; - } else { - /* success, put on NVME buffer list */ - lpfc_ncmd->flags &= ~LPFC_SBUF_XBUSY; - lpfc_ncmd->status = IOSTAT_SUCCESS; - num_posted++; - } - /* success, put on NVME buffer sgl list */ - list_add_tail(&lpfc_ncmd->list, &nvme_nblist); - } - } - - /* continue until a nembed page worth of sgls */ - if (post_cnt == 0) - continue; - - /* post block of NVME buffer list sgls */ - status = lpfc_sli4_post_nvme_sgl_block(phba, &blck_nblist, - post_cnt); - - /* don't reset xirtag due to hole in xri block */ - if (block_cnt == 0) - last_xritag = NO_XRI; - - /* reset NVME buffer post count for next round of posting */ - post_cnt = 0; - - /* put posted NVME buffer-sgl posted on NVME buffer sgl list */ - while (!list_empty(&blck_nblist)) { - list_remove_head(&blck_nblist, lpfc_ncmd, - struct lpfc_nvme_buf, list); - if (status) { - /* failure, put on abort nvme list */ - lpfc_ncmd->flags |= LPFC_SBUF_XBUSY; - } else { - /* success, put on NVME buffer list */ - lpfc_ncmd->flags &= ~LPFC_SBUF_XBUSY; - lpfc_ncmd->status = IOSTAT_SUCCESS; - num_posted++; - } - list_add_tail(&lpfc_ncmd->list, &nvme_nblist); - } - } - /* Push NVME buffers with sgl posted to the available list */ - while (!list_empty(&nvme_nblist)) { - list_remove_head(&nvme_nblist, lpfc_ncmd, - struct lpfc_nvme_buf, list); - lpfc_release_nvme_buf(phba, lpfc_ncmd); - } - return num_posted; -} - -/** - * lpfc_repost_nvme_sgl_list - Repost all the allocated nvme buffer sgls - * @phba: pointer to lpfc hba data structure. - * - * This routine walks the list of nvme buffers that have been allocated and - * repost them to the port by using SGL block post. This is needed after a - * pci_function_reset/warm_start or start. The lpfc_hba_down_post_s4 routine - * is responsible for moving all nvme buffers on the lpfc_abts_nvme_sgl_list - * to the lpfc_nvme_buf_list. If the repost fails, reject all nvme buffers. - * - * Returns: 0 = success, non-zero failure. - **/ -int -lpfc_repost_nvme_sgl_list(struct lpfc_hba *phba) -{ - LIST_HEAD(post_nblist); - int num_posted, rc = 0; - - /* get all NVME buffers need to repost to a local list */ - spin_lock_irq(&phba->nvme_buf_list_get_lock); - spin_lock(&phba->nvme_buf_list_put_lock); - list_splice_init(&phba->lpfc_nvme_buf_list_get, &post_nblist); - list_splice(&phba->lpfc_nvme_buf_list_put, &post_nblist); - phba->get_nvme_bufs = 0; - phba->put_nvme_bufs = 0; - spin_unlock(&phba->nvme_buf_list_put_lock); - spin_unlock_irq(&phba->nvme_buf_list_get_lock); - - /* post the list of nvme buffer sgls to port if available */ - if (!list_empty(&post_nblist)) { - num_posted = lpfc_post_nvme_sgl_list(phba, &post_nblist, - phba->sli4_hba.nvme_xri_cnt); - /* failed to post any nvme buffer, return error */ - if (num_posted == 0) - rc = -EIO; - } - return rc; -} - -/** - * lpfc_new_nvme_buf - Scsi buffer allocator for HBA with SLI4 IF spec - * @vport: The virtual port for which this call being executed. - * @num_to_allocate: The requested number of buffers to allocate. + * lpfc_get_nvme_buf - Get a nvme buffer from io_buf_list of the HBA + * @phba: The HBA for which this call is being executed. * - * This routine allocates nvme buffers for device with SLI-4 interface spec, - * the nvme buffer contains all the necessary information needed to initiate - * a NVME I/O. After allocating up to @num_to_allocate NVME buffers and put - * them on a list, it post them to the port by using SGL block post. + * This routine removes a nvme buffer from head of @hdwq io_buf_list + * and returns to caller. * * Return codes: - * int - number of nvme buffers that were allocated and posted. - * 0 = failure, less than num_to_alloc is a partial failure. + * NULL - Error + * Pointer to lpfc_nvme_buf - Success **/ -static int -lpfc_new_nvme_buf(struct lpfc_vport *vport, int num_to_alloc) +static struct lpfc_io_buf * +lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, + int idx, int expedite) { - struct lpfc_hba *phba = vport->phba; - struct lpfc_nvme_buf *lpfc_ncmd; + struct lpfc_io_buf *lpfc_ncmd; + struct lpfc_sli4_hdw_queue *qp; + struct sli4_sge *sgl; struct lpfc_iocbq *pwqeq; union lpfc_wqe128 *wqe; - struct sli4_sge *sgl; - dma_addr_t pdma_phys_sgl; - uint16_t iotag, lxri = 0; - int bcnt, num_posted; - LIST_HEAD(prep_nblist); - LIST_HEAD(post_nblist); - LIST_HEAD(nvme_nblist); - - for (bcnt = 0; bcnt < num_to_alloc; bcnt++) { - lpfc_ncmd = kzalloc(sizeof(struct lpfc_nvme_buf), GFP_KERNEL); - if (!lpfc_ncmd) - break; - /* - * Get memory from the pci pool to map the virt space to - * pci bus space for an I/O. The DMA buffer includes the - * number of SGE's necessary to support the sg_tablesize. - */ - lpfc_ncmd->data = dma_pool_zalloc(phba->lpfc_sg_dma_buf_pool, - GFP_KERNEL, - &lpfc_ncmd->dma_handle); - if (!lpfc_ncmd->data) { - kfree(lpfc_ncmd); - break; - } - lxri = lpfc_sli4_next_xritag(phba); - if (lxri == NO_XRI) { - dma_pool_free(phba->lpfc_sg_dma_buf_pool, - lpfc_ncmd->data, lpfc_ncmd->dma_handle); - kfree(lpfc_ncmd); - break; - } + lpfc_ncmd = lpfc_get_io_buf(phba, NULL, idx, expedite); + + if (lpfc_ncmd) { pwqeq = &(lpfc_ncmd->cur_iocbq); wqe = &pwqeq->wqe; - /* Allocate iotag for lpfc_ncmd->cur_iocbq. */ - iotag = lpfc_sli_next_iotag(phba, pwqeq); - if (iotag == 0) { - dma_pool_free(phba->lpfc_sg_dma_buf_pool, - lpfc_ncmd->data, lpfc_ncmd->dma_handle); - kfree(lpfc_ncmd); - lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, - "6121 Failed to allocated IOTAG for" - " XRI:0x%x\n", lxri); - lpfc_sli4_free_xri(phba, lxri); - break; - } - pwqeq->sli4_lxritag = lxri; - pwqeq->sli4_xritag = phba->sli4_hba.xri_ids[lxri]; - pwqeq->iocb_flag |= LPFC_IO_NVME; - pwqeq->context1 = lpfc_ncmd; + /* Setup key fields in buffer that may have been changed + * if other protocols used this buffer. + */ + pwqeq->iocb_flag = LPFC_IO_NVME; pwqeq->wqe_cmpl = lpfc_nvme_io_cmd_wqe_cmpl; - - /* Initialize local short-hand pointers. */ - lpfc_ncmd->nvme_sgl = lpfc_ncmd->data; - sgl = lpfc_ncmd->nvme_sgl; - pdma_phys_sgl = lpfc_ncmd->dma_handle; - lpfc_ncmd->dma_phys_sgl = pdma_phys_sgl; + lpfc_ncmd->start_time = jiffies; + lpfc_ncmd->flags = 0; /* Rsp SGE will be filled in when we rcv an IO * from the NVME Layer to be sent. * The cmd is going to be embedded so we need a SKIP SGE. */ + sgl = lpfc_ncmd->dma_sgl; bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP); bf_set(lpfc_sli4_sge_last, sgl, 0); sgl->word2 = cpu_to_le32(sgl->word2); /* Fill in word 3 / sgl_len during cmd submission */ - lpfc_ncmd->cur_iocbq.context1 = lpfc_ncmd; - /* Initialize WQE */ memset(wqe, 0, sizeof(union lpfc_wqe)); - /* add the nvme buffer to a post list */ - list_add_tail(&lpfc_ncmd->list, &post_nblist); - spin_lock_irq(&phba->nvme_buf_list_get_lock); - phba->sli4_hba.nvme_xri_cnt++; - spin_unlock_irq(&phba->nvme_buf_list_get_lock); - } - lpfc_printf_log(phba, KERN_INFO, LOG_NVME, - "6114 Allocate %d out of %d requested new NVME " - "buffers\n", bcnt, num_to_alloc); - - /* post the list of nvme buffer sgls to port if available */ - if (!list_empty(&post_nblist)) - num_posted = lpfc_post_nvme_sgl_list(phba, - &post_nblist, bcnt); - else - num_posted = 0; - - return num_posted; -} - -static inline struct lpfc_nvme_buf * -lpfc_nvme_buf(struct lpfc_hba *phba) -{ - struct lpfc_nvme_buf *lpfc_ncmd, *lpfc_ncmd_next; - - list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next, - &phba->lpfc_nvme_buf_list_get, list) { - list_del_init(&lpfc_ncmd->list); - phba->get_nvme_bufs--; - return lpfc_ncmd; - } - return NULL; -} - -/** - * lpfc_get_nvme_buf - Get a nvme buffer from lpfc_nvme_buf_list of the HBA - * @phba: The HBA for which this call is being executed. - * - * This routine removes a nvme buffer from head of @phba lpfc_nvme_buf_list list - * and returns to caller. - * - * Return codes: - * NULL - Error - * Pointer to lpfc_nvme_buf - Success - **/ -static struct lpfc_nvme_buf * -lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, - int expedite) -{ - struct lpfc_nvme_buf *lpfc_ncmd = NULL; - unsigned long iflag = 0; + if (lpfc_ndlp_check_qdepth(phba, ndlp)) { + atomic_inc(&ndlp->cmd_pending); + lpfc_ncmd->flags |= LPFC_SBUF_BUMP_QDEPTH; + } - spin_lock_irqsave(&phba->nvme_buf_list_get_lock, iflag); - if (phba->get_nvme_bufs > LPFC_NVME_EXPEDITE_XRICNT || expedite) - lpfc_ncmd = lpfc_nvme_buf(phba); - if (!lpfc_ncmd) { - spin_lock(&phba->nvme_buf_list_put_lock); - list_splice(&phba->lpfc_nvme_buf_list_put, - &phba->lpfc_nvme_buf_list_get); - phba->get_nvme_bufs += phba->put_nvme_bufs; - INIT_LIST_HEAD(&phba->lpfc_nvme_buf_list_put); - phba->put_nvme_bufs = 0; - spin_unlock(&phba->nvme_buf_list_put_lock); - if (phba->get_nvme_bufs > LPFC_NVME_EXPEDITE_XRICNT || expedite) - lpfc_ncmd = lpfc_nvme_buf(phba); + } else { + qp = &phba->sli4_hba.hdwq[idx]; + qp->empty_io_bufs++; } - spin_unlock_irqrestore(&phba->nvme_buf_list_get_lock, iflag); - if (lpfc_ndlp_check_qdepth(phba, ndlp) && lpfc_ncmd) { - atomic_inc(&ndlp->cmd_pending); - lpfc_ncmd->flags |= LPFC_BUMP_QDEPTH; - } return lpfc_ncmd; } @@ -2385,22 +2000,23 @@ lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, * @lpfc_ncmd: The nvme buffer which is being released. * * This routine releases @lpfc_ncmd nvme buffer by adding it to tail of @phba - * lpfc_nvme_buf_list list. For SLI4 XRI's are tied to the nvme buffer + * lpfc_io_buf_list list. For SLI4 XRI's are tied to the nvme buffer * and cannot be reused for at least RA_TOV amount of time if it was * aborted. **/ static void -lpfc_release_nvme_buf(struct lpfc_hba *phba, struct lpfc_nvme_buf *lpfc_ncmd) +lpfc_release_nvme_buf(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_ncmd) { + struct lpfc_sli4_hdw_queue *qp; unsigned long iflag = 0; - if ((lpfc_ncmd->flags & LPFC_BUMP_QDEPTH) && lpfc_ncmd->ndlp) + if ((lpfc_ncmd->flags & LPFC_SBUF_BUMP_QDEPTH) && lpfc_ncmd->ndlp) atomic_dec(&lpfc_ncmd->ndlp->cmd_pending); - lpfc_ncmd->nonsg_phys = 0; lpfc_ncmd->ndlp = NULL; - lpfc_ncmd->flags &= ~LPFC_BUMP_QDEPTH; + lpfc_ncmd->flags &= ~LPFC_SBUF_BUMP_QDEPTH; + qp = lpfc_ncmd->hdwq; if (lpfc_ncmd->flags & LPFC_SBUF_XBUSY) { lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, "6310 XB release deferred for " @@ -2408,20 +2024,13 @@ lpfc_release_nvme_buf(struct lpfc_hba *phba, struct lpfc_nvme_buf *lpfc_ncmd) lpfc_ncmd->cur_iocbq.sli4_xritag, lpfc_ncmd->cur_iocbq.iotag); - spin_lock_irqsave(&phba->sli4_hba.abts_nvme_buf_list_lock, - iflag); + spin_lock_irqsave(&qp->abts_nvme_buf_list_lock, iflag); list_add_tail(&lpfc_ncmd->list, - &phba->sli4_hba.lpfc_abts_nvme_buf_list); - spin_unlock_irqrestore(&phba->sli4_hba.abts_nvme_buf_list_lock, - iflag); - } else { - lpfc_ncmd->nvmeCmd = NULL; - lpfc_ncmd->cur_iocbq.iocb_flag = LPFC_IO_NVME; - spin_lock_irqsave(&phba->nvme_buf_list_put_lock, iflag); - list_add_tail(&lpfc_ncmd->list, &phba->lpfc_nvme_buf_list_put); - phba->put_nvme_bufs++; - spin_unlock_irqrestore(&phba->nvme_buf_list_put_lock, iflag); - } + &qp->lpfc_abts_nvme_buf_list); + qp->abts_nvme_io_bufs++; + spin_unlock_irqrestore(&qp->abts_nvme_buf_list_lock, iflag); + } else + lpfc_release_io_buf(phba, (struct lpfc_io_buf *)lpfc_ncmd, qp); } /** @@ -2448,8 +2057,6 @@ lpfc_nvme_create_localport(struct lpfc_vport *vport) struct nvme_fc_port_info nfcp_info; struct nvme_fc_local_port *localport; struct lpfc_nvme_lport *lport; - struct lpfc_nvme_ctrl_stat *cstat; - int len, i; /* Initialize this localport instance. The vport wwn usage ensures * that NPIV is accounted for. @@ -2464,12 +2071,13 @@ lpfc_nvme_create_localport(struct lpfc_vport *vport) * allocate + 3, one for cmd, one for rsp and one for this alignment */ lpfc_nvme_template.max_sgl_segments = phba->cfg_nvme_seg_cnt + 1; - lpfc_nvme_template.max_hw_queues = phba->cfg_nvme_io_channel; - cstat = kmalloc((sizeof(struct lpfc_nvme_ctrl_stat) * - phba->cfg_nvme_io_channel), GFP_KERNEL); - if (!cstat) - return -ENOMEM; + /* Advertise how many hw queues we support based on fcp_io_sched */ + if (phba->cfg_fcp_io_sched == LPFC_FCP_SCHED_BY_HDWQ) + lpfc_nvme_template.max_hw_queues = phba->cfg_hdw_queue; + else + lpfc_nvme_template.max_hw_queues = + phba->sli4_hba.num_present_cpu; /* localport is allocated from the stack, but the registration * call allocates heap memory as well as the private area. @@ -2493,7 +2101,6 @@ lpfc_nvme_create_localport(struct lpfc_vport *vport) lport = (struct lpfc_nvme_lport *)localport->private; vport->localport = localport; lport->vport = vport; - lport->cstat = cstat; vport->nvmei_support = 1; atomic_set(&lport->xmt_fcp_noxri, 0); @@ -2510,25 +2117,6 @@ lpfc_nvme_create_localport(struct lpfc_vport *vport) atomic_set(&lport->cmpl_ls_err, 0); atomic_set(&lport->fc4NvmeLsRequests, 0); atomic_set(&lport->fc4NvmeLsCmpls, 0); - - for (i = 0; i < phba->cfg_nvme_io_channel; i++) { - cstat = &lport->cstat[i]; - atomic_set(&cstat->fc4NvmeInputRequests, 0); - atomic_set(&cstat->fc4NvmeOutputRequests, 0); - atomic_set(&cstat->fc4NvmeControlRequests, 0); - atomic_set(&cstat->fc4NvmeIoCmpls, 0); - } - - /* Don't post more new bufs if repost already recovered - * the nvme sgls. - */ - if (phba->sli4_hba.nvme_xri_cnt == 0) { - len = lpfc_new_nvme_buf(vport, - phba->sli4_hba.nvme_xri_max); - vport->phba->total_nvme_bufs += len; - } - } else { - kfree(cstat); } return ret; @@ -2591,7 +2179,6 @@ lpfc_nvme_destroy_localport(struct lpfc_vport *vport) #if (IS_ENABLED(CONFIG_NVME_FC)) struct nvme_fc_local_port *localport; struct lpfc_nvme_lport *lport; - struct lpfc_nvme_ctrl_stat *cstat; int ret; DECLARE_COMPLETION_ONSTACK(lport_unreg_cmp); @@ -2600,7 +2187,6 @@ lpfc_nvme_destroy_localport(struct lpfc_vport *vport) localport = vport->localport; lport = (struct lpfc_nvme_lport *)localport->private; - cstat = lport->cstat; lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME, "6011 Destroying NVME localport %p\n", @@ -2617,7 +2203,6 @@ lpfc_nvme_destroy_localport(struct lpfc_vport *vport) */ lpfc_nvme_lport_unreg_wait(vport, lport, &lport_unreg_cmp); vport->localport = NULL; - kfree(cstat); /* Regardless of the unregister upcall response, clear * nvmei_support. All rports are unregistered and the @@ -2909,27 +2494,28 @@ lpfc_nvme_unregister_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp) **/ void lpfc_sli4_nvme_xri_aborted(struct lpfc_hba *phba, - struct sli4_wcqe_xri_aborted *axri) + struct sli4_wcqe_xri_aborted *axri, int idx) { uint16_t xri = bf_get(lpfc_wcqe_xa_xri, axri); - struct lpfc_nvme_buf *lpfc_ncmd, *next_lpfc_ncmd; + struct lpfc_io_buf *lpfc_ncmd, *next_lpfc_ncmd; struct nvmefc_fcp_req *nvme_cmd = NULL; struct lpfc_nodelist *ndlp; + struct lpfc_sli4_hdw_queue *qp; unsigned long iflag = 0; if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME)) return; + qp = &phba->sli4_hba.hdwq[idx]; spin_lock_irqsave(&phba->hbalock, iflag); - spin_lock(&phba->sli4_hba.abts_nvme_buf_list_lock); + spin_lock(&qp->abts_nvme_buf_list_lock); list_for_each_entry_safe(lpfc_ncmd, next_lpfc_ncmd, - &phba->sli4_hba.lpfc_abts_nvme_buf_list, - list) { + &qp->lpfc_abts_nvme_buf_list, list) { if (lpfc_ncmd->cur_iocbq.sli4_xritag == xri) { list_del_init(&lpfc_ncmd->list); + qp->abts_nvme_io_bufs--; lpfc_ncmd->flags &= ~LPFC_SBUF_XBUSY; lpfc_ncmd->status = IOSTAT_SUCCESS; - spin_unlock( - &phba->sli4_hba.abts_nvme_buf_list_lock); + spin_unlock(&qp->abts_nvme_buf_list_lock); spin_unlock_irqrestore(&phba->hbalock, iflag); ndlp = lpfc_ncmd->ndlp; @@ -2955,7 +2541,7 @@ lpfc_sli4_nvme_xri_aborted(struct lpfc_hba *phba, return; } } - spin_unlock(&phba->sli4_hba.abts_nvme_buf_list_lock); + spin_unlock(&qp->abts_nvme_buf_list_lock); spin_unlock_irqrestore(&phba->hbalock, iflag); lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, @@ -2979,14 +2565,16 @@ lpfc_nvme_wait_for_io_drain(struct lpfc_hba *phba) struct lpfc_sli_ring *pring; u32 i, wait_cnt = 0; - if (phba->sli_rev < LPFC_SLI_REV4 || !phba->sli4_hba.nvme_wq) + if (phba->sli_rev < LPFC_SLI_REV4 || !phba->sli4_hba.hdwq) return; /* Cycle through all NVME rings and make sure all outstanding * WQEs have been removed from the txcmplqs. */ - for (i = 0; i < phba->cfg_nvme_io_channel; i++) { - pring = phba->sli4_hba.nvme_wq[i]->pring; + for (i = 0; i < phba->cfg_hdw_queue; i++) { + if (!phba->sli4_hba.hdwq[i].nvme_wq) + continue; + pring = phba->sli4_hba.hdwq[i].nvme_wq->pring; if (!pring) continue; diff --git a/drivers/scsi/lpfc/lpfc_nvme.h b/drivers/scsi/lpfc/lpfc_nvme.h index b234d0298994..593c48ff634e 100644 --- a/drivers/scsi/lpfc/lpfc_nvme.h +++ b/drivers/scsi/lpfc/lpfc_nvme.h @@ -1,7 +1,7 @@ /******************************************************************* * This file is part of the Emulex Linux Device Driver for * * Fibre Channel Host Bus Adapters. * - * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term * + * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term * * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * * Copyright (C) 2004-2016 Emulex. All rights reserved. * * EMULEX and SLI are trademarks of Emulex. * @@ -30,6 +30,9 @@ #define LPFC_NVME_FB_SHIFT 9 #define LPFC_NVME_MAX_FB (1 << 20) /* 1M */ +#define LPFC_MAX_NVME_INFO_TMP_LEN 100 +#define LPFC_NVME_INFO_MORE_STR "\nCould be more info...\n" + #define lpfc_ndlp_get_nrport(ndlp) \ ((!ndlp->nrport || (ndlp->upcall_flags & NLP_WAIT_FOR_UNREG)) \ ? NULL : ndlp->nrport) @@ -40,19 +43,11 @@ struct lpfc_nvme_qhandle { uint32_t cpu_id; /* current cpu id at time of create */ }; -struct lpfc_nvme_ctrl_stat { - atomic_t fc4NvmeInputRequests; - atomic_t fc4NvmeOutputRequests; - atomic_t fc4NvmeControlRequests; - atomic_t fc4NvmeIoCmpls; -}; - /* Declare nvme-based local and remote port definitions. */ struct lpfc_nvme_lport { struct lpfc_vport *vport; struct completion *lport_unreg_cmp; /* Add stats counters here */ - struct lpfc_nvme_ctrl_stat *cstat; atomic_t fc4NvmeLsRequests; atomic_t fc4NvmeLsCmpls; atomic_t xmt_fcp_noxri; @@ -76,57 +71,6 @@ struct lpfc_nvme_rport { struct completion rport_unreg_done; }; -struct lpfc_nvme_buf { - struct list_head list; - struct nvmefc_fcp_req *nvmeCmd; - struct lpfc_nvme_rport *nrport; - struct lpfc_nodelist *ndlp; - - uint32_t timeout; - - uint16_t flags; /* TBD convert exch_busy to flags */ -#define LPFC_SBUF_XBUSY 0x1 /* SLI4 hba reported XB on WCQE cmpl */ -#define LPFC_BUMP_QDEPTH 0x2 /* bumped queue depth counter */ - uint16_t exch_busy; /* SLI4 hba reported XB on complete WCQE */ - uint16_t status; /* From IOCB Word 7- ulpStatus */ - uint16_t cpu; - uint16_t qidx; - uint16_t sqid; - uint32_t result; /* From IOCB Word 4. */ - - uint32_t seg_cnt; /* Number of scatter-gather segments returned by - * dma_map_sg. The driver needs this for calls - * to dma_unmap_sg. - */ - dma_addr_t nonsg_phys; /* Non scatter-gather physical address. */ - - /* - * data and dma_handle are the kernel virtual and bus address of the - * dma-able buffer containing the fcp_cmd, fcp_rsp and a scatter - * gather bde list that supports the sg_tablesize value. - */ - void *data; - dma_addr_t dma_handle; - - struct sli4_sge *nvme_sgl; - dma_addr_t dma_phys_sgl; - - /* cur_iocbq has phys of the dma-able buffer. - * Iotag is in here - */ - struct lpfc_iocbq cur_iocbq; - - wait_queue_head_t *waitq; - unsigned long start_time; -#ifdef CONFIG_SCSI_LPFC_DEBUG_FS - uint64_t ts_cmd_start; - uint64_t ts_last_cmd; - uint64_t ts_cmd_wqput; - uint64_t ts_isr_cmpl; - uint64_t ts_data_nvme; -#endif -}; - struct lpfc_nvme_fcpreq_priv { - struct lpfc_nvme_buf *nvme_buf; + struct lpfc_io_buf *nvme_buf; }; diff --git a/drivers/scsi/lpfc/lpfc_nvmet.c b/drivers/scsi/lpfc/lpfc_nvmet.c index 95fee83090eb..361e2b103648 100644 --- a/drivers/scsi/lpfc/lpfc_nvmet.c +++ b/drivers/scsi/lpfc/lpfc_nvmet.c @@ -1,7 +1,7 @@ /******************************************************************* * This file is part of the Emulex Linux Device Driver for * * Fibre Channsel Host Bus Adapters. * - * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term * + * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term * * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * * Copyright (C) 2004-2016 Emulex. All rights reserved. * * EMULEX and SLI are trademarks of Emulex. * @@ -73,6 +73,9 @@ static int lpfc_nvmet_unsol_ls_issue_abort(struct lpfc_hba *, uint32_t, uint16_t); static void lpfc_nvmet_wqfull_flush(struct lpfc_hba *, struct lpfc_queue *, struct lpfc_nvmet_rcv_ctx *); +static void lpfc_nvmet_fcp_rqst_defer_work(struct work_struct *); + +static void lpfc_nvmet_process_rcv_fcp_req(struct lpfc_nvmet_ctxbuf *ctx_buf); static union lpfc_wqe128 lpfc_tsend_cmd_template; static union lpfc_wqe128 lpfc_treceive_cmd_template; @@ -220,21 +223,19 @@ lpfc_nvmet_cmd_template(void) void lpfc_nvmet_defer_release(struct lpfc_hba *phba, struct lpfc_nvmet_rcv_ctx *ctxp) { - unsigned long iflag; + lockdep_assert_held(&ctxp->ctxlock); lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, "6313 NVMET Defer ctx release xri x%x flg x%x\n", ctxp->oxid, ctxp->flag); - spin_lock_irqsave(&phba->sli4_hba.abts_nvme_buf_list_lock, iflag); - if (ctxp->flag & LPFC_NVMET_CTX_RLS) { - spin_unlock_irqrestore(&phba->sli4_hba.abts_nvme_buf_list_lock, - iflag); + if (ctxp->flag & LPFC_NVMET_CTX_RLS) return; - } + ctxp->flag |= LPFC_NVMET_CTX_RLS; + spin_lock(&phba->sli4_hba.abts_nvmet_buf_list_lock); list_add_tail(&ctxp->list, &phba->sli4_hba.lpfc_abts_nvmet_ctx_list); - spin_unlock_irqrestore(&phba->sli4_hba.abts_nvme_buf_list_lock, iflag); + spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock); } /** @@ -325,7 +326,7 @@ lpfc_nvmet_ctxbuf_post(struct lpfc_hba *phba, struct lpfc_nvmet_ctxbuf *ctx_buf) struct rqb_dmabuf *nvmebuf; struct lpfc_nvmet_ctx_info *infop; uint32_t *payload; - uint32_t size, oxid, sid, rc; + uint32_t size, oxid, sid; int cpu; unsigned long iflag; @@ -341,6 +342,20 @@ lpfc_nvmet_ctxbuf_post(struct lpfc_hba *phba, struct lpfc_nvmet_ctxbuf *ctx_buf) "6411 NVMET free, already free IO x%x: %d %d\n", ctxp->oxid, ctxp->state, ctxp->entry_cnt); } + + if (ctxp->rqb_buffer) { + nvmebuf = ctxp->rqb_buffer; + spin_lock_irqsave(&ctxp->ctxlock, iflag); + ctxp->rqb_buffer = NULL; + if (ctxp->flag & LPFC_NVMET_CTX_REUSE_WQ) { + ctxp->flag &= ~LPFC_NVMET_CTX_REUSE_WQ; + spin_unlock_irqrestore(&ctxp->ctxlock, iflag); + nvmebuf->hrq->rqbp->rqb_free_buffer(phba, nvmebuf); + } else { + spin_unlock_irqrestore(&ctxp->ctxlock, iflag); + lpfc_rq_buf_free(phba, &nvmebuf->hbuf); /* repost */ + } + } ctxp->state = LPFC_NVMET_STE_FREE; spin_lock_irqsave(&phba->sli4_hba.nvmet_io_wait_lock, iflag); @@ -388,46 +403,30 @@ lpfc_nvmet_ctxbuf_post(struct lpfc_hba *phba, struct lpfc_nvmet_ctxbuf *ctx_buf) } #endif atomic_inc(&tgtp->rcv_fcp_cmd_in); - /* - * The calling sequence should be: - * nvmet_fc_rcv_fcp_req->lpfc_nvmet_xmt_fcp_op/cmp- req->done - * lpfc_nvmet_xmt_fcp_op_cmp should free the allocated ctxp. - * When we return from nvmet_fc_rcv_fcp_req, all relevant info - * the NVME command / FC header is stored. - * A buffer has already been reposted for this IO, so just free - * the nvmebuf. - */ - rc = nvmet_fc_rcv_fcp_req(phba->targetport, &ctxp->ctx.fcp_req, - payload, size); - /* Process FCP command */ - if (rc == 0) { - ctxp->rqb_buffer = NULL; - atomic_inc(&tgtp->rcv_fcp_cmd_out); - nvmebuf->hrq->rqbp->rqb_free_buffer(phba, nvmebuf); - return; - } + /* flag new work queued, replacement buffer has already + * been reposted + */ + spin_lock_irqsave(&ctxp->ctxlock, iflag); + ctxp->flag |= LPFC_NVMET_CTX_REUSE_WQ; + spin_unlock_irqrestore(&ctxp->ctxlock, iflag); - /* Processing of FCP command is deferred */ - if (rc == -EOVERFLOW) { - lpfc_nvmeio_data(phba, - "NVMET RCV BUSY: xri x%x sz %d " - "from %06x\n", - oxid, size, sid); - atomic_inc(&tgtp->rcv_fcp_cmd_out); - return; + if (!queue_work(phba->wq, &ctx_buf->defer_work)) { + atomic_inc(&tgtp->rcv_fcp_cmd_drop); + lpfc_printf_log(phba, KERN_ERR, LOG_NVME, + "6181 Unable to queue deferred work " + "for oxid x%x. " + "FCP Drop IO [x%x x%x x%x]\n", + ctxp->oxid, + atomic_read(&tgtp->rcv_fcp_cmd_in), + atomic_read(&tgtp->rcv_fcp_cmd_out), + atomic_read(&tgtp->xmt_fcp_release)); + + spin_lock_irqsave(&ctxp->ctxlock, iflag); + lpfc_nvmet_defer_release(phba, ctxp); + spin_unlock_irqrestore(&ctxp->ctxlock, iflag); + lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, sid, oxid); } - atomic_inc(&tgtp->rcv_fcp_cmd_drop); - lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, - "2582 FCP Drop IO x%x: err x%x: x%x x%x x%x\n", - ctxp->oxid, rc, - atomic_read(&tgtp->rcv_fcp_cmd_in), - atomic_read(&tgtp->rcv_fcp_cmd_out), - atomic_read(&tgtp->xmt_fcp_release)); - - lpfc_nvmet_defer_release(phba, ctxp); - lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, sid, oxid); - nvmebuf->hrq->rqbp->rqb_free_buffer(phba, nvmebuf); return; } spin_unlock_irqrestore(&phba->sli4_hba.nvmet_io_wait_lock, iflag); @@ -744,16 +743,6 @@ lpfc_nvmet_xmt_fcp_op_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, ktime_get_ns(); } } - if (phba->cpucheck_on & LPFC_CHECK_NVMET_IO) { - id = smp_processor_id(); - if (ctxp->cpu != id) - lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, - "6703 CPU Check cmpl: " - "cpu %d expect %d\n", - id, ctxp->cpu); - if (ctxp->cpu < LPFC_CHECK_CPU_CNT) - phba->cpucheck_cmpl_io[id]++; - } #endif rsp->done(rsp); #ifdef CONFIG_SCSI_LPFC_DEBUG_FS @@ -771,19 +760,22 @@ lpfc_nvmet_xmt_fcp_op_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, ctxp->ts_isr_data = cmdwqe->isr_timestamp; ctxp->ts_data_nvme = ktime_get_ns(); } - if (phba->cpucheck_on & LPFC_CHECK_NVMET_IO) { - id = smp_processor_id(); +#endif + rsp->done(rsp); + } +#ifdef CONFIG_SCSI_LPFC_DEBUG_FS + if (phba->cpucheck_on & LPFC_CHECK_NVMET_IO) { + id = smp_processor_id(); + if (id < LPFC_CHECK_CPU_CNT) { if (ctxp->cpu != id) - lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, + lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR, "6704 CPU Check cmdcmpl: " "cpu %d expect %d\n", id, ctxp->cpu); - if (ctxp->cpu < LPFC_CHECK_CPU_CNT) - phba->cpucheck_ccmpl_io[id]++; + phba->sli4_hba.hdwq[rsp->hwqid].cpucheck_cmpl_io[id]++; } -#endif - rsp->done(rsp); } +#endif } static int @@ -852,7 +844,7 @@ lpfc_nvmet_xmt_ls_rsp(struct nvmet_fc_target_port *tgtport, lpfc_nvmeio_data(phba, "NVMET LS RESP: xri x%x wqidx x%x len x%x\n", ctxp->oxid, nvmewqeq->hba_wqidx, rsp->rsplen); - rc = lpfc_sli4_issue_wqe(phba, LPFC_ELS_RING, nvmewqeq); + rc = lpfc_sli4_issue_wqe(phba, ctxp->hdwq, nvmewqeq); if (rc == WQE_SUCCESS) { /* * Okay to repost buffer here, but wait till cmpl @@ -908,18 +900,22 @@ lpfc_nvmet_xmt_fcp_op(struct nvmet_fc_target_port *tgtport, else ctxp->ts_nvme_data = ktime_get_ns(); } + + /* Setup the hdw queue if not already set */ + if (!ctxp->hdwq) + ctxp->hdwq = &phba->sli4_hba.hdwq[rsp->hwqid]; + if (phba->cpucheck_on & LPFC_CHECK_NVMET_IO) { int id = smp_processor_id(); - ctxp->cpu = id; - if (id < LPFC_CHECK_CPU_CNT) - phba->cpucheck_xmt_io[id]++; - if (rsp->hwqid != id) { - lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, - "6705 CPU Check OP: " - "cpu %d expect %d\n", - id, rsp->hwqid); - ctxp->cpu = rsp->hwqid; + if (id < LPFC_CHECK_CPU_CNT) { + if (rsp->hwqid != id) + lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR, + "6705 CPU Check OP: " + "cpu %d expect %d\n", + id, rsp->hwqid); + phba->sli4_hba.hdwq[rsp->hwqid].cpucheck_xmt_io[id]++; } + ctxp->cpu = id; /* Setup cpu for cmpl check */ } #endif @@ -954,7 +950,7 @@ lpfc_nvmet_xmt_fcp_op(struct nvmet_fc_target_port *tgtport, ctxp->oxid, rsp->op, rsp->rsplen); ctxp->flag |= LPFC_NVMET_IO_INP; - rc = lpfc_sli4_issue_wqe(phba, LPFC_FCP_RING, nvmewqeq); + rc = lpfc_sli4_issue_wqe(phba, ctxp->hdwq, nvmewqeq); if (rc == WQE_SUCCESS) { #ifdef CONFIG_SCSI_LPFC_DEBUG_FS if (!ctxp->ts_cmd_nvme) @@ -973,7 +969,7 @@ lpfc_nvmet_xmt_fcp_op(struct nvmet_fc_target_port *tgtport, * WQE release CQE */ ctxp->flag |= LPFC_NVMET_DEFER_WQFULL; - wq = phba->sli4_hba.nvme_wq[rsp->hwqid]; + wq = ctxp->hdwq->nvme_wq; pring = wq->pring; spin_lock_irqsave(&pring->ring_lock, iflags); list_add_tail(&nvmewqeq->list, &wq->wqfull_list); @@ -1024,6 +1020,9 @@ lpfc_nvmet_xmt_fcp_abort(struct nvmet_fc_target_port *tgtport, if (phba->pport->load_flag & FC_UNLOADING) return; + if (!ctxp->hdwq) + ctxp->hdwq = &phba->sli4_hba.hdwq[0]; + lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, "6103 NVMET Abort op: oxri x%x flg x%x ste %d\n", ctxp->oxid, ctxp->flag, ctxp->state); @@ -1034,7 +1033,6 @@ lpfc_nvmet_xmt_fcp_abort(struct nvmet_fc_target_port *tgtport, atomic_inc(&lpfc_nvmep->xmt_fcp_abort); spin_lock_irqsave(&ctxp->ctxlock, flags); - ctxp->state = LPFC_NVMET_STE_ABORT; /* Since iaab/iaar are NOT set, we need to check * if the firmware is in process of aborting IO @@ -1046,13 +1044,14 @@ lpfc_nvmet_xmt_fcp_abort(struct nvmet_fc_target_port *tgtport, ctxp->flag |= LPFC_NVMET_ABORT_OP; if (ctxp->flag & LPFC_NVMET_DEFER_WQFULL) { + spin_unlock_irqrestore(&ctxp->ctxlock, flags); lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, ctxp->sid, ctxp->oxid); - wq = phba->sli4_hba.nvme_wq[ctxp->wqeq->hba_wqidx]; - spin_unlock_irqrestore(&ctxp->ctxlock, flags); + wq = ctxp->hdwq->nvme_wq; lpfc_nvmet_wqfull_flush(phba, wq, ctxp); return; } + spin_unlock_irqrestore(&ctxp->ctxlock, flags); /* An state of LPFC_NVMET_STE_RCV means we have just received * the NVME command and have not started processing it. @@ -1064,7 +1063,6 @@ lpfc_nvmet_xmt_fcp_abort(struct nvmet_fc_target_port *tgtport, else lpfc_nvmet_sol_fcp_issue_abort(phba, ctxp, ctxp->sid, ctxp->oxid); - spin_unlock_irqrestore(&ctxp->ctxlock, flags); } static void @@ -1078,14 +1076,18 @@ lpfc_nvmet_xmt_fcp_release(struct nvmet_fc_target_port *tgtport, unsigned long flags; bool aborting = false; - if (ctxp->state != LPFC_NVMET_STE_DONE && - ctxp->state != LPFC_NVMET_STE_ABORT) { + spin_lock_irqsave(&ctxp->ctxlock, flags); + if (ctxp->flag & LPFC_NVMET_XBUSY) + lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR, + "6027 NVMET release with XBUSY flag x%x" + " oxid x%x\n", + ctxp->flag, ctxp->oxid); + else if (ctxp->state != LPFC_NVMET_STE_DONE && + ctxp->state != LPFC_NVMET_STE_ABORT) lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, "6413 NVMET release bad state %d %d oxid x%x\n", ctxp->state, ctxp->entry_cnt, ctxp->oxid); - } - spin_lock_irqsave(&ctxp->ctxlock, flags); if ((ctxp->flag & LPFC_NVMET_ABORT_OP) || (ctxp->flag & LPFC_NVMET_XBUSY)) { aborting = true; @@ -1114,6 +1116,8 @@ lpfc_nvmet_defer_rcv(struct nvmet_fc_target_port *tgtport, container_of(rsp, struct lpfc_nvmet_rcv_ctx, ctx.fcp_req); struct rqb_dmabuf *nvmebuf = ctxp->rqb_buffer; struct lpfc_hba *phba = ctxp->phba; + unsigned long iflag; + lpfc_nvmeio_data(phba, "NVMET DEFERRCV: xri x%x sz %d CPU %02x\n", ctxp->oxid, ctxp->size, smp_processor_id()); @@ -1132,6 +1136,9 @@ lpfc_nvmet_defer_rcv(struct nvmet_fc_target_port *tgtport, /* Free the nvmebuf since a new buffer already replaced it */ nvmebuf->hrq->rqbp->rqb_free_buffer(phba, nvmebuf); + spin_lock_irqsave(&ctxp->ctxlock, iflag); + ctxp->rqb_buffer = NULL; + spin_unlock_irqrestore(&ctxp->ctxlock, iflag); } static struct nvmet_fc_target_template lpfc_tgttemplate = { @@ -1163,9 +1170,9 @@ __lpfc_nvmet_clean_io_for_cpu(struct lpfc_hba *phba, spin_lock_irqsave(&infop->nvmet_ctx_list_lock, flags); list_for_each_entry_safe(ctx_buf, next_ctx_buf, &infop->nvmet_ctx_list, list) { - spin_lock(&phba->sli4_hba.abts_nvme_buf_list_lock); + spin_lock(&phba->sli4_hba.abts_nvmet_buf_list_lock); list_del_init(&ctx_buf->list); - spin_unlock(&phba->sli4_hba.abts_nvme_buf_list_lock); + spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock); __lpfc_clear_active_sglq(phba, ctx_buf->sglq->sli4_lxritag); ctx_buf->sglq->state = SGL_FREED; @@ -1195,9 +1202,9 @@ lpfc_nvmet_cleanup_io_context(struct lpfc_hba *phba) /* Cycle the the entire CPU context list for every MRQ */ for (i = 0; i < phba->cfg_nvmet_mrq; i++) { - for (j = 0; j < phba->sli4_hba.num_present_cpu; j++) { + for_each_present_cpu(j) { + infop = lpfc_get_ctx_list(phba, j, i); __lpfc_nvmet_clean_io_for_cpu(phba, infop); - infop++; /* next */ } } kfree(phba->sli4_hba.nvmet_ctx_info); @@ -1212,14 +1219,14 @@ lpfc_nvmet_setup_io_context(struct lpfc_hba *phba) union lpfc_wqe128 *wqe; struct lpfc_nvmet_ctx_info *last_infop; struct lpfc_nvmet_ctx_info *infop; - int i, j, idx; + int i, j, idx, cpu; lpfc_printf_log(phba, KERN_INFO, LOG_NVME, "6403 Allocate NVMET resources for %d XRIs\n", phba->sli4_hba.nvmet_xri_cnt); phba->sli4_hba.nvmet_ctx_info = kcalloc( - phba->sli4_hba.num_present_cpu * phba->cfg_nvmet_mrq, + phba->sli4_hba.num_possible_cpu * phba->cfg_nvmet_mrq, sizeof(struct lpfc_nvmet_ctx_info), GFP_KERNEL); if (!phba->sli4_hba.nvmet_ctx_info) { lpfc_printf_log(phba, KERN_ERR, LOG_INIT, @@ -1247,13 +1254,12 @@ lpfc_nvmet_setup_io_context(struct lpfc_hba *phba) * of the IO completion. Thus a context that was allocated for MRQ A * whose IO completed on CPU B will be freed to cpuB/mrqA. */ - infop = phba->sli4_hba.nvmet_ctx_info; - for (i = 0; i < phba->sli4_hba.num_present_cpu; i++) { + for_each_possible_cpu(i) { for (j = 0; j < phba->cfg_nvmet_mrq; j++) { + infop = lpfc_get_ctx_list(phba, i, j); INIT_LIST_HEAD(&infop->nvmet_ctx_list); spin_lock_init(&infop->nvmet_ctx_list_lock); infop->nvmet_ctx_list_cnt = 0; - infop++; } } @@ -1263,8 +1269,10 @@ lpfc_nvmet_setup_io_context(struct lpfc_hba *phba) * MRQ 1 cycling thru CPUs 0 - X, and so on. */ for (j = 0; j < phba->cfg_nvmet_mrq; j++) { - last_infop = lpfc_get_ctx_list(phba, 0, j); - for (i = phba->sli4_hba.num_present_cpu - 1; i >= 0; i--) { + last_infop = lpfc_get_ctx_list(phba, + cpumask_first(cpu_present_mask), + j); + for (i = phba->sli4_hba.num_possible_cpu - 1; i >= 0; i--) { infop = lpfc_get_ctx_list(phba, i, j); infop->nvmet_ctx_next_cpu = last_infop; last_infop = infop; @@ -1275,6 +1283,7 @@ lpfc_nvmet_setup_io_context(struct lpfc_hba *phba) * received command on a per xri basis. */ idx = 0; + cpu = cpumask_first(cpu_present_mask); for (i = 0; i < phba->sli4_hba.nvmet_xri_cnt; i++) { ctx_buf = kzalloc(sizeof(*ctx_buf), GFP_KERNEL); if (!ctx_buf) { @@ -1322,13 +1331,14 @@ lpfc_nvmet_setup_io_context(struct lpfc_hba *phba) "6407 Ran out of NVMET XRIs\n"); return -ENOMEM; } + INIT_WORK(&ctx_buf->defer_work, lpfc_nvmet_fcp_rqst_defer_work); /* * Add ctx to MRQidx context list. Our initial assumption * is MRQidx will be associated with CPUidx. This association * can change on the fly. */ - infop = lpfc_get_ctx_list(phba, idx, idx); + infop = lpfc_get_ctx_list(phba, cpu, idx); spin_lock(&infop->nvmet_ctx_list_lock); list_add_tail(&ctx_buf->list, &infop->nvmet_ctx_list); infop->nvmet_ctx_list_cnt++; @@ -1336,11 +1346,18 @@ lpfc_nvmet_setup_io_context(struct lpfc_hba *phba) /* Spread ctx structures evenly across all MRQs */ idx++; - if (idx >= phba->cfg_nvmet_mrq) + if (idx >= phba->cfg_nvmet_mrq) { idx = 0; + cpu = cpumask_first(cpu_present_mask); + continue; + } + cpu = cpumask_next(cpu, cpu_present_mask); + if (cpu == nr_cpu_ids) + cpu = cpumask_first(cpu_present_mask); + } - for (i = 0; i < phba->sli4_hba.num_present_cpu; i++) { + for_each_present_cpu(i) { for (j = 0; j < phba->cfg_nvmet_mrq; j++) { infop = lpfc_get_ctx_list(phba, i, j); lpfc_printf_log(phba, KERN_INFO, LOG_NVME | LOG_INIT, @@ -1378,7 +1395,7 @@ lpfc_nvmet_create_targetport(struct lpfc_hba *phba) * allocate + 3, one for cmd, one for rsp and one for this alignment */ lpfc_tgttemplate.max_sgl_segments = phba->cfg_nvme_seg_cnt + 1; - lpfc_tgttemplate.max_hw_queues = phba->cfg_nvme_io_channel; + lpfc_tgttemplate.max_hw_queues = phba->cfg_hdw_queue; lpfc_tgttemplate.target_features = NVMET_FCTGTFEAT_READDATA_RSP; #if (IS_ENABLED(CONFIG_NVME_TARGET_FC)) @@ -1503,13 +1520,14 @@ lpfc_sli4_nvmet_xri_aborted(struct lpfc_hba *phba, } spin_lock_irqsave(&phba->hbalock, iflag); - spin_lock(&phba->sli4_hba.abts_nvme_buf_list_lock); + spin_lock(&phba->sli4_hba.abts_nvmet_buf_list_lock); list_for_each_entry_safe(ctxp, next_ctxp, &phba->sli4_hba.lpfc_abts_nvmet_ctx_list, list) { if (ctxp->ctxbuf->sglq->sli4_xritag != xri) continue; + spin_lock(&ctxp->ctxlock); /* Check if we already received a free context call * and we have completed processing an abort situation. */ @@ -1519,7 +1537,8 @@ lpfc_sli4_nvmet_xri_aborted(struct lpfc_hba *phba, released = true; } ctxp->flag &= ~LPFC_NVMET_XBUSY; - spin_unlock(&phba->sli4_hba.abts_nvme_buf_list_lock); + spin_unlock(&ctxp->ctxlock); + spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock); rrq_empty = list_empty(&phba->active_rrq_list); spin_unlock_irqrestore(&phba->hbalock, iflag); @@ -1543,14 +1562,13 @@ lpfc_sli4_nvmet_xri_aborted(struct lpfc_hba *phba, lpfc_worker_wake_up(phba); return; } - spin_unlock(&phba->sli4_hba.abts_nvme_buf_list_lock); + spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock); spin_unlock_irqrestore(&phba->hbalock, iflag); } int lpfc_nvmet_rcv_unsol_abort(struct lpfc_vport *vport, struct fc_frame_header *fc_hdr) - { #if (IS_ENABLED(CONFIG_NVME_TARGET_FC)) struct lpfc_hba *phba = vport->phba; @@ -1562,14 +1580,14 @@ lpfc_nvmet_rcv_unsol_abort(struct lpfc_vport *vport, xri = be16_to_cpu(fc_hdr->fh_ox_id); spin_lock_irqsave(&phba->hbalock, iflag); - spin_lock(&phba->sli4_hba.abts_nvme_buf_list_lock); + spin_lock(&phba->sli4_hba.abts_nvmet_buf_list_lock); list_for_each_entry_safe(ctxp, next_ctxp, &phba->sli4_hba.lpfc_abts_nvmet_ctx_list, list) { if (ctxp->ctxbuf->sglq->sli4_xritag != xri) continue; - spin_unlock(&phba->sli4_hba.abts_nvme_buf_list_lock); + spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock); spin_unlock_irqrestore(&phba->hbalock, iflag); spin_lock_irqsave(&ctxp->ctxlock, iflag); @@ -1590,7 +1608,7 @@ lpfc_nvmet_rcv_unsol_abort(struct lpfc_vport *vport, lpfc_sli4_seq_abort_rsp(vport, fc_hdr, 1); return 0; } - spin_unlock(&phba->sli4_hba.abts_nvme_buf_list_lock); + spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock); spin_unlock_irqrestore(&phba->hbalock, iflag); lpfc_nvmeio_data(phba, "NVMET ABTS RCV: xri x%x CPU %02x rjt %d\n", @@ -1658,6 +1676,7 @@ lpfc_nvmet_wqfull_process(struct lpfc_hba *phba, #if (IS_ENABLED(CONFIG_NVME_TARGET_FC)) struct lpfc_sli_ring *pring; struct lpfc_iocbq *nvmewqeq; + struct lpfc_nvmet_rcv_ctx *ctxp; unsigned long iflags; int rc; @@ -1671,7 +1690,8 @@ lpfc_nvmet_wqfull_process(struct lpfc_hba *phba, list_remove_head(&wq->wqfull_list, nvmewqeq, struct lpfc_iocbq, list); spin_unlock_irqrestore(&pring->ring_lock, iflags); - rc = lpfc_sli4_issue_wqe(phba, LPFC_FCP_RING, nvmewqeq); + ctxp = (struct lpfc_nvmet_rcv_ctx *)nvmewqeq->context2; + rc = lpfc_sli4_issue_wqe(phba, ctxp->hdwq, nvmewqeq); spin_lock_irqsave(&pring->ring_lock, iflags); if (rc == -EBUSY) { /* WQ was full again, so put it back on the list */ @@ -1699,8 +1719,8 @@ lpfc_nvmet_destroy_targetport(struct lpfc_hba *phba) return; if (phba->targetport) { tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; - for (qidx = 0; qidx < phba->cfg_nvme_io_channel; qidx++) { - wq = phba->sli4_hba.nvme_wq[qidx]; + for (qidx = 0; qidx < phba->cfg_hdw_queue; qidx++) { + wq = phba->sli4_hba.hdwq[qidx].nvme_wq; lpfc_nvmet_wqfull_flush(phba, wq, NULL); } tgtp->tport_unreg_cmp = &tport_unreg_cmp; @@ -1775,6 +1795,7 @@ dropit: ctxp->state = LPFC_NVMET_STE_LS_RCV; ctxp->entry_cnt = 1; ctxp->rqb_buffer = (void *)nvmebuf; + ctxp->hdwq = &phba->sli4_hba.hdwq[0]; lpfc_nvmeio_data(phba, "NVMET LS RCV: xri x%x sz %d from %06x\n", oxid, size, sid); @@ -1814,6 +1835,86 @@ dropit: #endif } +static void +lpfc_nvmet_process_rcv_fcp_req(struct lpfc_nvmet_ctxbuf *ctx_buf) +{ +#if (IS_ENABLED(CONFIG_NVME_TARGET_FC)) + struct lpfc_nvmet_rcv_ctx *ctxp = ctx_buf->context; + struct lpfc_hba *phba = ctxp->phba; + struct rqb_dmabuf *nvmebuf = ctxp->rqb_buffer; + struct lpfc_nvmet_tgtport *tgtp; + uint32_t *payload; + uint32_t rc; + unsigned long iflags; + + if (!nvmebuf) { + lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, + "6159 process_rcv_fcp_req, nvmebuf is NULL, " + "oxid: x%x flg: x%x state: x%x\n", + ctxp->oxid, ctxp->flag, ctxp->state); + spin_lock_irqsave(&ctxp->ctxlock, iflags); + lpfc_nvmet_defer_release(phba, ctxp); + spin_unlock_irqrestore(&ctxp->ctxlock, iflags); + lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, ctxp->sid, + ctxp->oxid); + return; + } + + payload = (uint32_t *)(nvmebuf->dbuf.virt); + tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; + /* + * The calling sequence should be: + * nvmet_fc_rcv_fcp_req->lpfc_nvmet_xmt_fcp_op/cmp- req->done + * lpfc_nvmet_xmt_fcp_op_cmp should free the allocated ctxp. + * When we return from nvmet_fc_rcv_fcp_req, all relevant info + * the NVME command / FC header is stored. + * A buffer has already been reposted for this IO, so just free + * the nvmebuf. + */ + rc = nvmet_fc_rcv_fcp_req(phba->targetport, &ctxp->ctx.fcp_req, + payload, ctxp->size); + /* Process FCP command */ + if (rc == 0) { + atomic_inc(&tgtp->rcv_fcp_cmd_out); + return; + } + + /* Processing of FCP command is deferred */ + if (rc == -EOVERFLOW) { + lpfc_nvmeio_data(phba, "NVMET RCV BUSY: xri x%x sz %d " + "from %06x\n", + ctxp->oxid, ctxp->size, ctxp->sid); + atomic_inc(&tgtp->rcv_fcp_cmd_out); + atomic_inc(&tgtp->defer_fod); + return; + } + atomic_inc(&tgtp->rcv_fcp_cmd_drop); + lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, + "2582 FCP Drop IO x%x: err x%x: x%x x%x x%x\n", + ctxp->oxid, rc, + atomic_read(&tgtp->rcv_fcp_cmd_in), + atomic_read(&tgtp->rcv_fcp_cmd_out), + atomic_read(&tgtp->xmt_fcp_release)); + lpfc_nvmeio_data(phba, "NVMET FCP DROP: xri x%x sz %d from %06x\n", + ctxp->oxid, ctxp->size, ctxp->sid); + spin_lock_irqsave(&ctxp->ctxlock, iflags); + lpfc_nvmet_defer_release(phba, ctxp); + spin_unlock_irqrestore(&ctxp->ctxlock, iflags); + lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, ctxp->sid, ctxp->oxid); +#endif +} + +static void +lpfc_nvmet_fcp_rqst_defer_work(struct work_struct *work) +{ +#if (IS_ENABLED(CONFIG_NVME_TARGET_FC)) + struct lpfc_nvmet_ctxbuf *ctx_buf = + container_of(work, struct lpfc_nvmet_ctxbuf, defer_work); + + lpfc_nvmet_process_rcv_fcp_req(ctx_buf); +#endif +} + static struct lpfc_nvmet_ctxbuf * lpfc_nvmet_replenish_context(struct lpfc_hba *phba, struct lpfc_nvmet_ctx_info *current_infop) @@ -1838,7 +1939,7 @@ lpfc_nvmet_replenish_context(struct lpfc_hba *phba, else get_infop = current_infop->nvmet_ctx_next_cpu; - for (i = 0; i < phba->sli4_hba.num_present_cpu; i++) { + for (i = 0; i < phba->sli4_hba.num_possible_cpu; i++) { if (get_infop == current_infop) { get_infop = get_infop->nvmet_ctx_next_cpu; continue; @@ -1896,12 +1997,9 @@ lpfc_nvmet_unsol_fcp_buffer(struct lpfc_hba *phba, struct lpfc_nvmet_ctxbuf *ctx_buf; struct lpfc_nvmet_ctx_info *current_infop; uint32_t *payload; - uint32_t size, oxid, sid, rc, qno; + uint32_t size, oxid, sid, qno; unsigned long iflag; int current_cpu; -#ifdef CONFIG_SCSI_LPFC_DEBUG_FS - uint32_t id; -#endif if (!IS_ENABLED(CONFIG_NVME_TARGET_FC)) return; @@ -1910,11 +2008,9 @@ lpfc_nvmet_unsol_fcp_buffer(struct lpfc_hba *phba, if (!nvmebuf || !phba->targetport) { lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, "6157 NVMET FCP Drop IO\n"); - oxid = 0; - size = 0; - sid = 0; - ctxp = NULL; - goto dropit; + if (nvmebuf) + lpfc_rq_buf_free(phba, &nvmebuf->hbuf); + return; } /* @@ -1942,9 +2038,14 @@ lpfc_nvmet_unsol_fcp_buffer(struct lpfc_hba *phba, #ifdef CONFIG_SCSI_LPFC_DEBUG_FS if (phba->cpucheck_on & LPFC_CHECK_NVMET_RCV) { - id = smp_processor_id(); - if (id < LPFC_CHECK_CPU_CNT) - phba->cpucheck_rcv_io[id]++; + if (current_cpu < LPFC_CHECK_CPU_CNT) { + if (idx != current_cpu) + lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR, + "6703 CPU Check rcv: " + "cpu %d expect %d\n", + current_cpu, idx); + phba->sli4_hba.hdwq[idx].cpucheck_rcv_io[current_cpu]++; + } } #endif @@ -1995,6 +2096,7 @@ lpfc_nvmet_unsol_fcp_buffer(struct lpfc_hba *phba, ctxp->flag = 0; ctxp->ctxbuf = ctx_buf; ctxp->rqb_buffer = (void *)nvmebuf; + ctxp->hdwq = NULL; spin_lock_init(&ctxp->ctxlock); #ifdef CONFIG_SCSI_LPFC_DEBUG_FS @@ -2015,67 +2117,7 @@ lpfc_nvmet_unsol_fcp_buffer(struct lpfc_hba *phba, #endif atomic_inc(&tgtp->rcv_fcp_cmd_in); - /* - * The calling sequence should be: - * nvmet_fc_rcv_fcp_req -> lpfc_nvmet_xmt_fcp_op/cmp -> req->done - * lpfc_nvmet_xmt_fcp_op_cmp should free the allocated ctxp. - * When we return from nvmet_fc_rcv_fcp_req, all relevant info in - * the NVME command / FC header is stored, so we are free to repost - * the buffer. - */ - rc = nvmet_fc_rcv_fcp_req(phba->targetport, &ctxp->ctx.fcp_req, - payload, size); - - /* Process FCP command */ - if (rc == 0) { - ctxp->rqb_buffer = NULL; - atomic_inc(&tgtp->rcv_fcp_cmd_out); - lpfc_rq_buf_free(phba, &nvmebuf->hbuf); /* repost */ - return; - } - - /* Processing of FCP command is deferred */ - if (rc == -EOVERFLOW) { - /* - * Post a brand new DMA buffer to RQ and defer - * freeing rcv buffer till .defer_rcv callback - */ - qno = nvmebuf->idx; - lpfc_post_rq_buffer( - phba, phba->sli4_hba.nvmet_mrq_hdr[qno], - phba->sli4_hba.nvmet_mrq_data[qno], 1, qno); - - lpfc_nvmeio_data(phba, - "NVMET RCV BUSY: xri x%x sz %d from %06x\n", - oxid, size, sid); - atomic_inc(&tgtp->rcv_fcp_cmd_out); - atomic_inc(&tgtp->defer_fod); - return; - } - ctxp->rqb_buffer = nvmebuf; - - atomic_inc(&tgtp->rcv_fcp_cmd_drop); - lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, - "6159 FCP Drop IO x%x: err x%x: x%x x%x x%x\n", - ctxp->oxid, rc, - atomic_read(&tgtp->rcv_fcp_cmd_in), - atomic_read(&tgtp->rcv_fcp_cmd_out), - atomic_read(&tgtp->xmt_fcp_release)); -dropit: - lpfc_nvmeio_data(phba, "NVMET FCP DROP: xri x%x sz %d from %06x\n", - oxid, size, sid); - if (oxid) { - lpfc_nvmet_defer_release(phba, ctxp); - lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, sid, oxid); - lpfc_rq_buf_free(phba, &nvmebuf->hbuf); /* repost */ - return; - } - - if (ctx_buf) - lpfc_nvmet_ctxbuf_post(phba, ctx_buf); - - if (nvmebuf) - lpfc_rq_buf_free(phba, &nvmebuf->hbuf); /* repost */ + lpfc_nvmet_process_rcv_fcp_req(ctx_buf); } /** @@ -2660,15 +2702,17 @@ lpfc_nvmet_sol_fcp_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, if (ctxp->flag & LPFC_NVMET_ABORT_OP) atomic_inc(&tgtp->xmt_fcp_abort_cmpl); + spin_lock_irqsave(&ctxp->ctxlock, flags); ctxp->state = LPFC_NVMET_STE_DONE; /* Check if we already received a free context call * and we have completed processing an abort situation. */ - spin_lock_irqsave(&ctxp->ctxlock, flags); if ((ctxp->flag & LPFC_NVMET_CTX_RLS) && !(ctxp->flag & LPFC_NVMET_XBUSY)) { + spin_lock(&phba->sli4_hba.abts_nvmet_buf_list_lock); list_del(&ctxp->list); + spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock); released = true; } ctxp->flag &= ~LPFC_NVMET_ABORT_OP; @@ -2734,6 +2778,7 @@ lpfc_nvmet_unsol_fcp_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, } tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; + spin_lock_irqsave(&ctxp->ctxlock, flags); if (ctxp->flag & LPFC_NVMET_ABORT_OP) atomic_inc(&tgtp->xmt_fcp_abort_cmpl); @@ -2748,10 +2793,11 @@ lpfc_nvmet_unsol_fcp_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, * and we have completed processing an abort situation. */ ctxp->state = LPFC_NVMET_STE_DONE; - spin_lock_irqsave(&ctxp->ctxlock, flags); if ((ctxp->flag & LPFC_NVMET_CTX_RLS) && !(ctxp->flag & LPFC_NVMET_XBUSY)) { + spin_lock(&phba->sli4_hba.abts_nvmet_buf_list_lock); list_del(&ctxp->list); + spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock); released = true; } ctxp->flag &= ~LPFC_NVMET_ABORT_OP; @@ -2957,12 +3003,15 @@ lpfc_nvmet_sol_fcp_issue_abort(struct lpfc_hba *phba, (ndlp) ? ndlp->nlp_state : NLP_STE_MAX_STATE); /* No failure to an ABTS request. */ + spin_lock_irqsave(&ctxp->ctxlock, flags); ctxp->flag &= ~LPFC_NVMET_ABORT_OP; + spin_unlock_irqrestore(&ctxp->ctxlock, flags); return 0; } /* Issue ABTS for this WQE based on iotag */ ctxp->abort_wqeq = lpfc_sli_get_iocbq(phba); + spin_lock_irqsave(&ctxp->ctxlock, flags); if (!ctxp->abort_wqeq) { atomic_inc(&tgtp->xmt_abort_rsp_error); lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS, @@ -2970,11 +3019,13 @@ lpfc_nvmet_sol_fcp_issue_abort(struct lpfc_hba *phba, "xri: x%x\n", ctxp->oxid); /* No failure to an ABTS request. */ ctxp->flag &= ~LPFC_NVMET_ABORT_OP; + spin_unlock_irqrestore(&ctxp->ctxlock, flags); return 0; } abts_wqeq = ctxp->abort_wqeq; abts_wqe = &abts_wqeq->wqe; ctxp->state = LPFC_NVMET_STE_ABORT; + spin_unlock_irqrestore(&ctxp->ctxlock, flags); /* Announce entry to new IO submit field. */ lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, @@ -2995,7 +3046,9 @@ lpfc_nvmet_sol_fcp_issue_abort(struct lpfc_hba *phba, "NVME Req now. hba_flag x%x oxid x%x\n", phba->hba_flag, ctxp->oxid); lpfc_sli_release_iocbq(phba, abts_wqeq); + spin_lock_irqsave(&ctxp->ctxlock, flags); ctxp->flag &= ~LPFC_NVMET_ABORT_OP; + spin_unlock_irqrestore(&ctxp->ctxlock, flags); return 0; } @@ -3008,7 +3061,9 @@ lpfc_nvmet_sol_fcp_issue_abort(struct lpfc_hba *phba, "still pending on oxid x%x\n", ctxp->oxid); lpfc_sli_release_iocbq(phba, abts_wqeq); + spin_lock_irqsave(&ctxp->ctxlock, flags); ctxp->flag &= ~LPFC_NVMET_ABORT_OP; + spin_unlock_irqrestore(&ctxp->ctxlock, flags); return 0; } @@ -3052,7 +3107,10 @@ lpfc_nvmet_sol_fcp_issue_abort(struct lpfc_hba *phba, abts_wqeq->iocb_flag |= LPFC_IO_NVME; abts_wqeq->context2 = ctxp; abts_wqeq->vport = phba->pport; - rc = lpfc_sli4_issue_wqe(phba, LPFC_FCP_RING, abts_wqeq); + if (!ctxp->hdwq) + ctxp->hdwq = &phba->sli4_hba.hdwq[abts_wqeq->hba_wqidx]; + + rc = lpfc_sli4_issue_wqe(phba, ctxp->hdwq, abts_wqeq); spin_unlock_irqrestore(&phba->hbalock, flags); if (rc == WQE_SUCCESS) { atomic_inc(&tgtp->xmt_abort_sol); @@ -3060,7 +3118,9 @@ lpfc_nvmet_sol_fcp_issue_abort(struct lpfc_hba *phba, } atomic_inc(&tgtp->xmt_abort_rsp_error); + spin_lock_irqsave(&ctxp->ctxlock, flags); ctxp->flag &= ~LPFC_NVMET_ABORT_OP; + spin_unlock_irqrestore(&ctxp->ctxlock, flags); lpfc_sli_release_iocbq(phba, abts_wqeq); lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS, "6166 Failed ABORT issue_wqe with status x%x " @@ -3069,7 +3129,6 @@ lpfc_nvmet_sol_fcp_issue_abort(struct lpfc_hba *phba, return 1; } - static int lpfc_nvmet_unsol_fcp_issue_abort(struct lpfc_hba *phba, struct lpfc_nvmet_rcv_ctx *ctxp, @@ -3078,6 +3137,7 @@ lpfc_nvmet_unsol_fcp_issue_abort(struct lpfc_hba *phba, struct lpfc_nvmet_tgtport *tgtp; struct lpfc_iocbq *abts_wqeq; unsigned long flags; + bool released = false; int rc; tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; @@ -3104,7 +3164,10 @@ lpfc_nvmet_unsol_fcp_issue_abort(struct lpfc_hba *phba, abts_wqeq->wqe_cmpl = lpfc_nvmet_unsol_fcp_abort_cmp; abts_wqeq->iocb_cmpl = NULL; abts_wqeq->iocb_flag |= LPFC_IO_NVMET; - rc = lpfc_sli4_issue_wqe(phba, LPFC_FCP_RING, abts_wqeq); + if (!ctxp->hdwq) + ctxp->hdwq = &phba->sli4_hba.hdwq[abts_wqeq->hba_wqidx]; + + rc = lpfc_sli4_issue_wqe(phba, ctxp->hdwq, abts_wqeq); spin_unlock_irqrestore(&phba->hbalock, flags); if (rc == WQE_SUCCESS) { return 0; @@ -3112,8 +3175,12 @@ lpfc_nvmet_unsol_fcp_issue_abort(struct lpfc_hba *phba, aerr: spin_lock_irqsave(&ctxp->ctxlock, flags); - if (ctxp->flag & LPFC_NVMET_CTX_RLS) + if (ctxp->flag & LPFC_NVMET_CTX_RLS) { + spin_lock(&phba->sli4_hba.abts_nvmet_buf_list_lock); list_del(&ctxp->list); + spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock); + released = true; + } ctxp->flag &= ~(LPFC_NVMET_ABORT_OP | LPFC_NVMET_CTX_RLS); spin_unlock_irqrestore(&ctxp->ctxlock, flags); @@ -3121,7 +3188,8 @@ aerr: lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS, "6135 Failed to Issue ABTS for oxid x%x. Status x%x\n", ctxp->oxid, rc); - lpfc_nvmet_ctxbuf_post(phba, ctxp->ctxbuf); + if (released) + lpfc_nvmet_ctxbuf_post(phba, ctxp->ctxbuf); return 1; } @@ -3173,7 +3241,7 @@ lpfc_nvmet_unsol_ls_issue_abort(struct lpfc_hba *phba, abts_wqeq->wqe_cmpl = lpfc_nvmet_xmt_ls_abort_cmp; abts_wqeq->iocb_cmpl = 0; abts_wqeq->iocb_flag |= LPFC_IO_NVME_LS; - rc = lpfc_sli4_issue_wqe(phba, LPFC_ELS_RING, abts_wqeq); + rc = lpfc_sli4_issue_wqe(phba, ctxp->hdwq, abts_wqeq); spin_unlock_irqrestore(&phba->hbalock, flags); if (rc == WQE_SUCCESS) { atomic_inc(&tgtp->xmt_abort_unsol); diff --git a/drivers/scsi/lpfc/lpfc_nvmet.h b/drivers/scsi/lpfc/lpfc_nvmet.h index 0ec1082ce7ef..368deea2bcf8 100644 --- a/drivers/scsi/lpfc/lpfc_nvmet.h +++ b/drivers/scsi/lpfc/lpfc_nvmet.h @@ -1,7 +1,7 @@ /******************************************************************* * This file is part of the Emulex Linux Device Driver for * * Fibre Channel Host Bus Adapters. * - * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term * + * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term * * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * * Copyright (C) 2004-2016 Emulex. All rights reserved. * * EMULEX and SLI are trademarks of Emulex. * @@ -137,9 +137,11 @@ struct lpfc_nvmet_rcv_ctx { #define LPFC_NVMET_XBUSY 0x4 /* XB bit set on IO cmpl */ #define LPFC_NVMET_CTX_RLS 0x8 /* ctx free requested */ #define LPFC_NVMET_ABTS_RCV 0x10 /* ABTS received on exchange */ +#define LPFC_NVMET_CTX_REUSE_WQ 0x20 /* ctx reused via WQ */ #define LPFC_NVMET_DEFER_WQFULL 0x40 /* Waiting on a free WQE */ struct rqb_dmabuf *rqb_buffer; struct lpfc_nvmet_ctxbuf *ctxbuf; + struct lpfc_sli4_hdw_queue *hdwq; #ifdef CONFIG_SCSI_LPFC_DEBUG_FS uint64_t ts_isr_cmd; diff --git a/drivers/scsi/lpfc/lpfc_scsi.c b/drivers/scsi/lpfc/lpfc_scsi.c index b4f1a840b3b4..c98f264f1d83 100644 --- a/drivers/scsi/lpfc/lpfc_scsi.c +++ b/drivers/scsi/lpfc/lpfc_scsi.c @@ -1,7 +1,7 @@ /******************************************************************* * This file is part of the Emulex Linux Device Driver for * * Fibre Channel Host Bus Adapters. * - * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term * + * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term * * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * * Copyright (C) 2004-2016 Emulex. All rights reserved. * * EMULEX and SLI are trademarks of Emulex. * @@ -83,9 +83,9 @@ lpfc_rport_data_from_scsi_device(struct scsi_device *sdev) } static void -lpfc_release_scsi_buf_s4(struct lpfc_hba *phba, struct lpfc_scsi_buf *psb); +lpfc_release_scsi_buf_s4(struct lpfc_hba *phba, struct lpfc_io_buf *psb); static void -lpfc_release_scsi_buf_s3(struct lpfc_hba *phba, struct lpfc_scsi_buf *psb); +lpfc_release_scsi_buf_s3(struct lpfc_hba *phba, struct lpfc_io_buf *psb); static int lpfc_prot_group_type(struct lpfc_hba *phba, struct scsi_cmnd *sc); @@ -180,9 +180,9 @@ lpfc_cmd_guard_csum(struct scsi_cmnd *sc) **/ static void lpfc_sli4_set_rsp_sgl_last(struct lpfc_hba *phba, - struct lpfc_scsi_buf *lpfc_cmd) + struct lpfc_io_buf *lpfc_cmd) { - struct sli4_sge *sgl = (struct sli4_sge *)lpfc_cmd->fcp_bpl; + struct sli4_sge *sgl = (struct sli4_sge *)lpfc_cmd->dma_sgl; if (sgl) { sgl += 1; sgl->word2 = le32_to_cpu(sgl->word2); @@ -200,7 +200,7 @@ lpfc_sli4_set_rsp_sgl_last(struct lpfc_hba *phba, * function updates the statistical data for the command completion. **/ static void -lpfc_update_stats(struct lpfc_hba *phba, struct lpfc_scsi_buf *lpfc_cmd) +lpfc_update_stats(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_cmd) { struct lpfc_rport_data *rdata; struct lpfc_nodelist *pnode; @@ -389,12 +389,12 @@ static int lpfc_new_scsi_buf_s3(struct lpfc_vport *vport, int num_to_alloc) { struct lpfc_hba *phba = vport->phba; - struct lpfc_scsi_buf *psb; + struct lpfc_io_buf *psb; struct ulp_bde64 *bpl; IOCB_t *iocb; dma_addr_t pdma_phys_fcp_cmd; dma_addr_t pdma_phys_fcp_rsp; - dma_addr_t pdma_phys_bpl; + dma_addr_t pdma_phys_sgl; uint16_t iotag; int bcnt, bpl_size; @@ -408,7 +408,7 @@ lpfc_new_scsi_buf_s3(struct lpfc_vport *vport, int num_to_alloc) (int)sizeof(struct fcp_rsp), bpl_size); for (bcnt = 0; bcnt < num_to_alloc; bcnt++) { - psb = kzalloc(sizeof(struct lpfc_scsi_buf), GFP_KERNEL); + psb = kzalloc(sizeof(struct lpfc_io_buf), GFP_KERNEL); if (!psb) break; @@ -438,14 +438,14 @@ lpfc_new_scsi_buf_s3(struct lpfc_vport *vport, int num_to_alloc) psb->fcp_cmnd = psb->data; psb->fcp_rsp = psb->data + sizeof(struct fcp_cmnd); - psb->fcp_bpl = psb->data + sizeof(struct fcp_cmnd) + + psb->dma_sgl = psb->data + sizeof(struct fcp_cmnd) + sizeof(struct fcp_rsp); /* Initialize local short-hand pointers. */ - bpl = psb->fcp_bpl; + bpl = (struct ulp_bde64 *)psb->dma_sgl; pdma_phys_fcp_cmd = psb->dma_handle; pdma_phys_fcp_rsp = psb->dma_handle + sizeof(struct fcp_cmnd); - pdma_phys_bpl = psb->dma_handle + sizeof(struct fcp_cmnd) + + pdma_phys_sgl = psb->dma_handle + sizeof(struct fcp_cmnd) + sizeof(struct fcp_rsp); /* @@ -496,9 +496,9 @@ lpfc_new_scsi_buf_s3(struct lpfc_vport *vport, int num_to_alloc) iocb->un.fcpi64.bdl.bdeSize = (2 * sizeof(struct ulp_bde64)); iocb->un.fcpi64.bdl.addrLow = - putPaddrLow(pdma_phys_bpl); + putPaddrLow(pdma_phys_sgl); iocb->un.fcpi64.bdl.addrHigh = - putPaddrHigh(pdma_phys_bpl); + putPaddrHigh(pdma_phys_sgl); iocb->ulpBdeCount = 1; iocb->ulpLe = 1; } @@ -506,6 +506,7 @@ lpfc_new_scsi_buf_s3(struct lpfc_vport *vport, int num_to_alloc) psb->status = IOSTAT_SUCCESS; /* Put it back into the SCSI buffer list */ psb->cur_iocbq.context1 = psb; + spin_lock_init(&psb->buf_lock); lpfc_release_scsi_buf_s3(phba, psb); } @@ -524,20 +525,27 @@ void lpfc_sli4_vport_delete_fcp_xri_aborted(struct lpfc_vport *vport) { struct lpfc_hba *phba = vport->phba; - struct lpfc_scsi_buf *psb, *next_psb; + struct lpfc_io_buf *psb, *next_psb; + struct lpfc_sli4_hdw_queue *qp; unsigned long iflag = 0; + int idx; - if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_FCP)) + if (!(vport->cfg_enable_fc4_type & LPFC_ENABLE_FCP)) return; + spin_lock_irqsave(&phba->hbalock, iflag); - spin_lock(&phba->sli4_hba.abts_scsi_buf_list_lock); - list_for_each_entry_safe(psb, next_psb, - &phba->sli4_hba.lpfc_abts_scsi_buf_list, list) { - if (psb->rdata && psb->rdata->pnode - && psb->rdata->pnode->vport == vport) - psb->rdata = NULL; + for (idx = 0; idx < phba->cfg_hdw_queue; idx++) { + qp = &phba->sli4_hba.hdwq[idx]; + + spin_lock(&qp->abts_scsi_buf_list_lock); + list_for_each_entry_safe(psb, next_psb, + &qp->lpfc_abts_scsi_buf_list, list) { + if (psb->rdata && psb->rdata->pnode && + psb->rdata->pnode->vport == vport) + psb->rdata = NULL; + } + spin_unlock(&qp->abts_scsi_buf_list_lock); } - spin_unlock(&phba->sli4_hba.abts_scsi_buf_list_lock); spin_unlock_irqrestore(&phba->hbalock, iflag); } @@ -551,11 +559,12 @@ lpfc_sli4_vport_delete_fcp_xri_aborted(struct lpfc_vport *vport) **/ void lpfc_sli4_fcp_xri_aborted(struct lpfc_hba *phba, - struct sli4_wcqe_xri_aborted *axri) + struct sli4_wcqe_xri_aborted *axri, int idx) { uint16_t xri = bf_get(lpfc_wcqe_xa_xri, axri); uint16_t rxid = bf_get(lpfc_wcqe_xa_remote_xid, axri); - struct lpfc_scsi_buf *psb, *next_psb; + struct lpfc_io_buf *psb, *next_psb; + struct lpfc_sli4_hdw_queue *qp; unsigned long iflag = 0; struct lpfc_iocbq *iocbq; int i; @@ -565,16 +574,19 @@ lpfc_sli4_fcp_xri_aborted(struct lpfc_hba *phba, if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_FCP)) return; + + qp = &phba->sli4_hba.hdwq[idx]; spin_lock_irqsave(&phba->hbalock, iflag); - spin_lock(&phba->sli4_hba.abts_scsi_buf_list_lock); + spin_lock(&qp->abts_scsi_buf_list_lock); list_for_each_entry_safe(psb, next_psb, - &phba->sli4_hba.lpfc_abts_scsi_buf_list, list) { + &qp->lpfc_abts_scsi_buf_list, list) { if (psb->cur_iocbq.sli4_xritag == xri) { list_del(&psb->list); + qp->abts_scsi_io_bufs--; psb->exch_busy = 0; psb->status = IOSTAT_SUCCESS; spin_unlock( - &phba->sli4_hba.abts_scsi_buf_list_lock); + &qp->abts_scsi_buf_list_lock); if (psb->rdata && psb->rdata->pnode) ndlp = psb->rdata->pnode; else @@ -593,7 +605,7 @@ lpfc_sli4_fcp_xri_aborted(struct lpfc_hba *phba, return; } } - spin_unlock(&phba->sli4_hba.abts_scsi_buf_list_lock); + spin_unlock(&qp->abts_scsi_buf_list_lock); for (i = 1; i <= phba->sli.last_iotag; i++) { iocbq = phba->sli.iocbq_lookup[i]; @@ -602,7 +614,7 @@ lpfc_sli4_fcp_xri_aborted(struct lpfc_hba *phba, continue; if (iocbq->sli4_xritag != xri) continue; - psb = container_of(iocbq, struct lpfc_scsi_buf, cur_iocbq); + psb = container_of(iocbq, struct lpfc_io_buf, cur_iocbq); psb->exch_busy = 0; spin_unlock_irqrestore(&phba->hbalock, iflag); if (!list_empty(&pring->txq)) @@ -614,359 +626,6 @@ lpfc_sli4_fcp_xri_aborted(struct lpfc_hba *phba, } /** - * lpfc_sli4_post_scsi_sgl_list - Post blocks of scsi buffer sgls from a list - * @phba: pointer to lpfc hba data structure. - * @post_sblist: pointer to the scsi buffer list. - * - * This routine walks a list of scsi buffers that was passed in. It attempts - * to construct blocks of scsi buffer sgls which contains contiguous xris and - * uses the non-embedded SGL block post mailbox commands to post to the port. - * For single SCSI buffer sgl with non-contiguous xri, if any, it shall use - * embedded SGL post mailbox command for posting. The @post_sblist passed in - * must be local list, thus no lock is needed when manipulate the list. - * - * Returns: 0 = failure, non-zero number of successfully posted buffers. - **/ -static int -lpfc_sli4_post_scsi_sgl_list(struct lpfc_hba *phba, - struct list_head *post_sblist, int sb_count) -{ - struct lpfc_scsi_buf *psb, *psb_next; - int status, sgl_size; - int post_cnt = 0, block_cnt = 0, num_posting = 0, num_posted = 0; - dma_addr_t pdma_phys_bpl1; - int last_xritag = NO_XRI; - LIST_HEAD(prep_sblist); - LIST_HEAD(blck_sblist); - LIST_HEAD(scsi_sblist); - - /* sanity check */ - if (sb_count <= 0) - return -EINVAL; - - sgl_size = phba->cfg_sg_dma_buf_size - - (sizeof(struct fcp_cmnd) + sizeof(struct fcp_rsp)); - - list_for_each_entry_safe(psb, psb_next, post_sblist, list) { - list_del_init(&psb->list); - block_cnt++; - if ((last_xritag != NO_XRI) && - (psb->cur_iocbq.sli4_xritag != last_xritag + 1)) { - /* a hole in xri block, form a sgl posting block */ - list_splice_init(&prep_sblist, &blck_sblist); - post_cnt = block_cnt - 1; - /* prepare list for next posting block */ - list_add_tail(&psb->list, &prep_sblist); - block_cnt = 1; - } else { - /* prepare list for next posting block */ - list_add_tail(&psb->list, &prep_sblist); - /* enough sgls for non-embed sgl mbox command */ - if (block_cnt == LPFC_NEMBED_MBOX_SGL_CNT) { - list_splice_init(&prep_sblist, &blck_sblist); - post_cnt = block_cnt; - block_cnt = 0; - } - } - num_posting++; - last_xritag = psb->cur_iocbq.sli4_xritag; - - /* end of repost sgl list condition for SCSI buffers */ - if (num_posting == sb_count) { - if (post_cnt == 0) { - /* last sgl posting block */ - list_splice_init(&prep_sblist, &blck_sblist); - post_cnt = block_cnt; - } else if (block_cnt == 1) { - /* last single sgl with non-contiguous xri */ - if (sgl_size > SGL_PAGE_SIZE) - pdma_phys_bpl1 = psb->dma_phys_bpl + - SGL_PAGE_SIZE; - else - pdma_phys_bpl1 = 0; - status = lpfc_sli4_post_sgl(phba, - psb->dma_phys_bpl, - pdma_phys_bpl1, - psb->cur_iocbq.sli4_xritag); - if (status) { - /* failure, put on abort scsi list */ - psb->exch_busy = 1; - } else { - /* success, put on SCSI buffer list */ - psb->exch_busy = 0; - psb->status = IOSTAT_SUCCESS; - num_posted++; - } - /* success, put on SCSI buffer sgl list */ - list_add_tail(&psb->list, &scsi_sblist); - } - } - - /* continue until a nembed page worth of sgls */ - if (post_cnt == 0) - continue; - - /* post block of SCSI buffer list sgls */ - status = lpfc_sli4_post_scsi_sgl_block(phba, &blck_sblist, - post_cnt); - - /* don't reset xirtag due to hole in xri block */ - if (block_cnt == 0) - last_xritag = NO_XRI; - - /* reset SCSI buffer post count for next round of posting */ - post_cnt = 0; - - /* put posted SCSI buffer-sgl posted on SCSI buffer sgl list */ - while (!list_empty(&blck_sblist)) { - list_remove_head(&blck_sblist, psb, - struct lpfc_scsi_buf, list); - if (status) { - /* failure, put on abort scsi list */ - psb->exch_busy = 1; - } else { - /* success, put on SCSI buffer list */ - psb->exch_busy = 0; - psb->status = IOSTAT_SUCCESS; - num_posted++; - } - list_add_tail(&psb->list, &scsi_sblist); - } - } - /* Push SCSI buffers with sgl posted to the availble list */ - while (!list_empty(&scsi_sblist)) { - list_remove_head(&scsi_sblist, psb, - struct lpfc_scsi_buf, list); - lpfc_release_scsi_buf_s4(phba, psb); - } - return num_posted; -} - -/** - * lpfc_sli4_repost_scsi_sgl_list - Repost all the allocated scsi buffer sgls - * @phba: pointer to lpfc hba data structure. - * - * This routine walks the list of scsi buffers that have been allocated and - * repost them to the port by using SGL block post. This is needed after a - * pci_function_reset/warm_start or start. The lpfc_hba_down_post_s4 routine - * is responsible for moving all scsi buffers on the lpfc_abts_scsi_sgl_list - * to the lpfc_scsi_buf_list. If the repost fails, reject all scsi buffers. - * - * Returns: 0 = success, non-zero failure. - **/ -int -lpfc_sli4_repost_scsi_sgl_list(struct lpfc_hba *phba) -{ - LIST_HEAD(post_sblist); - int num_posted, rc = 0; - - /* get all SCSI buffers need to repost to a local list */ - spin_lock_irq(&phba->scsi_buf_list_get_lock); - spin_lock(&phba->scsi_buf_list_put_lock); - list_splice_init(&phba->lpfc_scsi_buf_list_get, &post_sblist); - list_splice(&phba->lpfc_scsi_buf_list_put, &post_sblist); - spin_unlock(&phba->scsi_buf_list_put_lock); - spin_unlock_irq(&phba->scsi_buf_list_get_lock); - - /* post the list of scsi buffer sgls to port if available */ - if (!list_empty(&post_sblist)) { - num_posted = lpfc_sli4_post_scsi_sgl_list(phba, &post_sblist, - phba->sli4_hba.scsi_xri_cnt); - /* failed to post any scsi buffer, return error */ - if (num_posted == 0) - rc = -EIO; - } - return rc; -} - -/** - * lpfc_new_scsi_buf_s4 - Scsi buffer allocator for HBA with SLI4 IF spec - * @vport: The virtual port for which this call being executed. - * @num_to_allocate: The requested number of buffers to allocate. - * - * This routine allocates scsi buffers for device with SLI-4 interface spec, - * the scsi buffer contains all the necessary information needed to initiate - * a SCSI I/O. After allocating up to @num_to_allocate SCSI buffers and put - * them on a list, it post them to the port by using SGL block post. - * - * Return codes: - * int - number of scsi buffers that were allocated and posted. - * 0 = failure, less than num_to_alloc is a partial failure. - **/ -static int -lpfc_new_scsi_buf_s4(struct lpfc_vport *vport, int num_to_alloc) -{ - struct lpfc_hba *phba = vport->phba; - struct lpfc_scsi_buf *psb; - struct sli4_sge *sgl; - IOCB_t *iocb; - dma_addr_t pdma_phys_fcp_cmd; - dma_addr_t pdma_phys_fcp_rsp; - dma_addr_t pdma_phys_bpl; - uint16_t iotag, lxri = 0; - int bcnt, num_posted, sgl_size; - LIST_HEAD(prep_sblist); - LIST_HEAD(post_sblist); - LIST_HEAD(scsi_sblist); - - sgl_size = phba->cfg_sg_dma_buf_size - - (sizeof(struct fcp_cmnd) + sizeof(struct fcp_rsp)); - - lpfc_printf_vlog(vport, KERN_INFO, LOG_FCP, - "9068 ALLOC %d scsi_bufs: %d (%d + %d + %d)\n", - num_to_alloc, phba->cfg_sg_dma_buf_size, sgl_size, - (int)sizeof(struct fcp_cmnd), - (int)sizeof(struct fcp_rsp)); - - for (bcnt = 0; bcnt < num_to_alloc; bcnt++) { - psb = kzalloc(sizeof(struct lpfc_scsi_buf), GFP_KERNEL); - if (!psb) - break; - /* - * Get memory from the pci pool to map the virt space to - * pci bus space for an I/O. The DMA buffer includes space - * for the struct fcp_cmnd, struct fcp_rsp and the number - * of bde's necessary to support the sg_tablesize. - */ - psb->data = dma_pool_zalloc(phba->lpfc_sg_dma_buf_pool, - GFP_KERNEL, &psb->dma_handle); - if (!psb->data) { - kfree(psb); - break; - } - - /* - * 4K Page alignment is CRITICAL to BlockGuard, double check - * to be sure. - */ - if ((phba->sli3_options & LPFC_SLI3_BG_ENABLED) && - (((unsigned long)(psb->data) & - (unsigned long)(SLI4_PAGE_SIZE - 1)) != 0)) { - lpfc_printf_log(phba, KERN_ERR, LOG_FCP, - "3369 Memory alignment error " - "addr=%lx\n", - (unsigned long)psb->data); - dma_pool_free(phba->lpfc_sg_dma_buf_pool, - psb->data, psb->dma_handle); - kfree(psb); - break; - } - - - lxri = lpfc_sli4_next_xritag(phba); - if (lxri == NO_XRI) { - dma_pool_free(phba->lpfc_sg_dma_buf_pool, - psb->data, psb->dma_handle); - kfree(psb); - break; - } - - /* Allocate iotag for psb->cur_iocbq. */ - iotag = lpfc_sli_next_iotag(phba, &psb->cur_iocbq); - if (iotag == 0) { - dma_pool_free(phba->lpfc_sg_dma_buf_pool, - psb->data, psb->dma_handle); - kfree(psb); - lpfc_printf_log(phba, KERN_ERR, LOG_FCP, - "3368 Failed to allocate IOTAG for" - " XRI:0x%x\n", lxri); - lpfc_sli4_free_xri(phba, lxri); - break; - } - psb->cur_iocbq.sli4_lxritag = lxri; - psb->cur_iocbq.sli4_xritag = phba->sli4_hba.xri_ids[lxri]; - psb->cur_iocbq.iocb_flag |= LPFC_IO_FCP; - psb->fcp_bpl = psb->data; - psb->fcp_cmnd = (psb->data + sgl_size); - psb->fcp_rsp = (struct fcp_rsp *)((uint8_t *)psb->fcp_cmnd + - sizeof(struct fcp_cmnd)); - - /* Initialize local short-hand pointers. */ - sgl = (struct sli4_sge *)psb->fcp_bpl; - pdma_phys_bpl = psb->dma_handle; - pdma_phys_fcp_cmd = (psb->dma_handle + sgl_size); - pdma_phys_fcp_rsp = pdma_phys_fcp_cmd + sizeof(struct fcp_cmnd); - - /* - * The first two bdes are the FCP_CMD and FCP_RSP. - * The balance are sg list bdes. Initialize the - * first two and leave the rest for queuecommand. - */ - sgl->addr_hi = cpu_to_le32(putPaddrHigh(pdma_phys_fcp_cmd)); - sgl->addr_lo = cpu_to_le32(putPaddrLow(pdma_phys_fcp_cmd)); - sgl->word2 = le32_to_cpu(sgl->word2); - bf_set(lpfc_sli4_sge_last, sgl, 0); - sgl->word2 = cpu_to_le32(sgl->word2); - sgl->sge_len = cpu_to_le32(sizeof(struct fcp_cmnd)); - sgl++; - - /* Setup the physical region for the FCP RSP */ - sgl->addr_hi = cpu_to_le32(putPaddrHigh(pdma_phys_fcp_rsp)); - sgl->addr_lo = cpu_to_le32(putPaddrLow(pdma_phys_fcp_rsp)); - sgl->word2 = le32_to_cpu(sgl->word2); - bf_set(lpfc_sli4_sge_last, sgl, 1); - sgl->word2 = cpu_to_le32(sgl->word2); - sgl->sge_len = cpu_to_le32(sizeof(struct fcp_rsp)); - - /* - * Since the IOCB for the FCP I/O is built into this - * lpfc_scsi_buf, initialize it with all known data now. - */ - iocb = &psb->cur_iocbq.iocb; - iocb->un.fcpi64.bdl.ulpIoTag32 = 0; - iocb->un.fcpi64.bdl.bdeFlags = BUFF_TYPE_BDE_64; - /* setting the BLP size to 2 * sizeof BDE may not be correct. - * We are setting the bpl to point to out sgl. An sgl's - * entries are 16 bytes, a bpl entries are 12 bytes. - */ - iocb->un.fcpi64.bdl.bdeSize = sizeof(struct fcp_cmnd); - iocb->un.fcpi64.bdl.addrLow = putPaddrLow(pdma_phys_fcp_cmd); - iocb->un.fcpi64.bdl.addrHigh = putPaddrHigh(pdma_phys_fcp_cmd); - iocb->ulpBdeCount = 1; - iocb->ulpLe = 1; - iocb->ulpClass = CLASS3; - psb->cur_iocbq.context1 = psb; - psb->dma_phys_bpl = pdma_phys_bpl; - - /* add the scsi buffer to a post list */ - list_add_tail(&psb->list, &post_sblist); - spin_lock_irq(&phba->scsi_buf_list_get_lock); - phba->sli4_hba.scsi_xri_cnt++; - spin_unlock_irq(&phba->scsi_buf_list_get_lock); - } - lpfc_printf_log(phba, KERN_INFO, LOG_BG | LOG_FCP, - "3021 Allocate %d out of %d requested new SCSI " - "buffers\n", bcnt, num_to_alloc); - - /* post the list of scsi buffer sgls to port if available */ - if (!list_empty(&post_sblist)) - num_posted = lpfc_sli4_post_scsi_sgl_list(phba, - &post_sblist, bcnt); - else - num_posted = 0; - - return num_posted; -} - -/** - * lpfc_new_scsi_buf - Wrapper funciton for scsi buffer allocator - * @vport: The virtual port for which this call being executed. - * @num_to_allocate: The requested number of buffers to allocate. - * - * This routine wraps the actual SCSI buffer allocator function pointer from - * the lpfc_hba struct. - * - * Return codes: - * int - number of scsi buffers that were allocated. - * 0 = failure, less than num_to_alloc is a partial failure. - **/ -static inline int -lpfc_new_scsi_buf(struct lpfc_vport *vport, int num_to_alloc) -{ - return vport->phba->lpfc_new_scsi_buf(vport, num_to_alloc); -} - -/** * lpfc_get_scsi_buf_s3 - Get a scsi buffer from lpfc_scsi_buf_list of the HBA * @phba: The HBA for which this call is being executed. * @@ -977,15 +636,16 @@ lpfc_new_scsi_buf(struct lpfc_vport *vport, int num_to_alloc) * NULL - Error * Pointer to lpfc_scsi_buf - Success **/ -static struct lpfc_scsi_buf* -lpfc_get_scsi_buf_s3(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp) +static struct lpfc_io_buf * +lpfc_get_scsi_buf_s3(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, + struct scsi_cmnd *cmnd) { - struct lpfc_scsi_buf * lpfc_cmd = NULL; + struct lpfc_io_buf *lpfc_cmd = NULL; struct list_head *scsi_buf_list_get = &phba->lpfc_scsi_buf_list_get; unsigned long iflag = 0; spin_lock_irqsave(&phba->scsi_buf_list_get_lock, iflag); - list_remove_head(scsi_buf_list_get, lpfc_cmd, struct lpfc_scsi_buf, + list_remove_head(scsi_buf_list_get, lpfc_cmd, struct lpfc_io_buf, list); if (!lpfc_cmd) { spin_lock(&phba->scsi_buf_list_put_lock); @@ -993,7 +653,7 @@ lpfc_get_scsi_buf_s3(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp) &phba->lpfc_scsi_buf_list_get); INIT_LIST_HEAD(&phba->lpfc_scsi_buf_list_put); list_remove_head(scsi_buf_list_get, lpfc_cmd, - struct lpfc_scsi_buf, list); + struct lpfc_io_buf, list); spin_unlock(&phba->scsi_buf_list_put_lock); } spin_unlock_irqrestore(&phba->scsi_buf_list_get_lock, iflag); @@ -1005,52 +665,107 @@ lpfc_get_scsi_buf_s3(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp) return lpfc_cmd; } /** - * lpfc_get_scsi_buf_s4 - Get a scsi buffer from lpfc_scsi_buf_list of the HBA + * lpfc_get_scsi_buf_s4 - Get a scsi buffer from io_buf_list of the HBA * @phba: The HBA for which this call is being executed. * - * This routine removes a scsi buffer from head of @phba lpfc_scsi_buf_list list + * This routine removes a scsi buffer from head of @hdwq io_buf_list * and returns to caller. * * Return codes: * NULL - Error * Pointer to lpfc_scsi_buf - Success **/ -static struct lpfc_scsi_buf* -lpfc_get_scsi_buf_s4(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp) +static struct lpfc_io_buf * +lpfc_get_scsi_buf_s4(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, + struct scsi_cmnd *cmnd) { - struct lpfc_scsi_buf *lpfc_cmd, *lpfc_cmd_next; - unsigned long iflag = 0; - int found = 0; + struct lpfc_io_buf *lpfc_cmd; + struct lpfc_sli4_hdw_queue *qp; + struct sli4_sge *sgl; + IOCB_t *iocb; + dma_addr_t pdma_phys_fcp_rsp; + dma_addr_t pdma_phys_fcp_cmd; + uint32_t sgl_size, cpu, idx; + int tag; - spin_lock_irqsave(&phba->scsi_buf_list_get_lock, iflag); - list_for_each_entry_safe(lpfc_cmd, lpfc_cmd_next, - &phba->lpfc_scsi_buf_list_get, list) { - if (lpfc_test_rrq_active(phba, ndlp, - lpfc_cmd->cur_iocbq.sli4_lxritag)) - continue; - list_del_init(&lpfc_cmd->list); - found = 1; - break; - } - if (!found) { - spin_lock(&phba->scsi_buf_list_put_lock); - list_splice(&phba->lpfc_scsi_buf_list_put, - &phba->lpfc_scsi_buf_list_get); - INIT_LIST_HEAD(&phba->lpfc_scsi_buf_list_put); - spin_unlock(&phba->scsi_buf_list_put_lock); - list_for_each_entry_safe(lpfc_cmd, lpfc_cmd_next, - &phba->lpfc_scsi_buf_list_get, list) { - if (lpfc_test_rrq_active( - phba, ndlp, lpfc_cmd->cur_iocbq.sli4_lxritag)) - continue; - list_del_init(&lpfc_cmd->list); - found = 1; - break; - } + cpu = smp_processor_id(); + if (cmnd && phba->cfg_fcp_io_sched == LPFC_FCP_SCHED_BY_HDWQ) { + tag = blk_mq_unique_tag(cmnd->request); + idx = blk_mq_unique_tag_to_hwq(tag); + } else { + idx = phba->sli4_hba.cpu_map[cpu].hdwq; } - spin_unlock_irqrestore(&phba->scsi_buf_list_get_lock, iflag); - if (!found) + + lpfc_cmd = lpfc_get_io_buf(phba, ndlp, idx, + !phba->cfg_xri_rebalancing); + if (!lpfc_cmd) { + qp = &phba->sli4_hba.hdwq[idx]; + qp->empty_io_bufs++; return NULL; + } + + sgl_size = phba->cfg_sg_dma_buf_size - + (sizeof(struct fcp_cmnd) + sizeof(struct fcp_rsp)); + + /* Setup key fields in buffer that may have been changed + * if other protocols used this buffer. + */ + lpfc_cmd->cur_iocbq.iocb_flag = LPFC_IO_FCP; + lpfc_cmd->prot_seg_cnt = 0; + lpfc_cmd->seg_cnt = 0; + lpfc_cmd->timeout = 0; + lpfc_cmd->flags = 0; + lpfc_cmd->start_time = jiffies; + lpfc_cmd->waitq = NULL; + lpfc_cmd->cpu = cpu; +#ifdef CONFIG_SCSI_LPFC_DEBUG_FS + lpfc_cmd->prot_data_type = 0; +#endif + lpfc_cmd->fcp_cmnd = (lpfc_cmd->data + sgl_size); + lpfc_cmd->fcp_rsp = (struct fcp_rsp *)((uint8_t *)lpfc_cmd->fcp_cmnd + + sizeof(struct fcp_cmnd)); + + /* + * The first two SGEs are the FCP_CMD and FCP_RSP. + * The balance are sg list bdes. Initialize the + * first two and leave the rest for queuecommand. + */ + sgl = (struct sli4_sge *)lpfc_cmd->dma_sgl; + pdma_phys_fcp_cmd = (lpfc_cmd->dma_handle + sgl_size); + sgl->addr_hi = cpu_to_le32(putPaddrHigh(pdma_phys_fcp_cmd)); + sgl->addr_lo = cpu_to_le32(putPaddrLow(pdma_phys_fcp_cmd)); + sgl->word2 = le32_to_cpu(sgl->word2); + bf_set(lpfc_sli4_sge_last, sgl, 0); + sgl->word2 = cpu_to_le32(sgl->word2); + sgl->sge_len = cpu_to_le32(sizeof(struct fcp_cmnd)); + sgl++; + + /* Setup the physical region for the FCP RSP */ + pdma_phys_fcp_rsp = pdma_phys_fcp_cmd + sizeof(struct fcp_cmnd); + sgl->addr_hi = cpu_to_le32(putPaddrHigh(pdma_phys_fcp_rsp)); + sgl->addr_lo = cpu_to_le32(putPaddrLow(pdma_phys_fcp_rsp)); + sgl->word2 = le32_to_cpu(sgl->word2); + bf_set(lpfc_sli4_sge_last, sgl, 1); + sgl->word2 = cpu_to_le32(sgl->word2); + sgl->sge_len = cpu_to_le32(sizeof(struct fcp_rsp)); + + /* + * Since the IOCB for the FCP I/O is built into this + * lpfc_io_buf, initialize it with all known data now. + */ + iocb = &lpfc_cmd->cur_iocbq.iocb; + iocb->un.fcpi64.bdl.ulpIoTag32 = 0; + iocb->un.fcpi64.bdl.bdeFlags = BUFF_TYPE_BDE_64; + /* setting the BLP size to 2 * sizeof BDE may not be correct. + * We are setting the bpl to point to out sgl. An sgl's + * entries are 16 bytes, a bpl entries are 12 bytes. + */ + iocb->un.fcpi64.bdl.bdeSize = sizeof(struct fcp_cmnd); + iocb->un.fcpi64.bdl.addrLow = putPaddrLow(pdma_phys_fcp_cmd); + iocb->un.fcpi64.bdl.addrHigh = putPaddrHigh(pdma_phys_fcp_cmd); + iocb->ulpBdeCount = 1; + iocb->ulpLe = 1; + iocb->ulpClass = CLASS3; if (lpfc_ndlp_check_qdepth(phba, ndlp)) { atomic_inc(&ndlp->cmd_pending); @@ -1069,10 +784,11 @@ lpfc_get_scsi_buf_s4(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp) * NULL - Error * Pointer to lpfc_scsi_buf - Success **/ -static struct lpfc_scsi_buf* -lpfc_get_scsi_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp) +static struct lpfc_io_buf* +lpfc_get_scsi_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, + struct scsi_cmnd *cmnd) { - return phba->lpfc_get_scsi_buf(phba, ndlp); + return phba->lpfc_get_scsi_buf(phba, ndlp, cmnd); } /** @@ -1084,12 +800,11 @@ lpfc_get_scsi_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp) * lpfc_scsi_buf_list list. **/ static void -lpfc_release_scsi_buf_s3(struct lpfc_hba *phba, struct lpfc_scsi_buf *psb) +lpfc_release_scsi_buf_s3(struct lpfc_hba *phba, struct lpfc_io_buf *psb) { unsigned long iflag = 0; psb->seg_cnt = 0; - psb->nonsg_phys = 0; psb->prot_seg_cnt = 0; spin_lock_irqsave(&phba->scsi_buf_list_put_lock, iflag); @@ -1104,34 +819,29 @@ lpfc_release_scsi_buf_s3(struct lpfc_hba *phba, struct lpfc_scsi_buf *psb) * @phba: The Hba for which this call is being executed. * @psb: The scsi buffer which is being released. * - * This routine releases @psb scsi buffer by adding it to tail of @phba - * lpfc_scsi_buf_list list. For SLI4 XRI's are tied to the scsi buffer + * This routine releases @psb scsi buffer by adding it to tail of @hdwq + * io_buf_list list. For SLI4 XRI's are tied to the scsi buffer * and cannot be reused for at least RA_TOV amount of time if it was * aborted. **/ static void -lpfc_release_scsi_buf_s4(struct lpfc_hba *phba, struct lpfc_scsi_buf *psb) +lpfc_release_scsi_buf_s4(struct lpfc_hba *phba, struct lpfc_io_buf *psb) { + struct lpfc_sli4_hdw_queue *qp; unsigned long iflag = 0; psb->seg_cnt = 0; - psb->nonsg_phys = 0; psb->prot_seg_cnt = 0; + qp = psb->hdwq; if (psb->exch_busy) { - spin_lock_irqsave(&phba->sli4_hba.abts_scsi_buf_list_lock, - iflag); + spin_lock_irqsave(&qp->abts_scsi_buf_list_lock, iflag); psb->pCmd = NULL; - list_add_tail(&psb->list, - &phba->sli4_hba.lpfc_abts_scsi_buf_list); - spin_unlock_irqrestore(&phba->sli4_hba.abts_scsi_buf_list_lock, - iflag); + list_add_tail(&psb->list, &qp->lpfc_abts_scsi_buf_list); + qp->abts_scsi_io_bufs++; + spin_unlock_irqrestore(&qp->abts_scsi_buf_list_lock, iflag); } else { - psb->pCmd = NULL; - psb->cur_iocbq.iocb_flag = LPFC_IO_FCP; - spin_lock_irqsave(&phba->scsi_buf_list_put_lock, iflag); - list_add_tail(&psb->list, &phba->lpfc_scsi_buf_list_put); - spin_unlock_irqrestore(&phba->scsi_buf_list_put_lock, iflag); + lpfc_release_io_buf(phba, (struct lpfc_io_buf *)psb, qp); } } @@ -1144,7 +854,7 @@ lpfc_release_scsi_buf_s4(struct lpfc_hba *phba, struct lpfc_scsi_buf *psb) * lpfc_scsi_buf_list list. **/ static void -lpfc_release_scsi_buf(struct lpfc_hba *phba, struct lpfc_scsi_buf *psb) +lpfc_release_scsi_buf(struct lpfc_hba *phba, struct lpfc_io_buf *psb) { if ((psb->flags & LPFC_SBUF_BUMP_QDEPTH) && psb->ndlp) atomic_dec(&psb->ndlp->cmd_pending); @@ -1168,12 +878,12 @@ lpfc_release_scsi_buf(struct lpfc_hba *phba, struct lpfc_scsi_buf *psb) * 0 - Success **/ static int -lpfc_scsi_prep_dma_buf_s3(struct lpfc_hba *phba, struct lpfc_scsi_buf *lpfc_cmd) +lpfc_scsi_prep_dma_buf_s3(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_cmd) { struct scsi_cmnd *scsi_cmnd = lpfc_cmd->pCmd; struct scatterlist *sgel = NULL; struct fcp_cmnd *fcp_cmnd = lpfc_cmd->fcp_cmnd; - struct ulp_bde64 *bpl = lpfc_cmd->fcp_bpl; + struct ulp_bde64 *bpl = (struct ulp_bde64 *)lpfc_cmd->dma_sgl; struct lpfc_iocbq *iocbq = &lpfc_cmd->cur_iocbq; IOCB_t *iocb_cmd = &lpfc_cmd->cur_iocbq.iocb; struct ulp_bde64 *data_bde = iocb_cmd->unsli3.fcp_ext.dbde; @@ -1320,7 +1030,7 @@ lpfc_bg_err_inject(struct lpfc_hba *phba, struct scsi_cmnd *sc, uint32_t *reftag, uint16_t *apptag, uint32_t new_guard) { struct scatterlist *sgpe; /* s/g prot entry */ - struct lpfc_scsi_buf *lpfc_cmd = NULL; + struct lpfc_io_buf *lpfc_cmd = NULL; struct scsi_dif_tuple *src = NULL; struct lpfc_nodelist *ndlp; struct lpfc_rport_data *rdata; @@ -1379,7 +1089,7 @@ lpfc_bg_err_inject(struct lpfc_hba *phba, struct scsi_cmnd *sc, if (sgpe) { src = (struct scsi_dif_tuple *)sg_virt(sgpe); src += blockoff; - lpfc_cmd = (struct lpfc_scsi_buf *)sc->host_scribble; + lpfc_cmd = (struct lpfc_io_buf *)sc->host_scribble; } /* Should we change the Reference Tag */ @@ -2684,7 +2394,7 @@ lpfc_prot_group_type(struct lpfc_hba *phba, struct scsi_cmnd *sc) **/ static int lpfc_bg_scsi_adjust_dl(struct lpfc_hba *phba, - struct lpfc_scsi_buf *lpfc_cmd) + struct lpfc_io_buf *lpfc_cmd) { struct scsi_cmnd *sc = lpfc_cmd->pCmd; int fcpdl; @@ -2724,11 +2434,11 @@ lpfc_bg_scsi_adjust_dl(struct lpfc_hba *phba, **/ static int lpfc_bg_scsi_prep_dma_buf_s3(struct lpfc_hba *phba, - struct lpfc_scsi_buf *lpfc_cmd) + struct lpfc_io_buf *lpfc_cmd) { struct scsi_cmnd *scsi_cmnd = lpfc_cmd->pCmd; struct fcp_cmnd *fcp_cmnd = lpfc_cmd->fcp_cmnd; - struct ulp_bde64 *bpl = lpfc_cmd->fcp_bpl; + struct ulp_bde64 *bpl = (struct ulp_bde64 *)lpfc_cmd->dma_sgl; IOCB_t *iocb_cmd = &lpfc_cmd->cur_iocbq.iocb; uint32_t num_bde = 0; int datasegcnt, protsegcnt, datadir = scsi_cmnd->sc_data_direction; @@ -2904,7 +2614,7 @@ lpfc_bg_csum(uint8_t *data, int count) * what type of T10-DIF error occurred. */ static void -lpfc_calc_bg_err(struct lpfc_hba *phba, struct lpfc_scsi_buf *lpfc_cmd) +lpfc_calc_bg_err(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_cmd) { struct scatterlist *sgpe; /* s/g prot entry */ struct scatterlist *sgde; /* s/g data entry */ @@ -3089,8 +2799,8 @@ out: * -1 - Internal error (bad profile, ...etc) */ static int -lpfc_parse_bg_err(struct lpfc_hba *phba, struct lpfc_scsi_buf *lpfc_cmd, - struct lpfc_iocbq *pIocbOut) +lpfc_parse_bg_err(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_cmd, + struct lpfc_iocbq *pIocbOut) { struct scsi_cmnd *cmd = lpfc_cmd->pCmd; struct sli3_bg_fields *bgf = &pIocbOut->iocb.unsli3.sli3_bg; @@ -3256,12 +2966,12 @@ out: * 0 - Success **/ static int -lpfc_scsi_prep_dma_buf_s4(struct lpfc_hba *phba, struct lpfc_scsi_buf *lpfc_cmd) +lpfc_scsi_prep_dma_buf_s4(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_cmd) { struct scsi_cmnd *scsi_cmnd = lpfc_cmd->pCmd; struct scatterlist *sgel = NULL; struct fcp_cmnd *fcp_cmnd = lpfc_cmd->fcp_cmnd; - struct sli4_sge *sgl = (struct sli4_sge *)lpfc_cmd->fcp_bpl; + struct sli4_sge *sgl = (struct sli4_sge *)lpfc_cmd->dma_sgl; struct sli4_sge *first_data_sgl; IOCB_t *iocb_cmd = &lpfc_cmd->cur_iocbq.iocb; dma_addr_t physaddr; @@ -3388,6 +3098,7 @@ lpfc_scsi_prep_dma_buf_s4(struct lpfc_hba *phba, struct lpfc_scsi_buf *lpfc_cmd) lpfc_cmd->cur_iocbq.priority = ((struct lpfc_device_data *) scsi_cmnd->device->hostdata)->priority; } + return 0; } @@ -3402,11 +3113,11 @@ lpfc_scsi_prep_dma_buf_s4(struct lpfc_hba *phba, struct lpfc_scsi_buf *lpfc_cmd) **/ static int lpfc_bg_scsi_prep_dma_buf_s4(struct lpfc_hba *phba, - struct lpfc_scsi_buf *lpfc_cmd) + struct lpfc_io_buf *lpfc_cmd) { struct scsi_cmnd *scsi_cmnd = lpfc_cmd->pCmd; struct fcp_cmnd *fcp_cmnd = lpfc_cmd->fcp_cmnd; - struct sli4_sge *sgl = (struct sli4_sge *)(lpfc_cmd->fcp_bpl); + struct sli4_sge *sgl = (struct sli4_sge *)(lpfc_cmd->dma_sgl); IOCB_t *iocb_cmd = &lpfc_cmd->cur_iocbq.iocb; uint32_t num_sge = 0; int datasegcnt, protsegcnt, datadir = scsi_cmnd->sc_data_direction; @@ -3578,7 +3289,7 @@ err: * 0 - Success **/ static inline int -lpfc_scsi_prep_dma_buf(struct lpfc_hba *phba, struct lpfc_scsi_buf *lpfc_cmd) +lpfc_scsi_prep_dma_buf(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_cmd) { return phba->lpfc_scsi_prep_dma_buf(phba, lpfc_cmd); } @@ -3597,7 +3308,7 @@ lpfc_scsi_prep_dma_buf(struct lpfc_hba *phba, struct lpfc_scsi_buf *lpfc_cmd) * 0 - Success **/ static inline int -lpfc_bg_scsi_prep_dma_buf(struct lpfc_hba *phba, struct lpfc_scsi_buf *lpfc_cmd) +lpfc_bg_scsi_prep_dma_buf(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_cmd) { return phba->lpfc_bg_scsi_prep_dma_buf(phba, lpfc_cmd); } @@ -3614,7 +3325,7 @@ lpfc_bg_scsi_prep_dma_buf(struct lpfc_hba *phba, struct lpfc_scsi_buf *lpfc_cmd) **/ static void lpfc_send_scsi_error_event(struct lpfc_hba *phba, struct lpfc_vport *vport, - struct lpfc_scsi_buf *lpfc_cmd, struct lpfc_iocbq *rsp_iocb) { + struct lpfc_io_buf *lpfc_cmd, struct lpfc_iocbq *rsp_iocb) { struct scsi_cmnd *cmnd = lpfc_cmd->pCmd; struct fcp_rsp *fcprsp = lpfc_cmd->fcp_rsp; uint32_t resp_info = fcprsp->rspStatus2; @@ -3706,7 +3417,7 @@ lpfc_send_scsi_error_event(struct lpfc_hba *phba, struct lpfc_vport *vport, * field of @lpfc_cmd for device with SLI-3 interface spec. **/ static void -lpfc_scsi_unprep_dma_buf(struct lpfc_hba *phba, struct lpfc_scsi_buf *psb) +lpfc_scsi_unprep_dma_buf(struct lpfc_hba *phba, struct lpfc_io_buf *psb) { /* * There are only two special cases to consider. (1) the scsi command @@ -3725,7 +3436,7 @@ lpfc_scsi_unprep_dma_buf(struct lpfc_hba *phba, struct lpfc_scsi_buf *psb) /** * lpfc_handler_fcp_err - FCP response handler * @vport: The virtual port for which this call is being executed. - * @lpfc_cmd: Pointer to lpfc_scsi_buf data structure. + * @lpfc_cmd: Pointer to lpfc_io_buf data structure. * @rsp_iocb: The response IOCB which contains FCP error. * * This routine is called to process response IOCB with status field @@ -3733,7 +3444,7 @@ lpfc_scsi_unprep_dma_buf(struct lpfc_hba *phba, struct lpfc_scsi_buf *psb) * based upon SCSI and FCP error. **/ static void -lpfc_handle_fcp_err(struct lpfc_vport *vport, struct lpfc_scsi_buf *lpfc_cmd, +lpfc_handle_fcp_err(struct lpfc_vport *vport, struct lpfc_io_buf *lpfc_cmd, struct lpfc_iocbq *rsp_iocb) { struct lpfc_hba *phba = vport->phba; @@ -3912,49 +3623,6 @@ lpfc_handle_fcp_err(struct lpfc_vport *vport, struct lpfc_scsi_buf *lpfc_cmd, } /** - * lpfc_sli4_scmd_to_wqidx_distr - scsi command to SLI4 WQ index distribution - * @phba: Pointer to HBA context object. - * - * This routine performs a roundrobin SCSI command to SLI4 FCP WQ index - * distribution. This is called by __lpfc_sli_issue_iocb_s4() with the hbalock - * held. - * If scsi-mq is enabled, get the default block layer mapping of software queues - * to hardware queues. This information is saved in request tag. - * - * Return: index into SLI4 fast-path FCP queue index. - **/ -int lpfc_sli4_scmd_to_wqidx_distr(struct lpfc_hba *phba, - struct lpfc_scsi_buf *lpfc_cmd) -{ - struct scsi_cmnd *cmnd = lpfc_cmd->pCmd; - struct lpfc_vector_map_info *cpup; - int chann, cpu; - uint32_t tag; - uint16_t hwq; - - if (cmnd) { - tag = blk_mq_unique_tag(cmnd->request); - hwq = blk_mq_unique_tag_to_hwq(tag); - - return hwq; - } - - if (phba->cfg_fcp_io_sched == LPFC_FCP_SCHED_BY_CPU - && phba->cfg_fcp_io_channel > 1) { - cpu = smp_processor_id(); - if (cpu < phba->sli4_hba.num_present_cpu) { - cpup = phba->sli4_hba.cpu_map; - cpup += cpu; - return cpup->channel_id; - } - } - chann = atomic_add_return(1, &phba->fcp_qidx); - chann = chann % phba->cfg_fcp_io_channel; - return chann; -} - - -/** * lpfc_scsi_cmd_iocb_cmpl - Scsi cmnd IOCB completion routine * @phba: The Hba for which this call is being executed. * @pIocbIn: The command IOCBQ for the scsi cmnd. @@ -3968,8 +3636,8 @@ static void lpfc_scsi_cmd_iocb_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pIocbIn, struct lpfc_iocbq *pIocbOut) { - struct lpfc_scsi_buf *lpfc_cmd = - (struct lpfc_scsi_buf *) pIocbIn->context1; + struct lpfc_io_buf *lpfc_cmd = + (struct lpfc_io_buf *) pIocbIn->context1; struct lpfc_vport *vport = pIocbIn->vport; struct lpfc_rport_data *rdata = lpfc_cmd->rdata; struct lpfc_nodelist *pnode = rdata->pnode; @@ -3977,14 +3645,35 @@ lpfc_scsi_cmd_iocb_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pIocbIn, unsigned long flags; struct lpfc_fast_path_event *fast_path_evt; struct Scsi_Host *shost; + int idx; uint32_t logit = LOG_FCP; +#ifdef CONFIG_SCSI_LPFC_DEBUG_FS + int cpu; +#endif - atomic_inc(&phba->fc4ScsiIoCmpls); + /* Guard against abort handler being called at same time */ + spin_lock(&lpfc_cmd->buf_lock); /* Sanity check on return of outstanding command */ cmd = lpfc_cmd->pCmd; - if (!cmd) + if (!cmd) { + lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT, + "2621 IO completion: Not an active IO\n"); + spin_unlock(&lpfc_cmd->buf_lock); return; + } + + idx = lpfc_cmd->cur_iocbq.hba_wqidx; + if (phba->sli4_hba.hdwq) + phba->sli4_hba.hdwq[idx].scsi_cstat.io_cmpls++; + +#ifdef CONFIG_SCSI_LPFC_DEBUG_FS + if (phba->cpucheck_on & LPFC_CHECK_SCSI_IO) { + cpu = smp_processor_id(); + if (cpu < LPFC_CHECK_CPU_CNT) + phba->sli4_hba.hdwq[idx].cpucheck_cmpl_io[cpu]++; + } +#endif shost = cmd->device->host; lpfc_cmd->result = (pIocbOut->iocb.un.ulpWord[4] & IOERR_PARAM_MASK); @@ -4178,29 +3867,24 @@ lpfc_scsi_cmd_iocb_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pIocbIn, } lpfc_scsi_unprep_dma_buf(phba, lpfc_cmd); - /* If pCmd was set to NULL from abort path, do not call scsi_done */ - if (xchg(&lpfc_cmd->pCmd, NULL) == NULL) { - lpfc_printf_vlog(vport, KERN_INFO, LOG_FCP, - "5688 FCP cmd already NULL, sid: 0x%06x, " - "did: 0x%06x, oxid: 0x%04x\n", - vport->fc_myDID, - (pnode) ? pnode->nlp_DID : 0, - phba->sli_rev == LPFC_SLI_REV4 ? - lpfc_cmd->cur_iocbq.sli4_xritag : 0xffff); - return; - } + lpfc_cmd->pCmd = NULL; + spin_unlock(&lpfc_cmd->buf_lock); /* The sdev is not guaranteed to be valid post scsi_done upcall. */ cmd->scsi_done(cmd); /* - * If there is a thread waiting for command completion + * If there is an abort thread waiting for command completion * wake up the thread. */ - spin_lock_irqsave(shost->host_lock, flags); - if (lpfc_cmd->waitq) - wake_up(lpfc_cmd->waitq); - spin_unlock_irqrestore(shost->host_lock, flags); + spin_lock(&lpfc_cmd->buf_lock); + if (unlikely(lpfc_cmd->cur_iocbq.iocb_flag & LPFC_DRIVER_ABORTED)) { + lpfc_cmd->cur_iocbq.iocb_flag &= ~LPFC_DRIVER_ABORTED; + if (lpfc_cmd->waitq) + wake_up(lpfc_cmd->waitq); + lpfc_cmd->waitq = NULL; + } + spin_unlock(&lpfc_cmd->buf_lock); lpfc_release_scsi_buf(phba, lpfc_cmd); } @@ -4233,7 +3917,7 @@ lpfc_fcpcmd_to_iocb(uint8_t *data, struct fcp_cmnd *fcp_cmnd) * to transfer for device with SLI3 interface spec. **/ static void -lpfc_scsi_prep_cmnd(struct lpfc_vport *vport, struct lpfc_scsi_buf *lpfc_cmd, +lpfc_scsi_prep_cmnd(struct lpfc_vport *vport, struct lpfc_io_buf *lpfc_cmd, struct lpfc_nodelist *pnode) { struct lpfc_hba *phba = vport->phba; @@ -4241,7 +3925,9 @@ lpfc_scsi_prep_cmnd(struct lpfc_vport *vport, struct lpfc_scsi_buf *lpfc_cmd, struct fcp_cmnd *fcp_cmnd = lpfc_cmd->fcp_cmnd; IOCB_t *iocb_cmd = &lpfc_cmd->cur_iocbq.iocb; struct lpfc_iocbq *piocbq = &(lpfc_cmd->cur_iocbq); + struct lpfc_sli4_hdw_queue *hdwq = NULL; int datadir = scsi_cmnd->sc_data_direction; + int idx; uint8_t *ptr; bool sli4; uint32_t fcpdl; @@ -4267,6 +3953,9 @@ lpfc_scsi_prep_cmnd(struct lpfc_vport *vport, struct lpfc_scsi_buf *lpfc_cmd, sli4 = (phba->sli_rev == LPFC_SLI_REV4); piocbq->iocb.un.fcpi.fcpi_XRdy = 0; + idx = lpfc_cmd->hdwq_no; + if (phba->sli4_hba.hdwq) + hdwq = &phba->sli4_hba.hdwq[idx]; /* * There are three possibilities here - use scatter-gather segment, use @@ -4288,19 +3977,22 @@ lpfc_scsi_prep_cmnd(struct lpfc_vport *vport, struct lpfc_scsi_buf *lpfc_cmd, vport->cfg_first_burst_size; } fcp_cmnd->fcpCntl3 = WRITE_DATA; - atomic_inc(&phba->fc4ScsiOutputRequests); + if (hdwq) + hdwq->scsi_cstat.output_requests++; } else { iocb_cmd->ulpCommand = CMD_FCP_IREAD64_CR; iocb_cmd->ulpPU = PARM_READ_CHECK; fcp_cmnd->fcpCntl3 = READ_DATA; - atomic_inc(&phba->fc4ScsiInputRequests); + if (hdwq) + hdwq->scsi_cstat.input_requests++; } } else { iocb_cmd->ulpCommand = CMD_FCP_ICMND64_CR; iocb_cmd->un.fcpi.fcpi_parm = 0; iocb_cmd->ulpPU = 0; fcp_cmnd->fcpCntl3 = 0; - atomic_inc(&phba->fc4ScsiControlRequests); + if (hdwq) + hdwq->scsi_cstat.control_requests++; } if (phba->sli_rev == 3 && !(phba->sli3_options & LPFC_SLI3_BG_ENABLED)) @@ -4328,7 +4020,7 @@ lpfc_scsi_prep_cmnd(struct lpfc_vport *vport, struct lpfc_scsi_buf *lpfc_cmd, /** * lpfc_scsi_prep_task_mgmt_cmd - Convert SLI3 scsi TM cmd to FCP info unit * @vport: The virtual port for which this call is being executed. - * @lpfc_cmd: Pointer to lpfc_scsi_buf data structure. + * @lpfc_cmd: Pointer to lpfc_io_buf data structure. * @lun: Logical unit number. * @task_mgmt_cmd: SCSI task management command. * @@ -4341,7 +4033,7 @@ lpfc_scsi_prep_cmnd(struct lpfc_vport *vport, struct lpfc_scsi_buf *lpfc_cmd, **/ static int lpfc_scsi_prep_task_mgmt_cmd(struct lpfc_vport *vport, - struct lpfc_scsi_buf *lpfc_cmd, + struct lpfc_io_buf *lpfc_cmd, uint64_t lun, uint8_t task_mgmt_cmd) { @@ -4413,14 +4105,12 @@ lpfc_scsi_api_table_setup(struct lpfc_hba *phba, uint8_t dev_grp) switch (dev_grp) { case LPFC_PCI_DEV_LP: - phba->lpfc_new_scsi_buf = lpfc_new_scsi_buf_s3; phba->lpfc_scsi_prep_dma_buf = lpfc_scsi_prep_dma_buf_s3; phba->lpfc_bg_scsi_prep_dma_buf = lpfc_bg_scsi_prep_dma_buf_s3; phba->lpfc_release_scsi_buf = lpfc_release_scsi_buf_s3; phba->lpfc_get_scsi_buf = lpfc_get_scsi_buf_s3; break; case LPFC_PCI_DEV_OC: - phba->lpfc_new_scsi_buf = lpfc_new_scsi_buf_s4; phba->lpfc_scsi_prep_dma_buf = lpfc_scsi_prep_dma_buf_s4; phba->lpfc_bg_scsi_prep_dma_buf = lpfc_bg_scsi_prep_dma_buf_s4; phba->lpfc_release_scsi_buf = lpfc_release_scsi_buf_s4; @@ -4452,8 +4142,8 @@ lpfc_tskmgmt_def_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocbq, struct lpfc_iocbq *rspiocbq) { - struct lpfc_scsi_buf *lpfc_cmd = - (struct lpfc_scsi_buf *) cmdiocbq->context1; + struct lpfc_io_buf *lpfc_cmd = + (struct lpfc_io_buf *) cmdiocbq->context1; if (lpfc_cmd) lpfc_release_scsi_buf(phba, lpfc_cmd); return; @@ -4652,9 +4342,12 @@ lpfc_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *cmnd) struct lpfc_hba *phba = vport->phba; struct lpfc_rport_data *rdata; struct lpfc_nodelist *ndlp; - struct lpfc_scsi_buf *lpfc_cmd; + struct lpfc_io_buf *lpfc_cmd; struct fc_rport *rport = starget_to_rport(scsi_target(cmnd->device)); - int err; + int err, idx; +#ifdef CONFIG_SCSI_LPFC_DEBUG_FS + int cpu; +#endif rdata = lpfc_rport_data_from_scsi_device(cmnd->device); @@ -4718,7 +4411,7 @@ lpfc_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *cmnd) } } - lpfc_cmd = lpfc_get_scsi_buf(phba, ndlp); + lpfc_cmd = lpfc_get_scsi_buf(phba, ndlp, cmnd); if (lpfc_cmd == NULL) { lpfc_rampdown_queue_depth(phba); @@ -4735,8 +4428,6 @@ lpfc_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *cmnd) lpfc_cmd->pCmd = cmnd; lpfc_cmd->rdata = rdata; lpfc_cmd->ndlp = ndlp; - lpfc_cmd->timeout = 0; - lpfc_cmd->start_time = jiffies; cmnd->host_scribble = (unsigned char *)lpfc_cmd; if (scsi_get_prot_op(cmnd) != SCSI_PROT_NORMAL) { @@ -4771,6 +4462,16 @@ lpfc_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *cmnd) lpfc_scsi_prep_cmnd(vport, lpfc_cmd, ndlp); +#ifdef CONFIG_SCSI_LPFC_DEBUG_FS + if (phba->cpucheck_on & LPFC_CHECK_SCSI_IO) { + cpu = smp_processor_id(); + if (cpu < LPFC_CHECK_CPU_CNT) { + struct lpfc_sli4_hdw_queue *hdwq = + &phba->sli4_hba.hdwq[lpfc_cmd->hdwq_no]; + hdwq->cpucheck_xmt_io[cpu]++; + } + } +#endif err = lpfc_sli_issue_iocb(phba, LPFC_FCP_RING, &lpfc_cmd->cur_iocbq, SLI_IOCB_RET_IOCB); if (err) { @@ -4791,16 +4492,6 @@ lpfc_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *cmnd) (uint32_t) (cmnd->request->timeout / 1000)); - switch (lpfc_cmd->fcp_cmnd->fcpCntl3) { - case WRITE_DATA: - atomic_dec(&phba->fc4ScsiOutputRequests); - break; - case READ_DATA: - atomic_dec(&phba->fc4ScsiInputRequests); - break; - default: - atomic_dec(&phba->fc4ScsiControlRequests); - } goto out_host_busy_free_buf; } if (phba->cfg_poll & ENABLE_FCP_RING_POLLING) { @@ -4811,10 +4502,26 @@ lpfc_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *cmnd) lpfc_poll_rearm_timer(phba); } + if (phba->cfg_xri_rebalancing) + lpfc_keep_pvt_pool_above_lowwm(phba, lpfc_cmd->hdwq_no); + return 0; out_host_busy_free_buf: + idx = lpfc_cmd->hdwq_no; lpfc_scsi_unprep_dma_buf(phba, lpfc_cmd); + if (phba->sli4_hba.hdwq) { + switch (lpfc_cmd->fcp_cmnd->fcpCntl3) { + case WRITE_DATA: + phba->sli4_hba.hdwq[idx].scsi_cstat.output_requests--; + break; + case READ_DATA: + phba->sli4_hba.hdwq[idx].scsi_cstat.input_requests--; + break; + default: + phba->sli4_hba.hdwq[idx].scsi_cstat.control_requests--; + } + } lpfc_release_scsi_buf(phba, lpfc_cmd); out_host_busy: return SCSI_MLQUEUE_HOST_BUSY; @@ -4846,7 +4553,7 @@ lpfc_abort_handler(struct scsi_cmnd *cmnd) struct lpfc_hba *phba = vport->phba; struct lpfc_iocbq *iocb; struct lpfc_iocbq *abtsiocb; - struct lpfc_scsi_buf *lpfc_cmd; + struct lpfc_io_buf *lpfc_cmd; IOCB_t *cmd, *icmd; int ret = SUCCESS, status = 0; struct lpfc_sli_ring *pring_s4 = NULL; @@ -4858,65 +4565,59 @@ lpfc_abort_handler(struct scsi_cmnd *cmnd) if (status != 0 && status != SUCCESS) return status; + lpfc_cmd = (struct lpfc_io_buf *)cmnd->host_scribble; + if (!lpfc_cmd) + return ret; + spin_lock_irqsave(&phba->hbalock, flags); /* driver queued commands are in process of being flushed */ if (phba->hba_flag & HBA_FCP_IOQ_FLUSH) { - spin_unlock_irqrestore(&phba->hbalock, flags); lpfc_printf_vlog(vport, KERN_WARNING, LOG_FCP, "3168 SCSI Layer abort requested I/O has been " "flushed by LLD.\n"); - return FAILED; + ret = FAILED; + goto out_unlock; } - lpfc_cmd = (struct lpfc_scsi_buf *)cmnd->host_scribble; - if (!lpfc_cmd || !lpfc_cmd->pCmd) { - spin_unlock_irqrestore(&phba->hbalock, flags); + /* Guard against IO completion being called at same time */ + spin_lock(&lpfc_cmd->buf_lock); + + if (!lpfc_cmd->pCmd) { lpfc_printf_vlog(vport, KERN_WARNING, LOG_FCP, "2873 SCSI Layer I/O Abort Request IO CMPL Status " "x%x ID %d LUN %llu\n", SUCCESS, cmnd->device->id, cmnd->device->lun); - return SUCCESS; + goto out_unlock_buf; } iocb = &lpfc_cmd->cur_iocbq; if (phba->sli_rev == LPFC_SLI_REV4) { - if (!(phba->cfg_fof) || - (!(iocb->iocb_flag & LPFC_IO_FOF))) { - pring_s4 = - phba->sli4_hba.fcp_wq[iocb->hba_wqidx]->pring; - } else { - iocb->hba_wqidx = 0; - pring_s4 = phba->sli4_hba.oas_wq->pring; - } + pring_s4 = phba->sli4_hba.hdwq[iocb->hba_wqidx].fcp_wq->pring; if (!pring_s4) { ret = FAILED; - goto out_unlock; + goto out_unlock_buf; } spin_lock(&pring_s4->ring_lock); } /* the command is in process of being cancelled */ if (!(iocb->iocb_flag & LPFC_IO_ON_TXCMPLQ)) { - if (phba->sli_rev == LPFC_SLI_REV4) - spin_unlock(&pring_s4->ring_lock); - spin_unlock_irqrestore(&phba->hbalock, flags); lpfc_printf_vlog(vport, KERN_WARNING, LOG_FCP, "3169 SCSI Layer abort requested I/O has been " "cancelled by LLD.\n"); - return FAILED; + ret = FAILED; + goto out_unlock_ring; } /* - * If pCmd field of the corresponding lpfc_scsi_buf structure + * If pCmd field of the corresponding lpfc_io_buf structure * points to a different SCSI command, then the driver has * already completed this command, but the midlayer did not * see the completion before the eh fired. Just return SUCCESS. */ if (lpfc_cmd->pCmd != cmnd) { - if (phba->sli_rev == LPFC_SLI_REV4) - spin_unlock(&pring_s4->ring_lock); lpfc_printf_vlog(vport, KERN_WARNING, LOG_FCP, "3170 SCSI Layer abort requested I/O has been " "completed by LLD.\n"); - goto out_unlock; + goto out_unlock_ring; } BUG_ON(iocb->context1 != lpfc_cmd); @@ -4927,6 +4628,7 @@ lpfc_abort_handler(struct scsi_cmnd *cmnd) "3389 SCSI Layer I/O Abort Request is pending\n"); if (phba->sli_rev == LPFC_SLI_REV4) spin_unlock(&pring_s4->ring_lock); + spin_unlock(&lpfc_cmd->buf_lock); spin_unlock_irqrestore(&phba->hbalock, flags); goto wait_for_cmpl; } @@ -4934,9 +4636,7 @@ lpfc_abort_handler(struct scsi_cmnd *cmnd) abtsiocb = __lpfc_sli_get_iocbq(phba); if (abtsiocb == NULL) { ret = FAILED; - if (phba->sli_rev == LPFC_SLI_REV4) - spin_unlock(&pring_s4->ring_lock); - goto out_unlock; + goto out_unlock_ring; } /* Indicate the IO is being aborted by the driver. */ @@ -4986,24 +4686,18 @@ lpfc_abort_handler(struct scsi_cmnd *cmnd) /* no longer need the lock after this point */ spin_unlock_irqrestore(&phba->hbalock, flags); - if (ret_val == IOCB_ERROR) { - if (phba->sli_rev == LPFC_SLI_REV4) - spin_lock_irqsave(&pring_s4->ring_lock, flags); - else - spin_lock_irqsave(&phba->hbalock, flags); /* Indicate the IO is not being aborted by the driver. */ iocb->iocb_flag &= ~LPFC_DRIVER_ABORTED; lpfc_cmd->waitq = NULL; - if (phba->sli_rev == LPFC_SLI_REV4) - spin_unlock_irqrestore(&pring_s4->ring_lock, flags); - else - spin_unlock_irqrestore(&phba->hbalock, flags); + spin_unlock(&lpfc_cmd->buf_lock); lpfc_sli_release_iocbq(phba, abtsiocb); ret = FAILED; goto out; } + spin_unlock(&lpfc_cmd->buf_lock); + if (phba->cfg_poll & DISABLE_FCP_RING_INT) lpfc_sli_handle_fast_ring_event(phba, &phba->sli.sli3_ring[LPFC_FCP_RING], HA_R0RE_REQ); @@ -5014,9 +4708,7 @@ wait_for_cmpl: (lpfc_cmd->pCmd != cmnd), msecs_to_jiffies(2*vport->cfg_devloss_tmo*1000)); - spin_lock_irqsave(shost->host_lock, flags); - lpfc_cmd->waitq = NULL; - spin_unlock_irqrestore(shost->host_lock, flags); + spin_lock(&lpfc_cmd->buf_lock); if (lpfc_cmd->pCmd == cmnd) { ret = FAILED; @@ -5027,8 +4719,14 @@ wait_for_cmpl: iocb->sli4_xritag, ret, cmnd->device->id, cmnd->device->lun); } + spin_unlock(&lpfc_cmd->buf_lock); goto out; +out_unlock_ring: + if (phba->sli_rev == LPFC_SLI_REV4) + spin_unlock(&pring_s4->ring_lock); +out_unlock_buf: + spin_unlock(&lpfc_cmd->buf_lock); out_unlock: spin_unlock_irqrestore(&phba->hbalock, flags); out: @@ -5066,7 +4764,7 @@ lpfc_taskmgmt_name(uint8_t task_mgmt_cmd) /** * lpfc_check_fcp_rsp - check the returned fcp_rsp to see if task failed * @vport: The virtual port for which this call is being executed. - * @lpfc_cmd: Pointer to lpfc_scsi_buf data structure. + * @lpfc_cmd: Pointer to lpfc_io_buf data structure. * * This routine checks the FCP RSP INFO to see if the tsk mgmt command succeded * @@ -5075,7 +4773,7 @@ lpfc_taskmgmt_name(uint8_t task_mgmt_cmd) * 0x2002 - Success **/ static int -lpfc_check_fcp_rsp(struct lpfc_vport *vport, struct lpfc_scsi_buf *lpfc_cmd) +lpfc_check_fcp_rsp(struct lpfc_vport *vport, struct lpfc_io_buf *lpfc_cmd) { struct fcp_rsp *fcprsp = lpfc_cmd->fcp_rsp; uint32_t rsp_info; @@ -5150,7 +4848,7 @@ lpfc_send_taskmgmt(struct lpfc_vport *vport, struct scsi_cmnd *cmnd, uint8_t task_mgmt_cmd) { struct lpfc_hba *phba = vport->phba; - struct lpfc_scsi_buf *lpfc_cmd; + struct lpfc_io_buf *lpfc_cmd; struct lpfc_iocbq *iocbq; struct lpfc_iocbq *iocbqrsp; struct lpfc_rport_data *rdata; @@ -5163,7 +4861,7 @@ lpfc_send_taskmgmt(struct lpfc_vport *vport, struct scsi_cmnd *cmnd, return FAILED; pnode = rdata->pnode; - lpfc_cmd = lpfc_get_scsi_buf(phba, pnode); + lpfc_cmd = lpfc_get_scsi_buf(phba, pnode, NULL); if (lpfc_cmd == NULL) return FAILED; lpfc_cmd->timeout = phba->cfg_task_mgmt_tmo; @@ -5671,6 +5369,12 @@ lpfc_slave_alloc(struct scsi_device *sdev) } sdev_cnt = atomic_inc_return(&phba->sdev_cnt); + /* For SLI4, all IO buffers are pre-allocated */ + if (phba->sli_rev == LPFC_SLI_REV4) + return 0; + + /* This code path is now ONLY for SLI3 adapters */ + /* * Populate the cmds_per_lun count scsi_bufs into this host's globally * available list of scsi buffers. Don't allocate more than the @@ -5702,7 +5406,7 @@ lpfc_slave_alloc(struct scsi_device *sdev) (phba->cfg_hba_queue_depth - total)); num_to_alloc = phba->cfg_hba_queue_depth - total; } - num_allocated = lpfc_new_scsi_buf(vport, num_to_alloc); + num_allocated = lpfc_new_scsi_buf_s3(vport, num_to_alloc); if (num_to_alloc != num_allocated) { lpfc_printf_vlog(vport, KERN_ERR, LOG_FCP, "0708 Allocation request of %d " diff --git a/drivers/scsi/lpfc/lpfc_scsi.h b/drivers/scsi/lpfc/lpfc_scsi.h index b759b089432c..f76667b7da7b 100644 --- a/drivers/scsi/lpfc/lpfc_scsi.h +++ b/drivers/scsi/lpfc/lpfc_scsi.h @@ -1,7 +1,7 @@ /******************************************************************* * This file is part of the Emulex Linux Device Driver for * * Fibre Channel Host Bus Adapters. * - * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term * + * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term * * “Broadcom” refers to Broadcom Inc and/or its subsidiaries. * * Copyright (C) 2004-2016 Emulex. All rights reserved. * * EMULEX and SLI are trademarks of Emulex. * @@ -130,62 +130,6 @@ struct lpfc_scsicmd_bkt { uint32_t cmd_count; }; -struct lpfc_scsi_buf { - struct list_head list; - struct scsi_cmnd *pCmd; - struct lpfc_rport_data *rdata; - struct lpfc_nodelist *ndlp; - - uint32_t timeout; - - uint16_t flags; /* TBD convert exch_busy to flags */ -#define LPFC_SBUF_XBUSY 0x1 /* SLI4 hba reported XB on WCQE cmpl */ -#define LPFC_SBUF_BUMP_QDEPTH 0x8 /* bumped queue depth counter */ - uint16_t exch_busy; /* SLI4 hba reported XB on complete WCQE */ - uint16_t status; /* From IOCB Word 7- ulpStatus */ - uint32_t result; /* From IOCB Word 4. */ - - uint32_t seg_cnt; /* Number of scatter-gather segments returned by - * dma_map_sg. The driver needs this for calls - * to dma_unmap_sg. */ - uint32_t prot_seg_cnt; /* seg_cnt's counterpart for protection data */ - - dma_addr_t nonsg_phys; /* Non scatter-gather physical address. */ - - /* - * data and dma_handle are the kernel virtual and bus address of the - * dma-able buffer containing the fcp_cmd, fcp_rsp and a scatter - * gather bde list that supports the sg_tablesize value. - */ - void *data; - dma_addr_t dma_handle; - - struct fcp_cmnd *fcp_cmnd; - struct fcp_rsp *fcp_rsp; - struct ulp_bde64 *fcp_bpl; - - dma_addr_t dma_phys_bpl; - - /* cur_iocbq has phys of the dma-able buffer. - * Iotag is in here - */ - struct lpfc_iocbq cur_iocbq; - uint16_t cpu; - - wait_queue_head_t *waitq; - unsigned long start_time; - -#ifdef CONFIG_SCSI_LPFC_DEBUG_FS - /* Used to restore any changes to protection data for error injection */ - void *prot_data_segment; - uint32_t prot_data; - uint32_t prot_data_type; -#define LPFC_INJERR_REFTAG 1 -#define LPFC_INJERR_APPTAG 2 -#define LPFC_INJERR_GUARD 3 -#endif -}; - #define LPFC_SCSI_DMA_EXT_SIZE 264 #define LPFC_BPL_SIZE 1024 #define MDAC_DIRECT_CMD 0x22 @@ -200,5 +144,6 @@ struct lpfc_scsi_buf { #define TXRDY_PAYLOAD_LEN 12 -int lpfc_sli4_scmd_to_wqidx_distr(struct lpfc_hba *phba, - struct lpfc_scsi_buf *lpfc_cmd); +/* For sysfs/debugfs tmp string max len */ +#define LPFC_MAX_SCSI_INFO_TMP_LEN 79 + diff --git a/drivers/scsi/lpfc/lpfc_sli.c b/drivers/scsi/lpfc/lpfc_sli.c index 2242e9b3ca12..d0817facdae3 100644 --- a/drivers/scsi/lpfc/lpfc_sli.c +++ b/drivers/scsi/lpfc/lpfc_sli.c @@ -1,7 +1,7 @@ /******************************************************************* * This file is part of the Emulex Linux Device Driver for * * Fibre Channel Host Bus Adapters. * - * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term * + * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term * * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * * Copyright (C) 2004-2016 Emulex. All rights reserved. * * EMULEX and SLI are trademarks of Emulex. * @@ -78,12 +78,13 @@ static void lpfc_sli4_send_seq_to_ulp(struct lpfc_vport *, struct hbq_dmabuf *); static void lpfc_sli4_handle_mds_loopback(struct lpfc_vport *vport, struct hbq_dmabuf *dmabuf); -static int lpfc_sli4_fp_handle_cqe(struct lpfc_hba *, struct lpfc_queue *, - struct lpfc_cqe *); +static bool lpfc_sli4_fp_handle_cqe(struct lpfc_hba *phba, + struct lpfc_queue *cq, struct lpfc_cqe *cqe); static int lpfc_sli4_post_sgl_list(struct lpfc_hba *, struct list_head *, int); static void lpfc_sli4_hba_handle_eqe(struct lpfc_hba *phba, - struct lpfc_eqe *eqe, uint32_t qidx); + struct lpfc_queue *eq, + struct lpfc_eqe *eqe); static bool lpfc_sli4_mbox_completions_pending(struct lpfc_hba *phba); static bool lpfc_sli4_process_missed_mbox_completions(struct lpfc_hba *phba); static int lpfc_sli4_abort_nvme_io(struct lpfc_hba *phba, @@ -160,7 +161,7 @@ lpfc_sli4_wq_put(struct lpfc_queue *q, union lpfc_wqe128 *wqe) } q->WQ_posted++; /* set consumption flag every once in a while */ - if (!((q->host_index + 1) % q->entry_repost)) + if (!((q->host_index + 1) % q->notify_interval)) bf_set(wqe_wqec, &wqe->generic.wqe_com, 1); else bf_set(wqe_wqec, &wqe->generic.wqe_com, 0); @@ -325,29 +326,16 @@ lpfc_sli4_mq_release(struct lpfc_queue *q) static struct lpfc_eqe * lpfc_sli4_eq_get(struct lpfc_queue *q) { - struct lpfc_hba *phba; struct lpfc_eqe *eqe; - uint32_t idx; /* sanity check on queue memory */ if (unlikely(!q)) return NULL; - phba = q->phba; - eqe = q->qe[q->hba_index].eqe; + eqe = q->qe[q->host_index].eqe; /* If the next EQE is not valid then we are done */ if (bf_get_le32(lpfc_eqe_valid, eqe) != q->qe_valid) return NULL; - /* If the host has not yet processed the next entry then we are done */ - idx = ((q->hba_index + 1) % q->entry_count); - if (idx == q->host_index) - return NULL; - - q->hba_index = idx; - /* if the index wrapped around, toggle the valid bit */ - if (phba->sli4_hba.pc_sli4_params.eqav && !q->hba_index) - q->qe_valid = (q->qe_valid) ? 0 : 1; - /* * insert barrier for instruction interlock : data from the hardware @@ -397,44 +385,25 @@ lpfc_sli4_if6_eq_clr_intr(struct lpfc_queue *q) } /** - * lpfc_sli4_eq_release - Indicates the host has finished processing an EQ + * lpfc_sli4_write_eq_db - write EQ DB for eqe's consumed or arm state + * @phba: adapter with EQ * @q: The Event Queue that the host has completed processing for. + * @count: Number of elements that have been consumed * @arm: Indicates whether the host wants to arms this CQ. * - * This routine will mark all Event Queue Entries on @q, from the last - * known completed entry to the last entry that was processed, as completed - * by clearing the valid bit for each completion queue entry. Then it will - * notify the HBA, by ringing the doorbell, that the EQEs have been processed. - * The internal host index in the @q will be updated by this routine to indicate - * that the host has finished processing the entries. The @arm parameter - * indicates that the queue should be rearmed when ringing the doorbell. - * - * This function will return the number of EQEs that were popped. + * This routine will notify the HBA, by ringing the doorbell, that count + * number of EQEs have been processed. The @arm parameter indicates whether + * the queue should be rearmed when ringing the doorbell. **/ -uint32_t -lpfc_sli4_eq_release(struct lpfc_queue *q, bool arm) +void +lpfc_sli4_write_eq_db(struct lpfc_hba *phba, struct lpfc_queue *q, + uint32_t count, bool arm) { - uint32_t released = 0; - struct lpfc_hba *phba; - struct lpfc_eqe *temp_eqe; struct lpfc_register doorbell; /* sanity check on queue memory */ - if (unlikely(!q)) - return 0; - phba = q->phba; - - /* while there are valid entries */ - while (q->hba_index != q->host_index) { - if (!phba->sli4_hba.pc_sli4_params.eqav) { - temp_eqe = q->qe[q->host_index].eqe; - bf_set_le32(lpfc_eqe_valid, temp_eqe, 0); - } - released++; - q->host_index = ((q->host_index + 1) % q->entry_count); - } - if (unlikely(released == 0 && !arm)) - return 0; + if (unlikely(!q || (count == 0 && !arm))) + return; /* ring doorbell for number popped */ doorbell.word0 = 0; @@ -442,7 +411,7 @@ lpfc_sli4_eq_release(struct lpfc_queue *q, bool arm) bf_set(lpfc_eqcq_doorbell_arm, &doorbell, 1); bf_set(lpfc_eqcq_doorbell_eqci, &doorbell, 1); } - bf_set(lpfc_eqcq_doorbell_num_released, &doorbell, released); + bf_set(lpfc_eqcq_doorbell_num_released, &doorbell, count); bf_set(lpfc_eqcq_doorbell_qt, &doorbell, LPFC_QUEUE_TYPE_EVENT); bf_set(lpfc_eqcq_doorbell_eqid_hi, &doorbell, (q->queue_id >> LPFC_EQID_HI_FIELD_SHIFT)); @@ -451,60 +420,112 @@ lpfc_sli4_eq_release(struct lpfc_queue *q, bool arm) /* PCI read to flush PCI pipeline on re-arming for INTx mode */ if ((q->phba->intr_type == INTx) && (arm == LPFC_QUEUE_REARM)) readl(q->phba->sli4_hba.EQDBregaddr); - return released; } /** - * lpfc_sli4_if6_eq_release - Indicates the host has finished processing an EQ + * lpfc_sli4_if6_write_eq_db - write EQ DB for eqe's consumed or arm state + * @phba: adapter with EQ * @q: The Event Queue that the host has completed processing for. + * @count: Number of elements that have been consumed * @arm: Indicates whether the host wants to arms this CQ. * - * This routine will mark all Event Queue Entries on @q, from the last - * known completed entry to the last entry that was processed, as completed - * by clearing the valid bit for each completion queue entry. Then it will - * notify the HBA, by ringing the doorbell, that the EQEs have been processed. - * The internal host index in the @q will be updated by this routine to indicate - * that the host has finished processing the entries. The @arm parameter - * indicates that the queue should be rearmed when ringing the doorbell. - * - * This function will return the number of EQEs that were popped. + * This routine will notify the HBA, by ringing the doorbell, that count + * number of EQEs have been processed. The @arm parameter indicates whether + * the queue should be rearmed when ringing the doorbell. **/ -uint32_t -lpfc_sli4_if6_eq_release(struct lpfc_queue *q, bool arm) +void +lpfc_sli4_if6_write_eq_db(struct lpfc_hba *phba, struct lpfc_queue *q, + uint32_t count, bool arm) { - uint32_t released = 0; - struct lpfc_hba *phba; - struct lpfc_eqe *temp_eqe; struct lpfc_register doorbell; /* sanity check on queue memory */ - if (unlikely(!q)) - return 0; - phba = q->phba; - - /* while there are valid entries */ - while (q->hba_index != q->host_index) { - if (!phba->sli4_hba.pc_sli4_params.eqav) { - temp_eqe = q->qe[q->host_index].eqe; - bf_set_le32(lpfc_eqe_valid, temp_eqe, 0); - } - released++; - q->host_index = ((q->host_index + 1) % q->entry_count); - } - if (unlikely(released == 0 && !arm)) - return 0; + if (unlikely(!q || (count == 0 && !arm))) + return; /* ring doorbell for number popped */ doorbell.word0 = 0; if (arm) bf_set(lpfc_if6_eq_doorbell_arm, &doorbell, 1); - bf_set(lpfc_if6_eq_doorbell_num_released, &doorbell, released); + bf_set(lpfc_if6_eq_doorbell_num_released, &doorbell, count); bf_set(lpfc_if6_eq_doorbell_eqid, &doorbell, q->queue_id); writel(doorbell.word0, q->phba->sli4_hba.EQDBregaddr); /* PCI read to flush PCI pipeline on re-arming for INTx mode */ if ((q->phba->intr_type == INTx) && (arm == LPFC_QUEUE_REARM)) readl(q->phba->sli4_hba.EQDBregaddr); - return released; +} + +static void +__lpfc_sli4_consume_eqe(struct lpfc_hba *phba, struct lpfc_queue *eq, + struct lpfc_eqe *eqe) +{ + if (!phba->sli4_hba.pc_sli4_params.eqav) + bf_set_le32(lpfc_eqe_valid, eqe, 0); + + eq->host_index = ((eq->host_index + 1) % eq->entry_count); + + /* if the index wrapped around, toggle the valid bit */ + if (phba->sli4_hba.pc_sli4_params.eqav && !eq->host_index) + eq->qe_valid = (eq->qe_valid) ? 0 : 1; +} + +static void +lpfc_sli4_eq_flush(struct lpfc_hba *phba, struct lpfc_queue *eq) +{ + struct lpfc_eqe *eqe; + uint32_t count = 0; + + /* walk all the EQ entries and drop on the floor */ + eqe = lpfc_sli4_eq_get(eq); + while (eqe) { + __lpfc_sli4_consume_eqe(phba, eq, eqe); + count++; + eqe = lpfc_sli4_eq_get(eq); + } + + /* Clear and re-arm the EQ */ + phba->sli4_hba.sli4_write_eq_db(phba, eq, count, LPFC_QUEUE_REARM); +} + +static int +lpfc_sli4_process_eq(struct lpfc_hba *phba, struct lpfc_queue *eq) +{ + struct lpfc_eqe *eqe; + int count = 0, consumed = 0; + + if (cmpxchg(&eq->queue_claimed, 0, 1) != 0) + goto rearm_and_exit; + + eqe = lpfc_sli4_eq_get(eq); + while (eqe) { + lpfc_sli4_hba_handle_eqe(phba, eq, eqe); + __lpfc_sli4_consume_eqe(phba, eq, eqe); + + consumed++; + if (!(++count % eq->max_proc_limit)) + break; + + if (!(count % eq->notify_interval)) { + phba->sli4_hba.sli4_write_eq_db(phba, eq, consumed, + LPFC_QUEUE_NOARM); + consumed = 0; + } + + eqe = lpfc_sli4_eq_get(eq); + } + eq->EQ_processed += count; + + /* Track the max number of EQEs processed in 1 intr */ + if (count > eq->EQ_max_eqe) + eq->EQ_max_eqe = count; + + eq->queue_claimed = 0; + +rearm_and_exit: + /* Always clear and re-arm the EQ */ + phba->sli4_hba.sli4_write_eq_db(phba, eq, consumed, LPFC_QUEUE_REARM); + + return count; } /** @@ -519,28 +540,16 @@ lpfc_sli4_if6_eq_release(struct lpfc_queue *q, bool arm) static struct lpfc_cqe * lpfc_sli4_cq_get(struct lpfc_queue *q) { - struct lpfc_hba *phba; struct lpfc_cqe *cqe; - uint32_t idx; /* sanity check on queue memory */ if (unlikely(!q)) return NULL; - phba = q->phba; - cqe = q->qe[q->hba_index].cqe; + cqe = q->qe[q->host_index].cqe; /* If the next CQE is not valid then we are done */ if (bf_get_le32(lpfc_cqe_valid, cqe) != q->qe_valid) return NULL; - /* If the host has not yet processed the next entry then we are done */ - idx = ((q->hba_index + 1) % q->entry_count); - if (idx == q->host_index) - return NULL; - - q->hba_index = idx; - /* if the index wrapped around, toggle the valid bit */ - if (phba->sli4_hba.pc_sli4_params.cqav && !q->hba_index) - q->qe_valid = (q->qe_valid) ? 0 : 1; /* * insert barrier for instruction interlock : data from the hardware @@ -554,107 +563,81 @@ lpfc_sli4_cq_get(struct lpfc_queue *q) return cqe; } +static void +__lpfc_sli4_consume_cqe(struct lpfc_hba *phba, struct lpfc_queue *cq, + struct lpfc_cqe *cqe) +{ + if (!phba->sli4_hba.pc_sli4_params.cqav) + bf_set_le32(lpfc_cqe_valid, cqe, 0); + + cq->host_index = ((cq->host_index + 1) % cq->entry_count); + + /* if the index wrapped around, toggle the valid bit */ + if (phba->sli4_hba.pc_sli4_params.cqav && !cq->host_index) + cq->qe_valid = (cq->qe_valid) ? 0 : 1; +} + /** - * lpfc_sli4_cq_release - Indicates the host has finished processing a CQ + * lpfc_sli4_write_cq_db - write cq DB for entries consumed or arm state. + * @phba: the adapter with the CQ * @q: The Completion Queue that the host has completed processing for. + * @count: the number of elements that were consumed * @arm: Indicates whether the host wants to arms this CQ. * - * This routine will mark all Completion queue entries on @q, from the last - * known completed entry to the last entry that was processed, as completed - * by clearing the valid bit for each completion queue entry. Then it will - * notify the HBA, by ringing the doorbell, that the CQEs have been processed. - * The internal host index in the @q will be updated by this routine to indicate - * that the host has finished processing the entries. The @arm parameter - * indicates that the queue should be rearmed when ringing the doorbell. - * - * This function will return the number of CQEs that were released. + * This routine will notify the HBA, by ringing the doorbell, that the + * CQEs have been processed. The @arm parameter specifies whether the + * queue should be rearmed when ringing the doorbell. **/ -uint32_t -lpfc_sli4_cq_release(struct lpfc_queue *q, bool arm) +void +lpfc_sli4_write_cq_db(struct lpfc_hba *phba, struct lpfc_queue *q, + uint32_t count, bool arm) { - uint32_t released = 0; - struct lpfc_hba *phba; - struct lpfc_cqe *temp_qe; struct lpfc_register doorbell; /* sanity check on queue memory */ - if (unlikely(!q)) - return 0; - phba = q->phba; - - /* while there are valid entries */ - while (q->hba_index != q->host_index) { - if (!phba->sli4_hba.pc_sli4_params.cqav) { - temp_qe = q->qe[q->host_index].cqe; - bf_set_le32(lpfc_cqe_valid, temp_qe, 0); - } - released++; - q->host_index = ((q->host_index + 1) % q->entry_count); - } - if (unlikely(released == 0 && !arm)) - return 0; + if (unlikely(!q || (count == 0 && !arm))) + return; /* ring doorbell for number popped */ doorbell.word0 = 0; if (arm) bf_set(lpfc_eqcq_doorbell_arm, &doorbell, 1); - bf_set(lpfc_eqcq_doorbell_num_released, &doorbell, released); + bf_set(lpfc_eqcq_doorbell_num_released, &doorbell, count); bf_set(lpfc_eqcq_doorbell_qt, &doorbell, LPFC_QUEUE_TYPE_COMPLETION); bf_set(lpfc_eqcq_doorbell_cqid_hi, &doorbell, (q->queue_id >> LPFC_CQID_HI_FIELD_SHIFT)); bf_set(lpfc_eqcq_doorbell_cqid_lo, &doorbell, q->queue_id); writel(doorbell.word0, q->phba->sli4_hba.CQDBregaddr); - return released; } /** - * lpfc_sli4_if6_cq_release - Indicates the host has finished processing a CQ + * lpfc_sli4_if6_write_cq_db - write cq DB for entries consumed or arm state. + * @phba: the adapter with the CQ * @q: The Completion Queue that the host has completed processing for. + * @count: the number of elements that were consumed * @arm: Indicates whether the host wants to arms this CQ. * - * This routine will mark all Completion queue entries on @q, from the last - * known completed entry to the last entry that was processed, as completed - * by clearing the valid bit for each completion queue entry. Then it will - * notify the HBA, by ringing the doorbell, that the CQEs have been processed. - * The internal host index in the @q will be updated by this routine to indicate - * that the host has finished processing the entries. The @arm parameter - * indicates that the queue should be rearmed when ringing the doorbell. - * - * This function will return the number of CQEs that were released. + * This routine will notify the HBA, by ringing the doorbell, that the + * CQEs have been processed. The @arm parameter specifies whether the + * queue should be rearmed when ringing the doorbell. **/ -uint32_t -lpfc_sli4_if6_cq_release(struct lpfc_queue *q, bool arm) +void +lpfc_sli4_if6_write_cq_db(struct lpfc_hba *phba, struct lpfc_queue *q, + uint32_t count, bool arm) { - uint32_t released = 0; - struct lpfc_hba *phba; - struct lpfc_cqe *temp_qe; struct lpfc_register doorbell; /* sanity check on queue memory */ - if (unlikely(!q)) - return 0; - phba = q->phba; - - /* while there are valid entries */ - while (q->hba_index != q->host_index) { - if (!phba->sli4_hba.pc_sli4_params.cqav) { - temp_qe = q->qe[q->host_index].cqe; - bf_set_le32(lpfc_cqe_valid, temp_qe, 0); - } - released++; - q->host_index = ((q->host_index + 1) % q->entry_count); - } - if (unlikely(released == 0 && !arm)) - return 0; + if (unlikely(!q || (count == 0 && !arm))) + return; /* ring doorbell for number popped */ doorbell.word0 = 0; if (arm) bf_set(lpfc_if6_cq_doorbell_arm, &doorbell, 1); - bf_set(lpfc_if6_cq_doorbell_num_released, &doorbell, released); + bf_set(lpfc_if6_cq_doorbell_num_released, &doorbell, count); bf_set(lpfc_if6_cq_doorbell_cqid, &doorbell, q->queue_id); writel(doorbell.word0, q->phba->sli4_hba.CQDBregaddr); - return released; } /** @@ -703,15 +686,15 @@ lpfc_sli4_rq_put(struct lpfc_queue *hq, struct lpfc_queue *dq, hq->RQ_buf_posted++; /* Ring The Header Receive Queue Doorbell */ - if (!(hq->host_index % hq->entry_repost)) { + if (!(hq->host_index % hq->notify_interval)) { doorbell.word0 = 0; if (hq->db_format == LPFC_DB_RING_FORMAT) { bf_set(lpfc_rq_db_ring_fm_num_posted, &doorbell, - hq->entry_repost); + hq->notify_interval); bf_set(lpfc_rq_db_ring_fm_id, &doorbell, hq->queue_id); } else if (hq->db_format == LPFC_DB_LIST_FORMAT) { bf_set(lpfc_rq_db_list_fm_num_posted, &doorbell, - hq->entry_repost); + hq->notify_interval); bf_set(lpfc_rq_db_list_fm_index, &doorbell, hq->host_index); bf_set(lpfc_rq_db_list_fm_id, &doorbell, hq->queue_id); @@ -1025,7 +1008,7 @@ lpfc_test_rrq_active(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, if (!ndlp->active_rrqs_xri_bitmap) return 0; if (test_bit(xritag, ndlp->active_rrqs_xri_bitmap)) - return 1; + return 1; else return 0; } @@ -1133,14 +1116,14 @@ __lpfc_sli_get_els_sglq(struct lpfc_hba *phba, struct lpfc_iocbq *piocbq) struct list_head *lpfc_els_sgl_list = &phba->sli4_hba.lpfc_els_sgl_list; struct lpfc_sglq *sglq = NULL; struct lpfc_sglq *start_sglq = NULL; - struct lpfc_scsi_buf *lpfc_cmd; + struct lpfc_io_buf *lpfc_cmd; struct lpfc_nodelist *ndlp; int found = 0; lockdep_assert_held(&phba->hbalock); if (piocbq->iocb_flag & LPFC_IO_FCP) { - lpfc_cmd = (struct lpfc_scsi_buf *) piocbq->context1; + lpfc_cmd = (struct lpfc_io_buf *) piocbq->context1; ndlp = lpfc_cmd->rdata->pnode; } else if ((piocbq->iocb.ulpCommand == CMD_GEN_REQUEST64_CR) && !(piocbq->iocb_flag & LPFC_IO_LIBDFC)) { @@ -1596,6 +1579,7 @@ lpfc_sli_ringtxcmpl_put(struct lpfc_hba *phba, struct lpfc_sli_ring *pring, list_add_tail(&piocb->list, &pring->txcmplq); piocb->iocb_flag |= LPFC_IO_ON_TXCMPLQ; + pring->txcmplq_cnt++; if ((unlikely(pring->ringno == LPFC_ELS_RING)) && (piocb->iocb.ulpCommand != CMD_ABORT_XRI_CN) && @@ -3008,6 +2992,7 @@ lpfc_sli_iocbq_lookup(struct lpfc_hba *phba, /* remove from txcmpl queue list */ list_del_init(&cmd_iocb->list); cmd_iocb->iocb_flag &= ~LPFC_IO_ON_TXCMPLQ; + pring->txcmplq_cnt--; return cmd_iocb; } } @@ -3045,6 +3030,7 @@ lpfc_sli_iocbq_lookup_by_tag(struct lpfc_hba *phba, /* remove from txcmpl queue list */ list_del_init(&cmd_iocb->list); cmd_iocb->iocb_flag &= ~LPFC_IO_ON_TXCMPLQ; + pring->txcmplq_cnt--; return cmd_iocb; } } @@ -3981,8 +3967,8 @@ lpfc_sli_abort_fcp_rings(struct lpfc_hba *phba) /* Look on all the FCP Rings for the iotag */ if (phba->sli_rev >= LPFC_SLI_REV4) { - for (i = 0; i < phba->cfg_fcp_io_channel; i++) { - pring = phba->sli4_hba.fcp_wq[i]->pring; + for (i = 0; i < phba->cfg_hdw_queue; i++) { + pring = phba->sli4_hba.hdwq[i].fcp_wq->pring; lpfc_sli_abort_iocb_ring(phba, pring); } } else { @@ -4006,12 +3992,13 @@ lpfc_sli_abort_nvme_rings(struct lpfc_hba *phba) struct lpfc_sli_ring *pring; uint32_t i; - if (phba->sli_rev < LPFC_SLI_REV4) + if ((phba->sli_rev < LPFC_SLI_REV4) || + !(phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME)) return; /* Abort all IO on each NVME ring. */ - for (i = 0; i < phba->cfg_nvme_io_channel; i++) { - pring = phba->sli4_hba.nvme_wq[i]->pring; + for (i = 0; i < phba->cfg_hdw_queue; i++) { + pring = phba->sli4_hba.hdwq[i].nvme_wq->pring; lpfc_sli_abort_wqe_ring(phba, pring); } } @@ -4044,8 +4031,8 @@ lpfc_sli_flush_fcp_rings(struct lpfc_hba *phba) /* Look on all the FCP Rings for the iotag */ if (phba->sli_rev >= LPFC_SLI_REV4) { - for (i = 0; i < phba->cfg_fcp_io_channel; i++) { - pring = phba->sli4_hba.fcp_wq[i]->pring; + for (i = 0; i < phba->cfg_hdw_queue; i++) { + pring = phba->sli4_hba.hdwq[i].fcp_wq->pring; spin_lock_irq(&pring->ring_lock); /* Retrieve everything on txq */ @@ -4110,7 +4097,8 @@ lpfc_sli_flush_nvme_rings(struct lpfc_hba *phba) uint32_t i; struct lpfc_iocbq *piocb, *next_iocb; - if (phba->sli_rev < LPFC_SLI_REV4) + if ((phba->sli_rev < LPFC_SLI_REV4) || + !(phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME)) return; /* Hint to other driver operations that a flush is in progress. */ @@ -4122,8 +4110,8 @@ lpfc_sli_flush_nvme_rings(struct lpfc_hba *phba) * a local driver reason code. This is a flush so no * abort exchange to FW. */ - for (i = 0; i < phba->cfg_nvme_io_channel; i++) { - pring = phba->sli4_hba.nvme_wq[i]->pring; + for (i = 0; i < phba->cfg_hdw_queue; i++) { + pring = phba->sli4_hba.hdwq[i].nvme_wq->pring; spin_lock_irq(&pring->ring_lock); list_for_each_entry_safe(piocb, next_iocb, @@ -5564,41 +5552,35 @@ lpfc_sli4_arm_cqeq_intr(struct lpfc_hba *phba) { int qidx; struct lpfc_sli4_hba *sli4_hba = &phba->sli4_hba; + struct lpfc_sli4_hdw_queue *qp; - sli4_hba->sli4_cq_release(sli4_hba->mbx_cq, LPFC_QUEUE_REARM); - sli4_hba->sli4_cq_release(sli4_hba->els_cq, LPFC_QUEUE_REARM); + sli4_hba->sli4_write_cq_db(phba, sli4_hba->mbx_cq, 0, LPFC_QUEUE_REARM); + sli4_hba->sli4_write_cq_db(phba, sli4_hba->els_cq, 0, LPFC_QUEUE_REARM); if (sli4_hba->nvmels_cq) - sli4_hba->sli4_cq_release(sli4_hba->nvmels_cq, - LPFC_QUEUE_REARM); + sli4_hba->sli4_write_cq_db(phba, sli4_hba->nvmels_cq, 0, + LPFC_QUEUE_REARM); - if (sli4_hba->fcp_cq) - for (qidx = 0; qidx < phba->cfg_fcp_io_channel; qidx++) - sli4_hba->sli4_cq_release(sli4_hba->fcp_cq[qidx], - LPFC_QUEUE_REARM); - - if (sli4_hba->nvme_cq) - for (qidx = 0; qidx < phba->cfg_nvme_io_channel; qidx++) - sli4_hba->sli4_cq_release(sli4_hba->nvme_cq[qidx], - LPFC_QUEUE_REARM); - - if (phba->cfg_fof) - sli4_hba->sli4_cq_release(sli4_hba->oas_cq, LPFC_QUEUE_REARM); + qp = sli4_hba->hdwq; + if (sli4_hba->hdwq) { + for (qidx = 0; qidx < phba->cfg_hdw_queue; qidx++) { + sli4_hba->sli4_write_cq_db(phba, qp[qidx].fcp_cq, 0, + LPFC_QUEUE_REARM); + sli4_hba->sli4_write_cq_db(phba, qp[qidx].nvme_cq, 0, + LPFC_QUEUE_REARM); + } - if (sli4_hba->hba_eq) - for (qidx = 0; qidx < phba->io_channel_irqs; qidx++) - sli4_hba->sli4_eq_release(sli4_hba->hba_eq[qidx], - LPFC_QUEUE_REARM); + for (qidx = 0; qidx < phba->cfg_irq_chann; qidx++) + sli4_hba->sli4_write_eq_db(phba, qp[qidx].hba_eq, + 0, LPFC_QUEUE_REARM); + } if (phba->nvmet_support) { for (qidx = 0; qidx < phba->cfg_nvmet_mrq; qidx++) { - sli4_hba->sli4_cq_release( - sli4_hba->nvmet_cqset[qidx], + sli4_hba->sli4_write_cq_db(phba, + sli4_hba->nvmet_cqset[qidx], 0, LPFC_QUEUE_REARM); } } - - if (phba->cfg_fof) - sli4_hba->sli4_eq_release(sli4_hba->fof_eq, LPFC_QUEUE_REARM); } /** @@ -6027,11 +6009,8 @@ lpfc_sli4_alloc_extent(struct lpfc_hba *phba, uint16_t type) list_add_tail(&rsrc_blks->list, ext_blk_list); rsrc_start = rsrc_id; if ((type == LPFC_RSC_TYPE_FCOE_XRI) && (j == 0)) { - phba->sli4_hba.scsi_xri_start = rsrc_start + + phba->sli4_hba.io_xri_start = rsrc_start + lpfc_sli4_get_iocb_cnt(phba); - phba->sli4_hba.nvme_xri_start = - phba->sli4_hba.scsi_xri_start + - phba->sli4_hba.scsi_xri_max; } while (rsrc_id < (rsrc_start + rsrc_size)) { @@ -7056,6 +7035,38 @@ lpfc_sli4_repost_sgl_list(struct lpfc_hba *phba, return total_cnt; } +/** + * lpfc_sli4_repost_io_sgl_list - Repost all the allocated nvme buffer sgls + * @phba: pointer to lpfc hba data structure. + * + * This routine walks the list of nvme buffers that have been allocated and + * repost them to the port by using SGL block post. This is needed after a + * pci_function_reset/warm_start or start. The lpfc_hba_down_post_s4 routine + * is responsible for moving all nvme buffers on the lpfc_abts_nvme_sgl_list + * to the lpfc_io_buf_list. If the repost fails, reject all nvme buffers. + * + * Returns: 0 = success, non-zero failure. + **/ +int +lpfc_sli4_repost_io_sgl_list(struct lpfc_hba *phba) +{ + LIST_HEAD(post_nblist); + int num_posted, rc = 0; + + /* get all NVME buffers need to repost to a local list */ + lpfc_io_buf_flush(phba, &post_nblist); + + /* post the list of nvme buffer sgls to port if available */ + if (!list_empty(&post_nblist)) { + num_posted = lpfc_sli4_post_io_sgl_list( + phba, &post_nblist, phba->sli4_hba.io_xri_cnt); + /* failed to post any nvme buffer, return error */ + if (num_posted == 0) + rc = -EIO; + } + return rc; +} + void lpfc_set_host_data(struct lpfc_hba *phba, LPFC_MBOXQ_t *mbox) { @@ -7144,7 +7155,7 @@ lpfc_post_rq_buffer(struct lpfc_hba *phba, struct lpfc_queue *hrq, int lpfc_sli4_hba_setup(struct lpfc_hba *phba) { - int rc, i, cnt; + int rc, i, cnt, len; LPFC_MBOXQ_t *mboxq; struct lpfc_mqe *mqe; uint8_t *vpd; @@ -7517,24 +7528,26 @@ lpfc_sli4_hba_setup(struct lpfc_hba *phba) /* We need 1 iocbq for every SGL, for IO processing */ cnt += phba->sli4_hba.nvmet_xri_cnt; } else { - /* update host scsi xri-sgl sizes and mappings */ - rc = lpfc_sli4_scsi_sgl_update(phba); + /* update host common xri-sgl sizes and mappings */ + rc = lpfc_sli4_io_sgl_update(phba); if (unlikely(rc)) { lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI, - "6309 Failed to update scsi-sgl size " + "6082 Failed to update nvme-sgl size " "and mapping: %d\n", rc); goto out_destroy_queue; } - /* update host nvme xri-sgl sizes and mappings */ - rc = lpfc_sli4_nvme_sgl_update(phba); + /* register the allocated common sgl pool to the port */ + rc = lpfc_sli4_repost_io_sgl_list(phba); if (unlikely(rc)) { lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI, - "6082 Failed to update nvme-sgl size " - "and mapping: %d\n", rc); + "6116 Error %d during nvme sgl post " + "operation\n", rc); + /* Some NVME buffers were moved to abort nvme list */ + /* A pci function reset will repost them */ + rc = -ENODEV; goto out_destroy_queue; } - cnt = phba->cfg_iocb_cnt * 1024; } @@ -7571,36 +7584,6 @@ lpfc_sli4_hba_setup(struct lpfc_hba *phba) } } - if (phba->cfg_enable_fc4_type & LPFC_ENABLE_FCP) { - /* register the allocated scsi sgl pool to the port */ - rc = lpfc_sli4_repost_scsi_sgl_list(phba); - if (unlikely(rc)) { - lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI, - "0383 Error %d during scsi sgl post " - "operation\n", rc); - /* Some Scsi buffers were moved to abort scsi list */ - /* A pci function reset will repost them */ - rc = -ENODEV; - goto out_destroy_queue; - } - } - - if ((phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) && - (phba->nvmet_support == 0)) { - - /* register the allocated nvme sgl pool to the port */ - rc = lpfc_repost_nvme_sgl_list(phba); - if (unlikely(rc)) { - lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI, - "6116 Error %d during nvme sgl post " - "operation\n", rc); - /* Some NVME buffers were moved to abort nvme list */ - /* A pci function reset will repost them */ - rc = -ENODEV; - goto out_destroy_queue; - } - } - /* Post the rpi header region to the device. */ rc = lpfc_sli4_post_all_rpi_hdrs(phba); if (unlikely(rc)) { @@ -7650,6 +7633,25 @@ lpfc_sli4_hba_setup(struct lpfc_hba *phba) lpfc_sli_read_link_ste(phba); } + /* Don't post more new bufs if repost already recovered + * the nvme sgls. + */ + if (phba->nvmet_support == 0) { + if (phba->sli4_hba.io_xri_cnt == 0) { + len = lpfc_new_io_buf( + phba, phba->sli4_hba.io_xri_max); + if (len == 0) { + rc = -ENOMEM; + goto out_unset_queue; + } + + if (phba->cfg_xri_rebalancing) + lpfc_create_multixri_pools(phba); + } + } else { + phba->cfg_xri_rebalancing = 0; + } + /* Arm the CQs and then EQs on device */ lpfc_sli4_arm_cqeq_intr(phba); @@ -7678,6 +7680,11 @@ lpfc_sli4_hba_setup(struct lpfc_hba *phba) phba->hb_outstanding = 0; phba->last_completion_time = jiffies; + /* start eq_delay heartbeat */ + if (phba->cfg_auto_imax) + queue_delayed_work(phba->wq, &phba->eq_delay_work, + msecs_to_jiffies(LPFC_EQ_DELAY_MSECS)); + /* Start error attention (ERATT) polling timer */ mod_timer(&phba->eratt_poll, jiffies + msecs_to_jiffies(1000 * phba->eratt_poll_interval)); @@ -7729,18 +7736,21 @@ lpfc_sli4_hba_setup(struct lpfc_hba *phba) lpfc_printf_log(phba, KERN_ERR, LOG_INIT | LOG_SLI, "3104 Adapter failed to issue " "DOWN_LINK mbox cmd, rc:x%x\n", rc); - goto out_unset_queue; + goto out_io_buff_free; } } else if (phba->cfg_suppress_link_up == LPFC_INITIALIZE_LINK) { /* don't perform init_link on SLI4 FC port loopback test */ if (!(phba->link_flag & LS_LOOPBACK_MODE)) { rc = phba->lpfc_hba_init_link(phba, MBX_NOWAIT); if (rc) - goto out_unset_queue; + goto out_io_buff_free; } } mempool_free(mboxq, phba->mbox_mem_pool); return rc; +out_io_buff_free: + /* Free allocated IO Buffers */ + lpfc_io_free(phba); out_unset_queue: /* Unset all the queues set up in this routine when error out */ lpfc_sli4_queue_unset(phba); @@ -7846,7 +7856,6 @@ lpfc_sli4_process_missed_mbox_completions(struct lpfc_hba *phba) struct lpfc_sli4_hba *sli4_hba = &phba->sli4_hba; uint32_t eqidx; struct lpfc_queue *fpeq = NULL; - struct lpfc_eqe *eqe; bool mbox_pending; if (unlikely(!phba) || (phba->sli_rev != LPFC_SLI_REV4)) @@ -7854,11 +7863,11 @@ lpfc_sli4_process_missed_mbox_completions(struct lpfc_hba *phba) /* Find the eq associated with the mcq */ - if (sli4_hba->hba_eq) - for (eqidx = 0; eqidx < phba->io_channel_irqs; eqidx++) - if (sli4_hba->hba_eq[eqidx]->queue_id == + if (sli4_hba->hdwq) + for (eqidx = 0; eqidx < phba->cfg_irq_chann; eqidx++) + if (sli4_hba->hdwq[eqidx].hba_eq->queue_id == sli4_hba->mbx_cq->assoc_qid) { - fpeq = sli4_hba->hba_eq[eqidx]; + fpeq = sli4_hba->hdwq[eqidx].hba_eq; break; } if (!fpeq) @@ -7880,14 +7889,11 @@ lpfc_sli4_process_missed_mbox_completions(struct lpfc_hba *phba) */ if (mbox_pending) - while ((eqe = lpfc_sli4_eq_get(fpeq))) { - lpfc_sli4_hba_handle_eqe(phba, eqe, eqidx); - fpeq->EQ_processed++; - } - - /* Always clear and re-arm the EQ */ - - sli4_hba->sli4_eq_release(fpeq, LPFC_QUEUE_REARM); + /* process and rearm the EQ */ + lpfc_sli4_process_eq(phba, fpeq); + else + /* Always clear and re-arm the EQ */ + sli4_hba->sli4_write_eq_db(phba, fpeq, 0, LPFC_QUEUE_REARM); return mbox_pending; @@ -8557,7 +8563,6 @@ lpfc_sli4_post_sync_mbox(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq) rc = lpfc_sli4_wait_bmbx_ready(phba, mboxq); if (rc) goto exit; - /* * Initialize the bootstrap memory region to avoid stale data areas * in the mailbox post. Then copy the caller's mailbox contents to @@ -9476,7 +9481,7 @@ lpfc_sli4_iocb2wqe(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq, bf_set(wqe_pbde, &wqe->fcp_iwrite.wqe_com, 0); if (phba->fcp_embed_io) { - struct lpfc_scsi_buf *lpfc_cmd; + struct lpfc_io_buf *lpfc_cmd; struct sli4_sge *sgl; struct fcp_cmnd *fcp_cmnd; uint32_t *ptr; @@ -9484,7 +9489,7 @@ lpfc_sli4_iocb2wqe(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq, /* 128 byte wqe support here */ lpfc_cmd = iocbq->context1; - sgl = (struct sli4_sge *)lpfc_cmd->fcp_bpl; + sgl = (struct sli4_sge *)lpfc_cmd->dma_sgl; fcp_cmnd = lpfc_cmd->fcp_cmnd; /* Word 0-2 - FCP_CMND */ @@ -9540,7 +9545,7 @@ lpfc_sli4_iocb2wqe(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq, bf_set(wqe_pbde, &wqe->fcp_iread.wqe_com, 0); if (phba->fcp_embed_io) { - struct lpfc_scsi_buf *lpfc_cmd; + struct lpfc_io_buf *lpfc_cmd; struct sli4_sge *sgl; struct fcp_cmnd *fcp_cmnd; uint32_t *ptr; @@ -9548,7 +9553,7 @@ lpfc_sli4_iocb2wqe(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq, /* 128 byte wqe support here */ lpfc_cmd = iocbq->context1; - sgl = (struct sli4_sge *)lpfc_cmd->fcp_bpl; + sgl = (struct sli4_sge *)lpfc_cmd->dma_sgl; fcp_cmnd = lpfc_cmd->fcp_cmnd; /* Word 0-2 - FCP_CMND */ @@ -9597,7 +9602,7 @@ lpfc_sli4_iocb2wqe(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq, /* Note, word 10 is already initialized to 0 */ if (phba->fcp_embed_io) { - struct lpfc_scsi_buf *lpfc_cmd; + struct lpfc_io_buf *lpfc_cmd; struct sli4_sge *sgl; struct fcp_cmnd *fcp_cmnd; uint32_t *ptr; @@ -9605,7 +9610,7 @@ lpfc_sli4_iocb2wqe(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq, /* 128 byte wqe support here */ lpfc_cmd = iocbq->context1; - sgl = (struct sli4_sge *)lpfc_cmd->fcp_bpl; + sgl = (struct sli4_sge *)lpfc_cmd->dma_sgl; fcp_cmnd = lpfc_cmd->fcp_cmnd; /* Word 0-2 - FCP_CMND */ @@ -9864,10 +9869,7 @@ __lpfc_sli_issue_iocb_s4(struct lpfc_hba *phba, uint32_t ring_number, /* Get the WQ */ if ((piocb->iocb_flag & LPFC_IO_FCP) || (piocb->iocb_flag & LPFC_USE_FCPWQIDX)) { - if (!phba->cfg_fof || (!(piocb->iocb_flag & LPFC_IO_OAS))) - wq = phba->sli4_hba.fcp_wq[piocb->hba_wqidx]; - else - wq = phba->sli4_hba.oas_wq; + wq = phba->sli4_hba.hdwq[piocb->hba_wqidx].fcp_wq; } else { wq = phba->sli4_hba.els_wq; } @@ -10001,29 +10003,20 @@ lpfc_sli_api_table_setup(struct lpfc_hba *phba, uint8_t dev_grp) struct lpfc_sli_ring * lpfc_sli4_calc_ring(struct lpfc_hba *phba, struct lpfc_iocbq *piocb) { + struct lpfc_io_buf *lpfc_cmd; + if (piocb->iocb_flag & (LPFC_IO_FCP | LPFC_USE_FCPWQIDX)) { - if (!(phba->cfg_fof) || - (!(piocb->iocb_flag & LPFC_IO_FOF))) { - if (unlikely(!phba->sli4_hba.fcp_wq)) - return NULL; - /* - * for abort iocb hba_wqidx should already - * be setup based on what work queue we used. - */ - if (!(piocb->iocb_flag & LPFC_USE_FCPWQIDX)) { - piocb->hba_wqidx = - lpfc_sli4_scmd_to_wqidx_distr(phba, - piocb->context1); - piocb->hba_wqidx = piocb->hba_wqidx % - phba->cfg_fcp_io_channel; - } - return phba->sli4_hba.fcp_wq[piocb->hba_wqidx]->pring; - } else { - if (unlikely(!phba->sli4_hba.oas_wq)) - return NULL; - piocb->hba_wqidx = 0; - return phba->sli4_hba.oas_wq->pring; + if (unlikely(!phba->sli4_hba.hdwq)) + return NULL; + /* + * for abort iocb hba_wqidx should already + * be setup based on what work queue we used. + */ + if (!(piocb->iocb_flag & LPFC_USE_FCPWQIDX)) { + lpfc_cmd = (struct lpfc_io_buf *)piocb->context1; + piocb->hba_wqidx = lpfc_cmd->hdwq_no; } + return phba->sli4_hba.hdwq[piocb->hba_wqidx].fcp_wq->pring; } else { if (unlikely(!phba->sli4_hba.els_wq)) return NULL; @@ -10049,12 +10042,9 @@ int lpfc_sli_issue_iocb(struct lpfc_hba *phba, uint32_t ring_number, struct lpfc_iocbq *piocb, uint32_t flag) { - struct lpfc_hba_eq_hdl *hba_eq_hdl; struct lpfc_sli_ring *pring; - struct lpfc_queue *fpeq; - struct lpfc_eqe *eqe; unsigned long iflags; - int rc, idx; + int rc; if (phba->sli_rev == LPFC_SLI_REV4) { pring = lpfc_sli4_calc_ring(phba, piocb); @@ -10064,34 +10054,6 @@ lpfc_sli_issue_iocb(struct lpfc_hba *phba, uint32_t ring_number, spin_lock_irqsave(&pring->ring_lock, iflags); rc = __lpfc_sli_issue_iocb(phba, ring_number, piocb, flag); spin_unlock_irqrestore(&pring->ring_lock, iflags); - - if (lpfc_fcp_look_ahead && (piocb->iocb_flag & LPFC_IO_FCP)) { - idx = piocb->hba_wqidx; - hba_eq_hdl = &phba->sli4_hba.hba_eq_hdl[idx]; - - if (atomic_dec_and_test(&hba_eq_hdl->hba_eq_in_use)) { - - /* Get associated EQ with this index */ - fpeq = phba->sli4_hba.hba_eq[idx]; - - /* Turn off interrupts from this EQ */ - phba->sli4_hba.sli4_eq_clr_intr(fpeq); - - /* - * Process all the events on FCP EQ - */ - while ((eqe = lpfc_sli4_eq_get(fpeq))) { - lpfc_sli4_hba_handle_eqe(phba, - eqe, idx); - fpeq->EQ_processed++; - } - - /* Always clear and re-arm the EQ */ - phba->sli4_hba.sli4_eq_release(fpeq, - LPFC_QUEUE_REARM); - } - atomic_inc(&hba_eq_hdl->hba_eq_in_use); - } } else { /* For now, SLI2/3 will still use hbalock */ spin_lock_irqsave(&phba->hbalock, iflags); @@ -10506,19 +10468,11 @@ lpfc_sli4_queue_init(struct lpfc_hba *phba) INIT_LIST_HEAD(&psli->mboxq); INIT_LIST_HEAD(&psli->mboxq_cmpl); /* Initialize list headers for txq and txcmplq as double linked lists */ - for (i = 0; i < phba->cfg_fcp_io_channel; i++) { - pring = phba->sli4_hba.fcp_wq[i]->pring; - pring->flag = 0; - pring->ringno = LPFC_FCP_RING; - INIT_LIST_HEAD(&pring->txq); - INIT_LIST_HEAD(&pring->txcmplq); - INIT_LIST_HEAD(&pring->iocb_continueq); - spin_lock_init(&pring->ring_lock); - } - for (i = 0; i < phba->cfg_nvme_io_channel; i++) { - pring = phba->sli4_hba.nvme_wq[i]->pring; + for (i = 0; i < phba->cfg_hdw_queue; i++) { + pring = phba->sli4_hba.hdwq[i].fcp_wq->pring; pring->flag = 0; pring->ringno = LPFC_FCP_RING; + pring->txcmplq_cnt = 0; INIT_LIST_HEAD(&pring->txq); INIT_LIST_HEAD(&pring->txcmplq); INIT_LIST_HEAD(&pring->iocb_continueq); @@ -10527,25 +10481,27 @@ lpfc_sli4_queue_init(struct lpfc_hba *phba) pring = phba->sli4_hba.els_wq->pring; pring->flag = 0; pring->ringno = LPFC_ELS_RING; + pring->txcmplq_cnt = 0; INIT_LIST_HEAD(&pring->txq); INIT_LIST_HEAD(&pring->txcmplq); INIT_LIST_HEAD(&pring->iocb_continueq); spin_lock_init(&pring->ring_lock); - if (phba->cfg_nvme_io_channel) { + if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { + for (i = 0; i < phba->cfg_hdw_queue; i++) { + pring = phba->sli4_hba.hdwq[i].nvme_wq->pring; + pring->flag = 0; + pring->ringno = LPFC_FCP_RING; + pring->txcmplq_cnt = 0; + INIT_LIST_HEAD(&pring->txq); + INIT_LIST_HEAD(&pring->txcmplq); + INIT_LIST_HEAD(&pring->iocb_continueq); + spin_lock_init(&pring->ring_lock); + } pring = phba->sli4_hba.nvmels_wq->pring; pring->flag = 0; pring->ringno = LPFC_ELS_RING; - INIT_LIST_HEAD(&pring->txq); - INIT_LIST_HEAD(&pring->txcmplq); - INIT_LIST_HEAD(&pring->iocb_continueq); - spin_lock_init(&pring->ring_lock); - } - - if (phba->cfg_fof) { - pring = phba->sli4_hba.oas_wq->pring; - pring->flag = 0; - pring->ringno = LPFC_FCP_RING; + pring->txcmplq_cnt = 0; INIT_LIST_HEAD(&pring->txq); INIT_LIST_HEAD(&pring->txcmplq); INIT_LIST_HEAD(&pring->iocb_continueq); @@ -11327,6 +11283,7 @@ lpfc_sli4_abort_nvme_io(struct lpfc_hba *phba, struct lpfc_sli_ring *pring, struct lpfc_iocbq *abtsiocbp; union lpfc_wqe128 *abts_wqe; int retval; + int idx = cmdiocb->hba_wqidx; /* * There are certain command types we don't want to abort. And we @@ -11382,7 +11339,8 @@ lpfc_sli4_abort_nvme_io(struct lpfc_hba *phba, struct lpfc_sli_ring *pring, abtsiocbp->iocb_flag |= LPFC_IO_NVME; abtsiocbp->vport = vport; abtsiocbp->wqe_cmpl = lpfc_nvme_abort_fcreq_cmpl; - retval = lpfc_sli4_issue_wqe(phba, LPFC_FCP_RING, abtsiocbp); + retval = lpfc_sli4_issue_wqe(phba, &phba->sli4_hba.hdwq[idx], + abtsiocbp); if (retval) { lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME, "6147 Failed abts issue_wqe with status x%x " @@ -11457,7 +11415,7 @@ lpfc_sli_validate_fcp_iocb(struct lpfc_iocbq *iocbq, struct lpfc_vport *vport, uint16_t tgt_id, uint64_t lun_id, lpfc_ctx_cmd ctx_cmd) { - struct lpfc_scsi_buf *lpfc_cmd; + struct lpfc_io_buf *lpfc_cmd; int rc = 1; if (iocbq->vport != vport) @@ -11467,7 +11425,7 @@ lpfc_sli_validate_fcp_iocb(struct lpfc_iocbq *iocbq, struct lpfc_vport *vport, !(iocbq->iocb_flag & LPFC_IO_ON_TXCMPLQ)) return rc; - lpfc_cmd = container_of(iocbq, struct lpfc_scsi_buf, cur_iocbq); + lpfc_cmd = container_of(iocbq, struct lpfc_io_buf, cur_iocbq); if (lpfc_cmd->pCmd == NULL) return rc; @@ -11694,14 +11652,14 @@ lpfc_sli_abort_taskmgmt(struct lpfc_vport *vport, struct lpfc_sli_ring *pring, uint16_t tgt_id, uint64_t lun_id, lpfc_ctx_cmd cmd) { struct lpfc_hba *phba = vport->phba; - struct lpfc_scsi_buf *lpfc_cmd; + struct lpfc_io_buf *lpfc_cmd; struct lpfc_iocbq *abtsiocbq; struct lpfc_nodelist *ndlp; struct lpfc_iocbq *iocbq; IOCB_t *icmd; int sum, i, ret_val; unsigned long iflags; - struct lpfc_sli_ring *pring_s4; + struct lpfc_sli_ring *pring_s4 = NULL; spin_lock_irqsave(&phba->hbalock, iflags); @@ -11719,17 +11677,46 @@ lpfc_sli_abort_taskmgmt(struct lpfc_vport *vport, struct lpfc_sli_ring *pring, cmd) != 0) continue; + /* Guard against IO completion being called at same time */ + lpfc_cmd = container_of(iocbq, struct lpfc_io_buf, cur_iocbq); + spin_lock(&lpfc_cmd->buf_lock); + + if (!lpfc_cmd->pCmd) { + spin_unlock(&lpfc_cmd->buf_lock); + continue; + } + + if (phba->sli_rev == LPFC_SLI_REV4) { + pring_s4 = + phba->sli4_hba.hdwq[iocbq->hba_wqidx].fcp_wq->pring; + if (!pring_s4) { + spin_unlock(&lpfc_cmd->buf_lock); + continue; + } + /* Note: both hbalock and ring_lock must be set here */ + spin_lock(&pring_s4->ring_lock); + } + /* * If the iocbq is already being aborted, don't take a second * action, but do count it. */ - if (iocbq->iocb_flag & LPFC_DRIVER_ABORTED) + if ((iocbq->iocb_flag & LPFC_DRIVER_ABORTED) || + !(iocbq->iocb_flag & LPFC_IO_ON_TXCMPLQ)) { + if (phba->sli_rev == LPFC_SLI_REV4) + spin_unlock(&pring_s4->ring_lock); + spin_unlock(&lpfc_cmd->buf_lock); continue; + } /* issue ABTS for this IOCB based on iotag */ abtsiocbq = __lpfc_sli_get_iocbq(phba); - if (abtsiocbq == NULL) + if (!abtsiocbq) { + if (phba->sli_rev == LPFC_SLI_REV4) + spin_unlock(&pring_s4->ring_lock); + spin_unlock(&lpfc_cmd->buf_lock); continue; + } icmd = &iocbq->iocb; abtsiocbq->iocb.un.acxri.abortType = ABORT_TYPE_ABTS; @@ -11750,7 +11737,6 @@ lpfc_sli_abort_taskmgmt(struct lpfc_vport *vport, struct lpfc_sli_ring *pring, if (iocbq->iocb_flag & LPFC_IO_FOF) abtsiocbq->iocb_flag |= LPFC_IO_FOF; - lpfc_cmd = container_of(iocbq, struct lpfc_scsi_buf, cur_iocbq); ndlp = lpfc_cmd->rdata->pnode; if (lpfc_is_link_up(phba) && @@ -11769,11 +11755,6 @@ lpfc_sli_abort_taskmgmt(struct lpfc_vport *vport, struct lpfc_sli_ring *pring, iocbq->iocb_flag |= LPFC_DRIVER_ABORTED; if (phba->sli_rev == LPFC_SLI_REV4) { - pring_s4 = lpfc_sli4_calc_ring(phba, abtsiocbq); - if (!pring_s4) - continue; - /* Note: both hbalock and ring_lock must be set here */ - spin_lock(&pring_s4->ring_lock); ret_val = __lpfc_sli_issue_iocb(phba, pring_s4->ringno, abtsiocbq, 0); spin_unlock(&pring_s4->ring_lock); @@ -11782,6 +11763,7 @@ lpfc_sli_abort_taskmgmt(struct lpfc_vport *vport, struct lpfc_sli_ring *pring, abtsiocbq, 0); } + spin_unlock(&lpfc_cmd->buf_lock); if (ret_val == IOCB_ERROR) __lpfc_sli_release_iocbq(phba, abtsiocbq); @@ -11816,7 +11798,7 @@ lpfc_sli_wake_iocb_wait(struct lpfc_hba *phba, { wait_queue_head_t *pdone_q; unsigned long iflags; - struct lpfc_scsi_buf *lpfc_cmd; + struct lpfc_io_buf *lpfc_cmd; spin_lock_irqsave(&phba->hbalock, iflags); if (cmdiocbq->iocb_flag & LPFC_IO_WAKE_TMO) { @@ -11845,7 +11827,7 @@ lpfc_sli_wake_iocb_wait(struct lpfc_hba *phba, /* Set the exchange busy flag for task management commands */ if ((cmdiocbq->iocb_flag & LPFC_IO_FCP) && !(cmdiocbq->iocb_flag & LPFC_IO_LIBDFC)) { - lpfc_cmd = container_of(cmdiocbq, struct lpfc_scsi_buf, + lpfc_cmd = container_of(cmdiocbq, struct lpfc_io_buf, cur_iocbq); lpfc_cmd->exch_busy = rspiocbq->iocb_flag & LPFC_EXCHANGE_BUSY; } @@ -12919,35 +12901,6 @@ lpfc_sli_intr_handler(int irq, void *dev_id) } /* lpfc_sli_intr_handler */ /** - * lpfc_sli4_fcp_xri_abort_event_proc - Process fcp xri abort event - * @phba: pointer to lpfc hba data structure. - * - * This routine is invoked by the worker thread to process all the pending - * SLI4 FCP abort XRI events. - **/ -void lpfc_sli4_fcp_xri_abort_event_proc(struct lpfc_hba *phba) -{ - struct lpfc_cq_event *cq_event; - - /* First, declare the fcp xri abort event has been handled */ - spin_lock_irq(&phba->hbalock); - phba->hba_flag &= ~FCP_XRI_ABORT_EVENT; - spin_unlock_irq(&phba->hbalock); - /* Now, handle all the fcp xri abort events */ - while (!list_empty(&phba->sli4_hba.sp_fcp_xri_aborted_work_queue)) { - /* Get the first event from the head of the event queue */ - spin_lock_irq(&phba->hbalock); - list_remove_head(&phba->sli4_hba.sp_fcp_xri_aborted_work_queue, - cq_event, struct lpfc_cq_event, list); - spin_unlock_irq(&phba->hbalock); - /* Notify aborted XRI for FCP work queue */ - lpfc_sli4_fcp_xri_aborted(phba, &cq_event->cqe.wcqe_axri); - /* Free the event processed back to the free pool */ - lpfc_sli4_cq_event_release(phba, cq_event); - } -} - -/** * lpfc_sli4_els_xri_abort_event_proc - Process els xri abort event * @phba: pointer to lpfc hba data structure. * @@ -13320,11 +13273,14 @@ out_no_mqe_complete: * Return: true if work posted to worker thread, otherwise false. **/ static bool -lpfc_sli4_sp_handle_mcqe(struct lpfc_hba *phba, struct lpfc_cqe *cqe) +lpfc_sli4_sp_handle_mcqe(struct lpfc_hba *phba, struct lpfc_queue *cq, + struct lpfc_cqe *cqe) { struct lpfc_mcqe mcqe; bool workposted; + cq->CQ_mbox++; + /* Copy the mailbox MCQE and convert endian order as needed */ lpfc_sli4_pcimem_bcopy(cqe, &mcqe, sizeof(struct lpfc_mcqe)); @@ -13443,17 +13399,8 @@ lpfc_sli4_sp_handle_abort_xri_wcqe(struct lpfc_hba *phba, switch (cq->subtype) { case LPFC_FCP: - cq_event = lpfc_cq_event_setup( - phba, wcqe, sizeof(struct sli4_wcqe_xri_aborted)); - if (!cq_event) - return false; - spin_lock_irqsave(&phba->hbalock, iflags); - list_add_tail(&cq_event->list, - &phba->sli4_hba.sp_fcp_xri_aborted_work_queue); - /* Set the fcp xri abort event flag */ - phba->hba_flag |= FCP_XRI_ABORT_EVENT; - spin_unlock_irqrestore(&phba->hbalock, iflags); - workposted = true; + lpfc_sli4_fcp_xri_aborted(phba, wcqe, cq->hdwq); + workposted = false; break; case LPFC_NVME_LS: /* NVME LS uses ELS resources */ case LPFC_ELS: @@ -13461,6 +13408,7 @@ lpfc_sli4_sp_handle_abort_xri_wcqe(struct lpfc_hba *phba, phba, wcqe, sizeof(struct sli4_wcqe_xri_aborted)); if (!cq_event) return false; + cq_event->hdwq = cq->hdwq; spin_lock_irqsave(&phba->hbalock, iflags); list_add_tail(&cq_event->list, &phba->sli4_hba.sp_els_xri_aborted_work_queue); @@ -13474,7 +13422,7 @@ lpfc_sli4_sp_handle_abort_xri_wcqe(struct lpfc_hba *phba, if (phba->nvmet_support) lpfc_sli4_nvmet_xri_aborted(phba, wcqe); else - lpfc_sli4_nvme_xri_aborted(phba, wcqe); + lpfc_sli4_nvme_xri_aborted(phba, wcqe, cq->hdwq); workposted = false; break; @@ -13592,7 +13540,7 @@ out: * lpfc_sli4_sp_handle_cqe - Process a slow path completion queue entry * @phba: Pointer to HBA context object. * @cq: Pointer to the completion queue. - * @wcqe: Pointer to a completion queue entry. + * @cqe: Pointer to a completion queue entry. * * This routine process a slow-path work-queue or receive queue completion queue * entry. @@ -13684,7 +13632,7 @@ lpfc_sli4_sp_handle_eqe(struct lpfc_hba *phba, struct lpfc_eqe *eqe, /* Save EQ associated with this CQ */ cq->assoc_qp = speq; - if (!queue_work(phba->wq, &cq->spwork)) + if (!queue_work_on(cq->chann, phba->wq, &cq->spwork)) lpfc_printf_log(phba, KERN_ERR, LOG_SLI, "0390 Cannot schedule soft IRQ " "for CQ eqcqid=%d, cqid=%d on CPU %d\n", @@ -13692,60 +13640,129 @@ lpfc_sli4_sp_handle_eqe(struct lpfc_hba *phba, struct lpfc_eqe *eqe, } /** - * lpfc_sli4_sp_process_cq - Process a slow-path event queue entry + * __lpfc_sli4_process_cq - Process elements of a CQ * @phba: Pointer to HBA context object. + * @cq: Pointer to CQ to be processed + * @handler: Routine to process each cqe + * @delay: Pointer to usdelay to set in case of rescheduling of the handler * - * This routine process a event queue entry from the slow-path event queue. - * It will check the MajorCode and MinorCode to determine this is for a - * completion event on a completion queue, if not, an error shall be logged - * and just return. Otherwise, it will get to the corresponding completion - * queue and process all the entries on that completion queue, rearm the - * completion queue, and then return. + * This routine processes completion queue entries in a CQ. While a valid + * queue element is found, the handler is called. During processing checks + * are made for periodic doorbell writes to let the hardware know of + * element consumption. + * + * If the max limit on cqes to process is hit, or there are no more valid + * entries, the loop stops. If we processed a sufficient number of elements, + * meaning there is sufficient load, rather than rearming and generating + * another interrupt, a cq rescheduling delay will be set. A delay of 0 + * indicates no rescheduling. * + * Returns True if work scheduled, False otherwise. **/ -static void -lpfc_sli4_sp_process_cq(struct work_struct *work) +static bool +__lpfc_sli4_process_cq(struct lpfc_hba *phba, struct lpfc_queue *cq, + bool (*handler)(struct lpfc_hba *, struct lpfc_queue *, + struct lpfc_cqe *), unsigned long *delay) { - struct lpfc_queue *cq = - container_of(work, struct lpfc_queue, spwork); - struct lpfc_hba *phba = cq->phba; struct lpfc_cqe *cqe; bool workposted = false; - int ccount = 0; + int count = 0, consumed = 0; + bool arm = true; + + /* default - no reschedule */ + *delay = 0; + + if (cmpxchg(&cq->queue_claimed, 0, 1) != 0) + goto rearm_and_exit; /* Process all the entries to the CQ */ + cqe = lpfc_sli4_cq_get(cq); + while (cqe) { +#if defined(CONFIG_SCSI_LPFC_DEBUG_FS) && defined(BUILD_NVME) + if (phba->ktime_on) + cq->isr_timestamp = ktime_get_ns(); + else + cq->isr_timestamp = 0; +#endif + workposted |= handler(phba, cq, cqe); + __lpfc_sli4_consume_cqe(phba, cq, cqe); + + consumed++; + if (!(++count % cq->max_proc_limit)) + break; + + if (!(count % cq->notify_interval)) { + phba->sli4_hba.sli4_write_cq_db(phba, cq, consumed, + LPFC_QUEUE_NOARM); + consumed = 0; + } + + cqe = lpfc_sli4_cq_get(cq); + } + if (count >= phba->cfg_cq_poll_threshold) { + *delay = 1; + arm = false; + } + + /* Track the max number of CQEs processed in 1 EQ */ + if (count > cq->CQ_max_cqe) + cq->CQ_max_cqe = count; + + cq->assoc_qp->EQ_cqe_cnt += count; + + /* Catch the no cq entry condition */ + if (unlikely(count == 0)) + lpfc_printf_log(phba, KERN_INFO, LOG_SLI, + "0369 No entry from completion queue " + "qid=%d\n", cq->queue_id); + + cq->queue_claimed = 0; + +rearm_and_exit: + phba->sli4_hba.sli4_write_cq_db(phba, cq, consumed, + arm ? LPFC_QUEUE_REARM : LPFC_QUEUE_NOARM); + + return workposted; +} + +/** + * lpfc_sli4_sp_process_cq - Process a slow-path event queue entry + * @cq: pointer to CQ to process + * + * This routine calls the cq processing routine with a handler specific + * to the type of queue bound to it. + * + * The CQ routine returns two values: the first is the calling status, + * which indicates whether work was queued to the background discovery + * thread. If true, the routine should wakeup the discovery thread; + * the second is the delay parameter. If non-zero, rather than rearming + * the CQ and yet another interrupt, the CQ handler should be queued so + * that it is processed in a subsequent polling action. The value of + * the delay indicates when to reschedule it. + **/ +static void +__lpfc_sli4_sp_process_cq(struct lpfc_queue *cq) +{ + struct lpfc_hba *phba = cq->phba; + unsigned long delay; + bool workposted = false; + + /* Process and rearm the CQ */ switch (cq->type) { case LPFC_MCQ: - while ((cqe = lpfc_sli4_cq_get(cq))) { - workposted |= lpfc_sli4_sp_handle_mcqe(phba, cqe); - if (!(++ccount % cq->entry_repost)) - break; - cq->CQ_mbox++; - } + workposted |= __lpfc_sli4_process_cq(phba, cq, + lpfc_sli4_sp_handle_mcqe, + &delay); break; case LPFC_WCQ: - while ((cqe = lpfc_sli4_cq_get(cq))) { - if (cq->subtype == LPFC_FCP || - cq->subtype == LPFC_NVME) { -#ifdef CONFIG_SCSI_LPFC_DEBUG_FS - if (phba->ktime_on) - cq->isr_timestamp = ktime_get_ns(); - else - cq->isr_timestamp = 0; -#endif - workposted |= lpfc_sli4_fp_handle_cqe(phba, cq, - cqe); - } else { - workposted |= lpfc_sli4_sp_handle_cqe(phba, cq, - cqe); - } - if (!(++ccount % cq->entry_repost)) - break; - } - - /* Track the max number of CQEs processed in 1 EQ */ - if (ccount > cq->CQ_max_cqe) - cq->CQ_max_cqe = ccount; + if (cq->subtype == LPFC_FCP || cq->subtype == LPFC_NVME) + workposted |= __lpfc_sli4_process_cq(phba, cq, + lpfc_sli4_fp_handle_cqe, + &delay); + else + workposted |= __lpfc_sli4_process_cq(phba, cq, + lpfc_sli4_sp_handle_cqe, + &delay); break; default: lpfc_printf_log(phba, KERN_ERR, LOG_SLI, @@ -13754,14 +13771,14 @@ lpfc_sli4_sp_process_cq(struct work_struct *work) return; } - /* Catch the no cq entry condition, log an error */ - if (unlikely(ccount == 0)) - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, - "0371 No entry from the CQ: identifier " - "(x%x), type (%d)\n", cq->queue_id, cq->type); - - /* In any case, flash and re-arm the RCQ */ - phba->sli4_hba.sli4_cq_release(cq, LPFC_QUEUE_REARM); + if (delay) { + if (!queue_delayed_work_on(cq->chann, phba->wq, + &cq->sched_spwork, delay)) + lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + "0394 Cannot schedule soft IRQ " + "for cqid=%d on CPU %d\n", + cq->queue_id, cq->chann); + } /* wake up worker thread if there are works to be done */ if (workposted) @@ -13769,6 +13786,36 @@ lpfc_sli4_sp_process_cq(struct work_struct *work) } /** + * lpfc_sli4_sp_process_cq - slow-path work handler when started by + * interrupt + * @work: pointer to work element + * + * translates from the work handler and calls the slow-path handler. + **/ +static void +lpfc_sli4_sp_process_cq(struct work_struct *work) +{ + struct lpfc_queue *cq = container_of(work, struct lpfc_queue, spwork); + + __lpfc_sli4_sp_process_cq(cq); +} + +/** + * lpfc_sli4_dly_sp_process_cq - slow-path work handler when started by timer + * @work: pointer to work element + * + * translates from the work handler and calls the slow-path handler. + **/ +static void +lpfc_sli4_dly_sp_process_cq(struct work_struct *work) +{ + struct lpfc_queue *cq = container_of(to_delayed_work(work), + struct lpfc_queue, sched_spwork); + + __lpfc_sli4_sp_process_cq(cq); +} + +/** * lpfc_sli4_fp_handle_fcp_wcqe - Process fast-path work queue completion entry * @phba: Pointer to HBA context object. * @cq: Pointer to associated CQ @@ -13999,13 +14046,16 @@ out: /** * lpfc_sli4_fp_handle_cqe - Process fast-path work queue completion entry + * @phba: adapter with cq * @cq: Pointer to the completion queue. * @eqe: Pointer to fast-path completion queue entry. * * This routine process a fast-path work queue completion entry from fast-path * event queue for FCP command response completion. + * + * Return: true if work posted to worker thread, otherwise false. **/ -static int +static bool lpfc_sli4_fp_handle_cqe(struct lpfc_hba *phba, struct lpfc_queue *cq, struct lpfc_cqe *cqe) { @@ -14072,10 +14122,11 @@ lpfc_sli4_fp_handle_cqe(struct lpfc_hba *phba, struct lpfc_queue *cq, * completion queue, and then return. **/ static void -lpfc_sli4_hba_handle_eqe(struct lpfc_hba *phba, struct lpfc_eqe *eqe, - uint32_t qidx) +lpfc_sli4_hba_handle_eqe(struct lpfc_hba *phba, struct lpfc_queue *eq, + struct lpfc_eqe *eqe) { struct lpfc_queue *cq = NULL; + uint32_t qidx = eq->hdwq; uint16_t cqid, id; if (unlikely(bf_get_le32(lpfc_eqe_major_code, eqe) != 0)) { @@ -14090,6 +14141,14 @@ lpfc_sli4_hba_handle_eqe(struct lpfc_hba *phba, struct lpfc_eqe *eqe, /* Get the reference to the corresponding CQ */ cqid = bf_get_le32(lpfc_eqe_resource_id, eqe); + /* Use the fast lookup method first */ + if (cqid <= phba->sli4_hba.cq_max) { + cq = phba->sli4_hba.cq_lookup[cqid]; + if (cq) + goto work_cq; + } + + /* Next check for NVMET completion */ if (phba->cfg_nvmet_mrq && phba->sli4_hba.nvmet_cqset) { id = phba->sli4_hba.nvmet_cqset[0]->queue_id; if ((cqid >= id) && (cqid < (id + phba->cfg_nvmet_mrq))) { @@ -14099,20 +14158,6 @@ lpfc_sli4_hba_handle_eqe(struct lpfc_hba *phba, struct lpfc_eqe *eqe, } } - if (phba->sli4_hba.nvme_cq_map && - (cqid == phba->sli4_hba.nvme_cq_map[qidx])) { - /* Process NVME / NVMET command completion */ - cq = phba->sli4_hba.nvme_cq[qidx]; - goto process_cq; - } - - if (phba->sli4_hba.fcp_cq_map && - (cqid == phba->sli4_hba.fcp_cq_map[qidx])) { - /* Process FCP command completion */ - cq = phba->sli4_hba.fcp_cq[qidx]; - goto process_cq; - } - if (phba->sli4_hba.nvmels_cq && (cqid == phba->sli4_hba.nvmels_cq->queue_id)) { /* Process NVME unsol rcv */ @@ -14121,7 +14166,8 @@ lpfc_sli4_hba_handle_eqe(struct lpfc_hba *phba, struct lpfc_eqe *eqe, /* Otherwise this is a Slow path event */ if (cq == NULL) { - lpfc_sli4_sp_handle_eqe(phba, eqe, phba->sli4_hba.hba_eq[qidx]); + lpfc_sli4_sp_handle_eqe(phba, eqe, + phba->sli4_hba.hdwq[qidx].hba_eq); return; } @@ -14134,10 +14180,8 @@ process_cq: return; } - /* Save EQ associated with this CQ */ - cq->assoc_qp = phba->sli4_hba.hba_eq[qidx]; - - if (!queue_work(phba->wq, &cq->irqwork)) +work_cq: + if (!queue_work_on(cq->chann, phba->wq, &cq->irqwork)) lpfc_printf_log(phba, KERN_ERR, LOG_SLI, "0363 Cannot schedule soft IRQ " "for CQ eqcqid=%d, cqid=%d on CPU %d\n", @@ -14145,219 +14189,73 @@ process_cq: } /** - * lpfc_sli4_hba_process_cq - Process a fast-path event queue entry - * @phba: Pointer to HBA context object. - * @eqe: Pointer to fast-path event queue entry. + * __lpfc_sli4_hba_process_cq - Process a fast-path event queue entry + * @cq: Pointer to CQ to be processed * - * This routine process a event queue entry from the fast-path event queue. - * It will check the MajorCode and MinorCode to determine this is for a - * completion event on a completion queue, if not, an error shall be logged - * and just return. Otherwise, it will get to the corresponding completion - * queue and process all the entries on the completion queue, rearm the - * completion queue, and then return. + * This routine calls the cq processing routine with the handler for + * fast path CQEs. + * + * The CQ routine returns two values: the first is the calling status, + * which indicates whether work was queued to the background discovery + * thread. If true, the routine should wakeup the discovery thread; + * the second is the delay parameter. If non-zero, rather than rearming + * the CQ and yet another interrupt, the CQ handler should be queued so + * that it is processed in a subsequent polling action. The value of + * the delay indicates when to reschedule it. **/ static void -lpfc_sli4_hba_process_cq(struct work_struct *work) +__lpfc_sli4_hba_process_cq(struct lpfc_queue *cq) { - struct lpfc_queue *cq = - container_of(work, struct lpfc_queue, irqwork); struct lpfc_hba *phba = cq->phba; - struct lpfc_cqe *cqe; + unsigned long delay; bool workposted = false; - int ccount = 0; - - /* Process all the entries to the CQ */ - while ((cqe = lpfc_sli4_cq_get(cq))) { -#ifdef CONFIG_SCSI_LPFC_DEBUG_FS - if (phba->ktime_on) - cq->isr_timestamp = ktime_get_ns(); - else - cq->isr_timestamp = 0; -#endif - workposted |= lpfc_sli4_fp_handle_cqe(phba, cq, cqe); - if (!(++ccount % cq->entry_repost)) - break; - } - /* Track the max number of CQEs processed in 1 EQ */ - if (ccount > cq->CQ_max_cqe) - cq->CQ_max_cqe = ccount; - cq->assoc_qp->EQ_cqe_cnt += ccount; + /* process and rearm the CQ */ + workposted |= __lpfc_sli4_process_cq(phba, cq, lpfc_sli4_fp_handle_cqe, + &delay); - /* Catch the no cq entry condition */ - if (unlikely(ccount == 0)) - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, - "0369 No entry from fast-path completion " - "queue fcpcqid=%d\n", cq->queue_id); - - /* In any case, flash and re-arm the CQ */ - phba->sli4_hba.sli4_cq_release(cq, LPFC_QUEUE_REARM); + if (delay) { + if (!queue_delayed_work_on(cq->chann, phba->wq, + &cq->sched_irqwork, delay)) + lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + "0367 Cannot schedule soft IRQ " + "for cqid=%d on CPU %d\n", + cq->queue_id, cq->chann); + } /* wake up worker thread if there are works to be done */ if (workposted) lpfc_worker_wake_up(phba); } -static void -lpfc_sli4_eq_flush(struct lpfc_hba *phba, struct lpfc_queue *eq) -{ - struct lpfc_eqe *eqe; - - /* walk all the EQ entries and drop on the floor */ - while ((eqe = lpfc_sli4_eq_get(eq))) - ; - - /* Clear and re-arm the EQ */ - phba->sli4_hba.sli4_eq_release(eq, LPFC_QUEUE_REARM); -} - - /** - * lpfc_sli4_fof_handle_eqe - Process a Flash Optimized Fabric event queue - * entry - * @phba: Pointer to HBA context object. - * @eqe: Pointer to fast-path event queue entry. + * lpfc_sli4_hba_process_cq - fast-path work handler when started by + * interrupt + * @work: pointer to work element * - * This routine process a event queue entry from the Flash Optimized Fabric - * event queue. It will check the MajorCode and MinorCode to determine this - * is for a completion event on a completion queue, if not, an error shall be - * logged and just return. Otherwise, it will get to the corresponding - * completion queue and process all the entries on the completion queue, rearm - * the completion queue, and then return. + * translates from the work handler and calls the fast-path handler. **/ static void -lpfc_sli4_fof_handle_eqe(struct lpfc_hba *phba, struct lpfc_eqe *eqe) +lpfc_sli4_hba_process_cq(struct work_struct *work) { - struct lpfc_queue *cq; - uint16_t cqid; + struct lpfc_queue *cq = container_of(work, struct lpfc_queue, irqwork); - if (unlikely(bf_get_le32(lpfc_eqe_major_code, eqe) != 0)) { - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, - "9147 Not a valid completion " - "event: majorcode=x%x, minorcode=x%x\n", - bf_get_le32(lpfc_eqe_major_code, eqe), - bf_get_le32(lpfc_eqe_minor_code, eqe)); - return; - } - - /* Get the reference to the corresponding CQ */ - cqid = bf_get_le32(lpfc_eqe_resource_id, eqe); - - /* Next check for OAS */ - cq = phba->sli4_hba.oas_cq; - if (unlikely(!cq)) { - if (phba->sli.sli_flag & LPFC_SLI_ACTIVE) - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, - "9148 OAS completion queue " - "does not exist\n"); - return; - } - - if (unlikely(cqid != cq->queue_id)) { - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, - "9149 Miss-matched fast-path compl " - "queue id: eqcqid=%d, fcpcqid=%d\n", - cqid, cq->queue_id); - return; - } - - /* Save EQ associated with this CQ */ - cq->assoc_qp = phba->sli4_hba.fof_eq; - - /* CQ work will be processed on CPU affinitized to this IRQ */ - if (!queue_work(phba->wq, &cq->irqwork)) - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, - "0367 Cannot schedule soft IRQ " - "for CQ eqcqid=%d, cqid=%d on CPU %d\n", - cqid, cq->queue_id, smp_processor_id()); + __lpfc_sli4_hba_process_cq(cq); } /** - * lpfc_sli4_fof_intr_handler - HBA interrupt handler to SLI-4 device - * @irq: Interrupt number. - * @dev_id: The device context pointer. + * lpfc_sli4_hba_process_cq - fast-path work handler when started by timer + * @work: pointer to work element * - * This function is directly called from the PCI layer as an interrupt - * service routine when device with SLI-4 interface spec is enabled with - * MSI-X multi-message interrupt mode and there is a Flash Optimized Fabric - * IOCB ring event in the HBA. However, when the device is enabled with either - * MSI or Pin-IRQ interrupt mode, this function is called as part of the - * device-level interrupt handler. When the PCI slot is in error recovery - * or the HBA is undergoing initialization, the interrupt handler will not - * process the interrupt. The Flash Optimized Fabric ring event are handled in - * the intrrupt context. This function is called without any lock held. - * It gets the hbalock to access and update SLI data structures. Note that, - * the EQ to CQ are one-to-one map such that the EQ index is - * equal to that of CQ index. - * - * This function returns IRQ_HANDLED when interrupt is handled else it - * returns IRQ_NONE. + * translates from the work handler and calls the fast-path handler. **/ -irqreturn_t -lpfc_sli4_fof_intr_handler(int irq, void *dev_id) +static void +lpfc_sli4_dly_hba_process_cq(struct work_struct *work) { - struct lpfc_hba *phba; - struct lpfc_hba_eq_hdl *hba_eq_hdl; - struct lpfc_queue *eq; - struct lpfc_eqe *eqe; - unsigned long iflag; - int ecount = 0; - - /* Get the driver's phba structure from the dev_id */ - hba_eq_hdl = (struct lpfc_hba_eq_hdl *)dev_id; - phba = hba_eq_hdl->phba; - - if (unlikely(!phba)) - return IRQ_NONE; - - /* Get to the EQ struct associated with this vector */ - eq = phba->sli4_hba.fof_eq; - if (unlikely(!eq)) - return IRQ_NONE; - - /* Check device state for handling interrupt */ - if (unlikely(lpfc_intr_state_check(phba))) { - /* Check again for link_state with lock held */ - spin_lock_irqsave(&phba->hbalock, iflag); - if (phba->link_state < LPFC_LINK_DOWN) - /* Flush, clear interrupt, and rearm the EQ */ - lpfc_sli4_eq_flush(phba, eq); - spin_unlock_irqrestore(&phba->hbalock, iflag); - return IRQ_NONE; - } - - /* - * Process all the event on FCP fast-path EQ - */ - while ((eqe = lpfc_sli4_eq_get(eq))) { - lpfc_sli4_fof_handle_eqe(phba, eqe); - if (!(++ecount % eq->entry_repost)) - break; - eq->EQ_processed++; - } + struct lpfc_queue *cq = container_of(to_delayed_work(work), + struct lpfc_queue, sched_irqwork); - /* Track the max number of EQEs processed in 1 intr */ - if (ecount > eq->EQ_max_eqe) - eq->EQ_max_eqe = ecount; - - - if (unlikely(ecount == 0)) { - eq->EQ_no_entry++; - - if (phba->intr_type == MSIX) - /* MSI-X treated interrupt served as no EQ share INT */ - lpfc_printf_log(phba, KERN_WARNING, LOG_SLI, - "9145 MSI-X interrupt with no EQE\n"); - else { - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, - "9146 ISR interrupt with no EQE\n"); - /* Non MSI-X treated on interrupt as EQ share INT */ - return IRQ_NONE; - } - } - /* Always clear and re-arm the fast-path EQ */ - phba->sli4_hba.sli4_eq_release(eq, LPFC_QUEUE_REARM); - return IRQ_HANDLED; + __lpfc_sli4_hba_process_cq(cq); } /** @@ -14392,10 +14290,11 @@ lpfc_sli4_hba_intr_handler(int irq, void *dev_id) struct lpfc_hba *phba; struct lpfc_hba_eq_hdl *hba_eq_hdl; struct lpfc_queue *fpeq; - struct lpfc_eqe *eqe; unsigned long iflag; int ecount = 0; int hba_eqidx; + struct lpfc_eq_intr_info *eqi; + uint32_t icnt; /* Get the driver's phba structure from the dev_id */ hba_eq_hdl = (struct lpfc_hba_eq_hdl *)dev_id; @@ -14404,23 +14303,14 @@ lpfc_sli4_hba_intr_handler(int irq, void *dev_id) if (unlikely(!phba)) return IRQ_NONE; - if (unlikely(!phba->sli4_hba.hba_eq)) + if (unlikely(!phba->sli4_hba.hdwq)) return IRQ_NONE; /* Get to the EQ struct associated with this vector */ - fpeq = phba->sli4_hba.hba_eq[hba_eqidx]; + fpeq = phba->sli4_hba.hdwq[hba_eqidx].hba_eq; if (unlikely(!fpeq)) return IRQ_NONE; - if (lpfc_fcp_look_ahead) { - if (atomic_dec_and_test(&hba_eq_hdl->hba_eq_in_use)) - phba->sli4_hba.sli4_eq_clr_intr(fpeq); - else { - atomic_inc(&hba_eq_hdl->hba_eq_in_use); - return IRQ_NONE; - } - } - /* Check device state for handling interrupt */ if (unlikely(lpfc_intr_state_check(phba))) { /* Check again for link_state with lock held */ @@ -14429,36 +14319,25 @@ lpfc_sli4_hba_intr_handler(int irq, void *dev_id) /* Flush, clear interrupt, and rearm the EQ */ lpfc_sli4_eq_flush(phba, fpeq); spin_unlock_irqrestore(&phba->hbalock, iflag); - if (lpfc_fcp_look_ahead) - atomic_inc(&hba_eq_hdl->hba_eq_in_use); return IRQ_NONE; } - /* - * Process all the event on FCP fast-path EQ - */ - while ((eqe = lpfc_sli4_eq_get(fpeq))) { - lpfc_sli4_hba_handle_eqe(phba, eqe, hba_eqidx); - if (!(++ecount % fpeq->entry_repost)) - break; - fpeq->EQ_processed++; - } + eqi = phba->sli4_hba.eq_info; + icnt = this_cpu_inc_return(eqi->icnt); + fpeq->last_cpu = smp_processor_id(); - /* Track the max number of EQEs processed in 1 intr */ - if (ecount > fpeq->EQ_max_eqe) - fpeq->EQ_max_eqe = ecount; + if (icnt > LPFC_EQD_ISR_TRIGGER && + phba->cfg_irq_chann == 1 && + phba->cfg_auto_imax && + fpeq->q_mode != LPFC_MAX_AUTO_EQ_DELAY && + phba->sli.sli_flag & LPFC_SLI_USE_EQDR) + lpfc_sli4_mod_hba_eq_delay(phba, fpeq, LPFC_MAX_AUTO_EQ_DELAY); - /* Always clear and re-arm the fast-path EQ */ - phba->sli4_hba.sli4_eq_release(fpeq, LPFC_QUEUE_REARM); + /* process and rearm the EQ */ + ecount = lpfc_sli4_process_eq(phba, fpeq); if (unlikely(ecount == 0)) { fpeq->EQ_no_entry++; - - if (lpfc_fcp_look_ahead) { - atomic_inc(&hba_eq_hdl->hba_eq_in_use); - return IRQ_NONE; - } - if (phba->intr_type == MSIX) /* MSI-X treated interrupt served as no EQ share INT */ lpfc_printf_log(phba, KERN_WARNING, LOG_SLI, @@ -14468,9 +14347,6 @@ lpfc_sli4_hba_intr_handler(int irq, void *dev_id) return IRQ_NONE; } - if (lpfc_fcp_look_ahead) - atomic_inc(&hba_eq_hdl->hba_eq_in_use); - return IRQ_HANDLED; } /* lpfc_sli4_fp_intr_handler */ @@ -14508,20 +14384,13 @@ lpfc_sli4_intr_handler(int irq, void *dev_id) /* * Invoke fast-path host attention interrupt handling as appropriate. */ - for (qidx = 0; qidx < phba->io_channel_irqs; qidx++) { + for (qidx = 0; qidx < phba->cfg_irq_chann; qidx++) { hba_irq_rc = lpfc_sli4_hba_intr_handler(irq, &phba->sli4_hba.hba_eq_hdl[qidx]); if (hba_irq_rc == IRQ_HANDLED) hba_handled |= true; } - if (phba->cfg_fof) { - hba_irq_rc = lpfc_sli4_fof_intr_handler(irq, - &phba->sli4_hba.hba_eq_hdl[qidx]); - if (hba_irq_rc == IRQ_HANDLED) - hba_handled |= true; - } - return (hba_handled == true) ? IRQ_HANDLED : IRQ_NONE; } /* lpfc_sli4_intr_handler */ @@ -14553,6 +14422,9 @@ lpfc_sli4_queue_free(struct lpfc_queue *queue) kfree(queue->rqbp); } + if (!list_empty(&queue->cpu_list)) + list_del(&queue->cpu_list); + if (!list_empty(&queue->wq_list)) list_del(&queue->wq_list); @@ -14601,6 +14473,7 @@ lpfc_sli4_queue_alloc(struct lpfc_hba *phba, uint32_t page_size, INIT_LIST_HEAD(&queue->wqfull_list); INIT_LIST_HEAD(&queue->page_list); INIT_LIST_HEAD(&queue->child_list); + INIT_LIST_HEAD(&queue->cpu_list); /* Set queue parameters now. If the system cannot provide memory * resources, the free routine needs to know what was allocated. @@ -14633,8 +14506,10 @@ lpfc_sli4_queue_alloc(struct lpfc_hba *phba, uint32_t page_size, } INIT_WORK(&queue->irqwork, lpfc_sli4_hba_process_cq); INIT_WORK(&queue->spwork, lpfc_sli4_sp_process_cq); + INIT_DELAYED_WORK(&queue->sched_irqwork, lpfc_sli4_dly_hba_process_cq); + INIT_DELAYED_WORK(&queue->sched_spwork, lpfc_sli4_dly_sp_process_cq); - /* entry_repost will be set during q creation */ + /* notify_interval will be set during q creation */ return queue; out_fail: @@ -14671,43 +14546,76 @@ lpfc_dual_chute_pci_bar_map(struct lpfc_hba *phba, uint16_t pci_barset) } /** - * lpfc_modify_hba_eq_delay - Modify Delay Multiplier on FCP EQs - * @phba: HBA structure that indicates port to create a queue on. - * @startq: The starting FCP EQ to modify + * lpfc_modify_hba_eq_delay - Modify Delay Multiplier on EQs + * @phba: HBA structure that EQs are on. + * @startq: The starting EQ index to modify + * @numq: The number of EQs (consecutive indexes) to modify + * @usdelay: amount of delay * - * This function sends an MODIFY_EQ_DELAY mailbox command to the HBA. - * The command allows up to LPFC_MAX_EQ_DELAY_EQID_CNT EQ ID's to be - * updated in one mailbox command. + * This function revises the EQ delay on 1 or more EQs. The EQ delay + * is set either by writing to a register (if supported by the SLI Port) + * or by mailbox command. The mailbox command allows several EQs to be + * updated at once. * - * The @phba struct is used to send mailbox command to HBA. The @startq - * is used to get the starting FCP EQ to change. - * This function is asynchronous and will wait for the mailbox - * command to finish before continuing. + * The @phba struct is used to send a mailbox command to HBA. The @startq + * is used to get the starting EQ index to change. The @numq value is + * used to specify how many consecutive EQ indexes, starting at EQ index, + * are to be changed. This function is asynchronous and will wait for any + * mailbox commands to finish before returning. * - * On success this function will return a zero. If unable to allocate enough - * memory this function will return -ENOMEM. If the queue create mailbox command - * fails this function will return -ENXIO. + * On success this function will return a zero. If unable to allocate + * enough memory this function will return -ENOMEM. If a mailbox command + * fails this function will return -ENXIO. Note: on ENXIO, some EQs may + * have had their delay multipler changed. **/ -int +void lpfc_modify_hba_eq_delay(struct lpfc_hba *phba, uint32_t startq, - uint32_t numq, uint32_t imax) + uint32_t numq, uint32_t usdelay) { struct lpfc_mbx_modify_eq_delay *eq_delay; LPFC_MBOXQ_t *mbox; struct lpfc_queue *eq; - int cnt, rc, length, status = 0; + int cnt = 0, rc, length; uint32_t shdr_status, shdr_add_status; - uint32_t result, val; + uint32_t dmult; int qidx; union lpfc_sli4_cfg_shdr *shdr; - uint16_t dmult; - if (startq >= phba->io_channel_irqs) - return 0; + if (startq >= phba->cfg_irq_chann) + return; + + if (usdelay > 0xFFFF) { + lpfc_printf_log(phba, KERN_INFO, LOG_INIT | LOG_FCP | LOG_NVME, + "6429 usdelay %d too large. Scaled down to " + "0xFFFF.\n", usdelay); + usdelay = 0xFFFF; + } + + /* set values by EQ_DELAY register if supported */ + if (phba->sli.sli_flag & LPFC_SLI_USE_EQDR) { + for (qidx = startq; qidx < phba->cfg_irq_chann; qidx++) { + eq = phba->sli4_hba.hdwq[qidx].hba_eq; + if (!eq) + continue; + + lpfc_sli4_mod_hba_eq_delay(phba, eq, usdelay); + + if (++cnt >= numq) + break; + } + + return; + } + + /* Otherwise, set values by mailbox cmd */ mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL); - if (!mbox) - return -ENOMEM; + if (!mbox) { + lpfc_printf_log(phba, KERN_ERR, LOG_INIT | LOG_FCP | LOG_NVME, + "6428 Failed allocating mailbox cmd buffer." + " EQ delay was not set.\n"); + return; + } length = (sizeof(struct lpfc_mbx_modify_eq_delay) - sizeof(struct lpfc_sli4_cfg_mhdr)); lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_COMMON, @@ -14716,45 +14624,22 @@ lpfc_modify_hba_eq_delay(struct lpfc_hba *phba, uint32_t startq, eq_delay = &mbox->u.mqe.un.eq_delay; /* Calculate delay multiper from maximum interrupt per second */ - result = imax / phba->io_channel_irqs; - if (result > LPFC_DMULT_CONST || result == 0) - dmult = 0; - else - dmult = LPFC_DMULT_CONST/result - 1; + dmult = (usdelay * LPFC_DMULT_CONST) / LPFC_SEC_TO_USEC; + if (dmult) + dmult--; if (dmult > LPFC_DMULT_MAX) dmult = LPFC_DMULT_MAX; - cnt = 0; - for (qidx = startq; qidx < phba->io_channel_irqs; qidx++) { - eq = phba->sli4_hba.hba_eq[qidx]; + for (qidx = startq; qidx < phba->cfg_irq_chann; qidx++) { + eq = phba->sli4_hba.hdwq[qidx].hba_eq; if (!eq) continue; - eq->q_mode = imax; + eq->q_mode = usdelay; eq_delay->u.request.eq[cnt].eq_id = eq->queue_id; eq_delay->u.request.eq[cnt].phase = 0; eq_delay->u.request.eq[cnt].delay_multi = dmult; - cnt++; - - /* q_mode is only used for auto_imax */ - if (phba->sli.sli_flag & LPFC_SLI_USE_EQDR) { - /* Use EQ Delay Register method for q_mode */ - - /* Convert for EQ Delay register */ - val = phba->cfg_fcp_imax; - if (val) { - /* First, interrupts per sec per EQ */ - val = phba->cfg_fcp_imax / - phba->io_channel_irqs; - - /* us delay between each interrupt */ - val = LPFC_SEC_TO_USEC / val; - } - eq->q_mode = val; - } else { - eq->q_mode = imax; - } - if (cnt >= numq) + if (++cnt >= numq) break; } eq_delay->u.request.num_eq = cnt; @@ -14772,10 +14657,9 @@ lpfc_modify_hba_eq_delay(struct lpfc_hba *phba, uint32_t startq, "2512 MODIFY_EQ_DELAY mailbox failed with " "status x%x add_status x%x, mbx status x%x\n", shdr_status, shdr_add_status, rc); - status = -ENXIO; } mempool_free(mbox, phba->mbox_mem_pool); - return status; + return; } /** @@ -14900,8 +14784,8 @@ lpfc_eq_create(struct lpfc_hba *phba, struct lpfc_queue *eq, uint32_t imax) if (eq->queue_id == 0xFFFF) status = -ENXIO; eq->host_index = 0; - eq->hba_index = 0; - eq->entry_repost = LPFC_EQ_REPOST; + eq->notify_interval = LPFC_EQ_NOTIFY_INTRVL; + eq->max_proc_limit = LPFC_EQ_MAX_PROC_LIMIT; mempool_free(mbox, phba->mbox_mem_pool); return status; @@ -15039,10 +14923,13 @@ lpfc_cq_create(struct lpfc_hba *phba, struct lpfc_queue *cq, cq->subtype = subtype; cq->queue_id = bf_get(lpfc_mbx_cq_create_q_id, &cq_create->u.response); cq->assoc_qid = eq->queue_id; + cq->assoc_qp = eq; cq->host_index = 0; - cq->hba_index = 0; - cq->entry_repost = LPFC_CQ_REPOST; + cq->notify_interval = LPFC_CQ_NOTIFY_INTRVL; + cq->max_proc_limit = min(phba->cfg_cq_max_proc_limit, cq->entry_count); + if (cq->queue_id > phba->sli4_hba.cq_max) + phba->sli4_hba.cq_max = cq->queue_id; out: mempool_free(mbox, phba->mbox_mem_pool); return status; @@ -15052,7 +14939,7 @@ out: * lpfc_cq_create_set - Create a set of Completion Queues on the HBA for MRQ * @phba: HBA structure that indicates port to create a queue on. * @cqp: The queue structure array to use to create the completion queues. - * @eqp: The event queue array to bind these completion queues to. + * @hdwq: The hardware queue array with the EQ to bind completion queues to. * * This function creates a set of completion queue, s to support MRQ * as detailed in @cqp, on a port, @@ -15072,7 +14959,8 @@ out: **/ int lpfc_cq_create_set(struct lpfc_hba *phba, struct lpfc_queue **cqp, - struct lpfc_queue **eqp, uint32_t type, uint32_t subtype) + struct lpfc_sli4_hdw_queue *hdwq, uint32_t type, + uint32_t subtype) { struct lpfc_queue *cq; struct lpfc_queue *eq; @@ -15087,7 +14975,7 @@ lpfc_cq_create_set(struct lpfc_hba *phba, struct lpfc_queue **cqp, /* sanity check on queue memory */ numcq = phba->cfg_nvmet_mrq; - if (!cqp || !eqp || !numcq) + if (!cqp || !hdwq || !numcq) return -ENODEV; mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL); @@ -15114,7 +15002,7 @@ lpfc_cq_create_set(struct lpfc_hba *phba, struct lpfc_queue **cqp, for (idx = 0; idx < numcq; idx++) { cq = cqp[idx]; - eq = eqp[idx]; + eq = hdwq[idx].hba_eq; if (!cq || !eq) { status = -ENOMEM; goto out; @@ -15247,9 +15135,11 @@ lpfc_cq_create_set(struct lpfc_hba *phba, struct lpfc_queue **cqp, cq->type = type; cq->subtype = subtype; cq->assoc_qid = eq->queue_id; + cq->assoc_qp = eq; cq->host_index = 0; - cq->hba_index = 0; - cq->entry_repost = LPFC_CQ_REPOST; + cq->notify_interval = LPFC_CQ_NOTIFY_INTRVL; + cq->max_proc_limit = min(phba->cfg_cq_max_proc_limit, + cq->entry_count); cq->chann = idx; rc = 0; @@ -15287,6 +15177,8 @@ lpfc_cq_create_set(struct lpfc_hba *phba, struct lpfc_queue **cqp, for (idx = 0; idx < numcq; idx++) { cq = cqp[idx]; cq->queue_id = rc + idx; + if (cq->queue_id > phba->sli4_hba.cq_max) + phba->sli4_hba.cq_max = cq->queue_id; } out: @@ -15499,7 +15391,6 @@ lpfc_mq_create(struct lpfc_hba *phba, struct lpfc_queue *mq, mq->subtype = subtype; mq->host_index = 0; mq->hba_index = 0; - mq->entry_repost = LPFC_MQ_REPOST; /* link the mq onto the parent cq child list */ list_add_tail(&mq->list, &cq->child_list); @@ -15765,7 +15656,7 @@ lpfc_wq_create(struct lpfc_hba *phba, struct lpfc_queue *wq, wq->subtype = subtype; wq->host_index = 0; wq->hba_index = 0; - wq->entry_repost = LPFC_RELEASE_NOTIFICATION_INTERVAL; + wq->notify_interval = LPFC_WQ_NOTIFY_INTRVL; /* link the wq onto the parent cq child list */ list_add_tail(&wq->list, &cq->child_list); @@ -15959,7 +15850,7 @@ lpfc_rq_create(struct lpfc_hba *phba, struct lpfc_queue *hrq, hrq->subtype = subtype; hrq->host_index = 0; hrq->hba_index = 0; - hrq->entry_repost = LPFC_RQ_REPOST; + hrq->notify_interval = LPFC_RQ_NOTIFY_INTRVL; /* now create the data queue */ lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_FCOE, @@ -16052,7 +15943,7 @@ lpfc_rq_create(struct lpfc_hba *phba, struct lpfc_queue *hrq, drq->subtype = subtype; drq->host_index = 0; drq->hba_index = 0; - drq->entry_repost = LPFC_RQ_REPOST; + drq->notify_interval = LPFC_RQ_NOTIFY_INTRVL; /* link the header and data RQs onto the parent cq child list */ list_add_tail(&hrq->list, &cq->child_list); @@ -16210,7 +16101,7 @@ lpfc_mrq_create(struct lpfc_hba *phba, struct lpfc_queue **hrqp, hrq->subtype = subtype; hrq->host_index = 0; hrq->hba_index = 0; - hrq->entry_repost = LPFC_RQ_REPOST; + hrq->notify_interval = LPFC_RQ_NOTIFY_INTRVL; drq->db_format = LPFC_DB_RING_FORMAT; drq->db_regaddr = phba->sli4_hba.RQDBregaddr; @@ -16219,7 +16110,7 @@ lpfc_mrq_create(struct lpfc_hba *phba, struct lpfc_queue **hrqp, drq->subtype = subtype; drq->host_index = 0; drq->hba_index = 0; - drq->entry_repost = LPFC_RQ_REPOST; + drq->notify_interval = LPFC_RQ_NOTIFY_INTRVL; list_add_tail(&hrq->list, &cq->child_list); list_add_tail(&drq->list, &cq->child_list); @@ -16279,6 +16170,7 @@ lpfc_eq_destroy(struct lpfc_hba *phba, struct lpfc_queue *eq) /* sanity check on queue memory */ if (!eq) return -ENODEV; + mbox = mempool_alloc(eq->phba->mbox_mem_pool, GFP_KERNEL); if (!mbox) return -ENOMEM; @@ -16828,22 +16720,21 @@ lpfc_sli4_post_sgl_list(struct lpfc_hba *phba, } /** - * lpfc_sli4_post_scsi_sgl_block - post a block of scsi sgl list to firmware + * lpfc_sli4_post_io_sgl_block - post a block of nvme sgl list to firmware * @phba: pointer to lpfc hba data structure. - * @sblist: pointer to scsi buffer list. + * @nblist: pointer to nvme buffer list. * @count: number of scsi buffers on the list. * * This routine is invoked to post a block of @count scsi sgl pages from a - * SCSI buffer list @sblist to the HBA using non-embedded mailbox command. + * SCSI buffer list @nblist to the HBA using non-embedded mailbox command. * No Lock is held. * **/ -int -lpfc_sli4_post_scsi_sgl_block(struct lpfc_hba *phba, - struct list_head *sblist, - int count) +static int +lpfc_sli4_post_io_sgl_block(struct lpfc_hba *phba, struct list_head *nblist, + int count) { - struct lpfc_scsi_buf *psb; + struct lpfc_io_buf *lpfc_ncmd; struct lpfc_mbx_post_uembed_sgl_page1 *sgl; struct sgl_page_pairs *sgl_pg_pairs; void *viraddr; @@ -16861,25 +16752,25 @@ lpfc_sli4_post_scsi_sgl_block(struct lpfc_hba *phba, sizeof(union lpfc_sli4_cfg_shdr) + sizeof(uint32_t); if (reqlen > SLI4_PAGE_SIZE) { lpfc_printf_log(phba, KERN_WARNING, LOG_INIT, - "0217 Block sgl registration required DMA " + "6118 Block sgl registration required DMA " "size (%d) great than a page\n", reqlen); return -ENOMEM; } mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL); if (!mbox) { lpfc_printf_log(phba, KERN_ERR, LOG_INIT, - "0283 Failed to allocate mbox cmd memory\n"); + "6119 Failed to allocate mbox cmd memory\n"); return -ENOMEM; } /* Allocate DMA memory and set up the non-embedded mailbox command */ alloclen = lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_FCOE, - LPFC_MBOX_OPCODE_FCOE_POST_SGL_PAGES, reqlen, - LPFC_SLI4_MBX_NEMBED); + LPFC_MBOX_OPCODE_FCOE_POST_SGL_PAGES, + reqlen, LPFC_SLI4_MBX_NEMBED); if (alloclen < reqlen) { lpfc_printf_log(phba, KERN_ERR, LOG_INIT, - "2561 Allocated DMA memory size (%d) is " + "6120 Allocated DMA memory size (%d) is " "less than the requested DMA memory " "size (%d)\n", alloclen, reqlen); lpfc_sli4_mbox_cmd_free(phba, mbox); @@ -16894,14 +16785,15 @@ lpfc_sli4_post_scsi_sgl_block(struct lpfc_hba *phba, sgl_pg_pairs = &sgl->sgl_pg_pairs; pg_pairs = 0; - list_for_each_entry(psb, sblist, list) { + list_for_each_entry(lpfc_ncmd, nblist, list) { /* Set up the sge entry */ sgl_pg_pairs->sgl_pg0_addr_lo = - cpu_to_le32(putPaddrLow(psb->dma_phys_bpl)); + cpu_to_le32(putPaddrLow(lpfc_ncmd->dma_phys_sgl)); sgl_pg_pairs->sgl_pg0_addr_hi = - cpu_to_le32(putPaddrHigh(psb->dma_phys_bpl)); + cpu_to_le32(putPaddrHigh(lpfc_ncmd->dma_phys_sgl)); if (phba->cfg_sg_dma_buf_size > SGL_PAGE_SIZE) - pdma_phys_bpl1 = psb->dma_phys_bpl + SGL_PAGE_SIZE; + pdma_phys_bpl1 = lpfc_ncmd->dma_phys_sgl + + SGL_PAGE_SIZE; else pdma_phys_bpl1 = 0; sgl_pg_pairs->sgl_pg1_addr_lo = @@ -16910,7 +16802,7 @@ lpfc_sli4_post_scsi_sgl_block(struct lpfc_hba *phba, cpu_to_le32(putPaddrHigh(pdma_phys_bpl1)); /* Keep the first xritag on the list */ if (pg_pairs == 0) - xritag_start = psb->cur_iocbq.sli4_xritag; + xritag_start = lpfc_ncmd->cur_iocbq.sli4_xritag; sgl_pg_pairs++; pg_pairs++; } @@ -16919,20 +16811,20 @@ lpfc_sli4_post_scsi_sgl_block(struct lpfc_hba *phba, /* Perform endian conversion if necessary */ sgl->word0 = cpu_to_le32(sgl->word0); - if (!phba->sli4_hba.intr_enable) + if (!phba->sli4_hba.intr_enable) { rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL); - else { + } else { mbox_tmo = lpfc_mbox_tmo_val(phba, mbox); rc = lpfc_sli_issue_mbox_wait(phba, mbox, mbox_tmo); } - shdr = (union lpfc_sli4_cfg_shdr *) &sgl->cfg_shdr; + shdr = (union lpfc_sli4_cfg_shdr *)&sgl->cfg_shdr; shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response); shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response); if (rc != MBX_TIMEOUT) lpfc_sli4_mbox_cmd_free(phba, mbox); if (shdr_status || shdr_add_status || rc) { lpfc_printf_log(phba, KERN_ERR, LOG_SLI, - "2564 POST_SGL_BLOCK mailbox command failed " + "6125 POST_SGL_BLOCK mailbox command failed " "status x%x add_status x%x mbx status x%x\n", shdr_status, shdr_add_status, rc); rc = -ENXIO; @@ -16941,6 +16833,134 @@ lpfc_sli4_post_scsi_sgl_block(struct lpfc_hba *phba, } /** + * lpfc_sli4_post_io_sgl_list - Post blocks of nvme buffer sgls from a list + * @phba: pointer to lpfc hba data structure. + * @post_nblist: pointer to the nvme buffer list. + * + * This routine walks a list of nvme buffers that was passed in. It attempts + * to construct blocks of nvme buffer sgls which contains contiguous xris and + * uses the non-embedded SGL block post mailbox commands to post to the port. + * For single NVME buffer sgl with non-contiguous xri, if any, it shall use + * embedded SGL post mailbox command for posting. The @post_nblist passed in + * must be local list, thus no lock is needed when manipulate the list. + * + * Returns: 0 = failure, non-zero number of successfully posted buffers. + **/ +int +lpfc_sli4_post_io_sgl_list(struct lpfc_hba *phba, + struct list_head *post_nblist, int sb_count) +{ + struct lpfc_io_buf *lpfc_ncmd, *lpfc_ncmd_next; + int status, sgl_size; + int post_cnt = 0, block_cnt = 0, num_posting = 0, num_posted = 0; + dma_addr_t pdma_phys_sgl1; + int last_xritag = NO_XRI; + int cur_xritag; + LIST_HEAD(prep_nblist); + LIST_HEAD(blck_nblist); + LIST_HEAD(nvme_nblist); + + /* sanity check */ + if (sb_count <= 0) + return -EINVAL; + + sgl_size = phba->cfg_sg_dma_buf_size; + list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next, post_nblist, list) { + list_del_init(&lpfc_ncmd->list); + block_cnt++; + if ((last_xritag != NO_XRI) && + (lpfc_ncmd->cur_iocbq.sli4_xritag != last_xritag + 1)) { + /* a hole in xri block, form a sgl posting block */ + list_splice_init(&prep_nblist, &blck_nblist); + post_cnt = block_cnt - 1; + /* prepare list for next posting block */ + list_add_tail(&lpfc_ncmd->list, &prep_nblist); + block_cnt = 1; + } else { + /* prepare list for next posting block */ + list_add_tail(&lpfc_ncmd->list, &prep_nblist); + /* enough sgls for non-embed sgl mbox command */ + if (block_cnt == LPFC_NEMBED_MBOX_SGL_CNT) { + list_splice_init(&prep_nblist, &blck_nblist); + post_cnt = block_cnt; + block_cnt = 0; + } + } + num_posting++; + last_xritag = lpfc_ncmd->cur_iocbq.sli4_xritag; + + /* end of repost sgl list condition for NVME buffers */ + if (num_posting == sb_count) { + if (post_cnt == 0) { + /* last sgl posting block */ + list_splice_init(&prep_nblist, &blck_nblist); + post_cnt = block_cnt; + } else if (block_cnt == 1) { + /* last single sgl with non-contiguous xri */ + if (sgl_size > SGL_PAGE_SIZE) + pdma_phys_sgl1 = + lpfc_ncmd->dma_phys_sgl + + SGL_PAGE_SIZE; + else + pdma_phys_sgl1 = 0; + cur_xritag = lpfc_ncmd->cur_iocbq.sli4_xritag; + status = lpfc_sli4_post_sgl( + phba, lpfc_ncmd->dma_phys_sgl, + pdma_phys_sgl1, cur_xritag); + if (status) { + /* Post error. Buffer unavailable. */ + lpfc_ncmd->flags |= + LPFC_SBUF_NOT_POSTED; + } else { + /* Post success. Bffer available. */ + lpfc_ncmd->flags &= + ~LPFC_SBUF_NOT_POSTED; + lpfc_ncmd->status = IOSTAT_SUCCESS; + num_posted++; + } + /* success, put on NVME buffer sgl list */ + list_add_tail(&lpfc_ncmd->list, &nvme_nblist); + } + } + + /* continue until a nembed page worth of sgls */ + if (post_cnt == 0) + continue; + + /* post block of NVME buffer list sgls */ + status = lpfc_sli4_post_io_sgl_block(phba, &blck_nblist, + post_cnt); + + /* don't reset xirtag due to hole in xri block */ + if (block_cnt == 0) + last_xritag = NO_XRI; + + /* reset NVME buffer post count for next round of posting */ + post_cnt = 0; + + /* put posted NVME buffer-sgl posted on NVME buffer sgl list */ + while (!list_empty(&blck_nblist)) { + list_remove_head(&blck_nblist, lpfc_ncmd, + struct lpfc_io_buf, list); + if (status) { + /* Post error. Mark buffer unavailable. */ + lpfc_ncmd->flags |= LPFC_SBUF_NOT_POSTED; + } else { + /* Post success, Mark buffer available. */ + lpfc_ncmd->flags &= ~LPFC_SBUF_NOT_POSTED; + lpfc_ncmd->status = IOSTAT_SUCCESS; + num_posted++; + } + list_add_tail(&lpfc_ncmd->list, &nvme_nblist); + } + } + /* Push NVME buffers with sgl posted to the available list */ + lpfc_io_buf_replenish(phba, &nvme_nblist); + + return num_posted; +} + +/** * lpfc_fc_frame_check - Check that this frame is a valid frame to handle * @phba: pointer to lpfc_hba struct that the frame was received on * @fc_hdr: A pointer to the FC Header data (In Big Endian Format) @@ -19500,7 +19520,7 @@ lpfc_drain_txq(struct lpfc_hba *phba) if (phba->link_flag & LS_MDS_LOOPBACK) { /* MDS WQE are posted only to first WQ*/ - wq = phba->sli4_hba.fcp_wq[0]; + wq = phba->sli4_hba.hdwq[0].fcp_wq; if (unlikely(!wq)) return 0; pring = wq->pring; @@ -19708,7 +19728,7 @@ lpfc_wqe_bpl2sgl(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeq, * @pwqe: Pointer to command WQE. **/ int -lpfc_sli4_issue_wqe(struct lpfc_hba *phba, uint32_t ring_number, +lpfc_sli4_issue_wqe(struct lpfc_hba *phba, struct lpfc_sli4_hdw_queue *qp, struct lpfc_iocbq *pwqe) { union lpfc_wqe128 *wqe = &pwqe->wqe; @@ -19722,7 +19742,8 @@ lpfc_sli4_issue_wqe(struct lpfc_hba *phba, uint32_t ring_number, /* NVME_LS and NVME_LS ABTS requests. */ if (pwqe->iocb_flag & LPFC_IO_NVME_LS) { pring = phba->sli4_hba.nvmels_wq->pring; - spin_lock_irqsave(&pring->ring_lock, iflags); + lpfc_qp_spin_lock_irqsave(&pring->ring_lock, iflags, + qp, wq_access); sglq = __lpfc_sli_get_els_sglq(phba, pwqe); if (!sglq) { spin_unlock_irqrestore(&pring->ring_lock, iflags); @@ -19750,12 +19771,13 @@ lpfc_sli4_issue_wqe(struct lpfc_hba *phba, uint32_t ring_number, /* NVME_FCREQ and NVME_ABTS requests */ if (pwqe->iocb_flag & LPFC_IO_NVME) { /* Get the IO distribution (hba_wqidx) for WQ assignment. */ - pring = phba->sli4_hba.nvme_wq[pwqe->hba_wqidx]->pring; + wq = qp->nvme_wq; + pring = wq->pring; - spin_lock_irqsave(&pring->ring_lock, iflags); - wq = phba->sli4_hba.nvme_wq[pwqe->hba_wqidx]; - bf_set(wqe_cqid, &wqe->generic.wqe_com, - phba->sli4_hba.nvme_cq[pwqe->hba_wqidx]->queue_id); + bf_set(wqe_cqid, &wqe->generic.wqe_com, qp->nvme_cq_map); + + lpfc_qp_spin_lock_irqsave(&pring->ring_lock, iflags, + qp, wq_access); ret = lpfc_sli4_wq_put(wq, wqe); if (ret) { spin_unlock_irqrestore(&pring->ring_lock, iflags); @@ -19769,9 +19791,9 @@ lpfc_sli4_issue_wqe(struct lpfc_hba *phba, uint32_t ring_number, /* NVMET requests */ if (pwqe->iocb_flag & LPFC_IO_NVMET) { /* Get the IO distribution (hba_wqidx) for WQ assignment. */ - pring = phba->sli4_hba.nvme_wq[pwqe->hba_wqidx]->pring; + wq = qp->nvme_wq; + pring = wq->pring; - spin_lock_irqsave(&pring->ring_lock, iflags); ctxp = pwqe->context2; sglq = ctxp->ctxbuf->sglq; if (pwqe->sli4_xritag == NO_XRI) { @@ -19780,9 +19802,10 @@ lpfc_sli4_issue_wqe(struct lpfc_hba *phba, uint32_t ring_number, } bf_set(wqe_xri_tag, &pwqe->wqe.xmit_bls_rsp.wqe_com, pwqe->sli4_xritag); - wq = phba->sli4_hba.nvme_wq[pwqe->hba_wqidx]; - bf_set(wqe_cqid, &wqe->generic.wqe_com, - phba->sli4_hba.nvme_cq[pwqe->hba_wqidx]->queue_id); + bf_set(wqe_cqid, &wqe->generic.wqe_com, qp->nvme_cq_map); + + lpfc_qp_spin_lock_irqsave(&pring->ring_lock, iflags, + qp, wq_access); ret = lpfc_sli4_wq_put(wq, wqe); if (ret) { spin_unlock_irqrestore(&pring->ring_lock, iflags); @@ -19794,3 +19817,647 @@ lpfc_sli4_issue_wqe(struct lpfc_hba *phba, uint32_t ring_number, } return WQE_ERROR; } + +#ifdef LPFC_MXP_STAT +/** + * lpfc_snapshot_mxp - Snapshot pbl, pvt and busy count + * @phba: pointer to lpfc hba data structure. + * @hwqid: belong to which HWQ. + * + * The purpose of this routine is to take a snapshot of pbl, pvt and busy count + * 15 seconds after a test case is running. + * + * The user should call lpfc_debugfs_multixripools_write before running a test + * case to clear stat_snapshot_taken. Then the user starts a test case. During + * test case is running, stat_snapshot_taken is incremented by 1 every time when + * this routine is called from heartbeat timer. When stat_snapshot_taken is + * equal to LPFC_MXP_SNAPSHOT_TAKEN, a snapshot is taken. + **/ +void lpfc_snapshot_mxp(struct lpfc_hba *phba, u32 hwqid) +{ + struct lpfc_sli4_hdw_queue *qp; + struct lpfc_multixri_pool *multixri_pool; + struct lpfc_pvt_pool *pvt_pool; + struct lpfc_pbl_pool *pbl_pool; + u32 txcmplq_cnt; + + qp = &phba->sli4_hba.hdwq[hwqid]; + multixri_pool = qp->p_multixri_pool; + if (!multixri_pool) + return; + + if (multixri_pool->stat_snapshot_taken == LPFC_MXP_SNAPSHOT_TAKEN) { + pvt_pool = &qp->p_multixri_pool->pvt_pool; + pbl_pool = &qp->p_multixri_pool->pbl_pool; + txcmplq_cnt = qp->fcp_wq->pring->txcmplq_cnt; + if (qp->nvme_wq) + txcmplq_cnt += qp->nvme_wq->pring->txcmplq_cnt; + + multixri_pool->stat_pbl_count = pbl_pool->count; + multixri_pool->stat_pvt_count = pvt_pool->count; + multixri_pool->stat_busy_count = txcmplq_cnt; + } + + multixri_pool->stat_snapshot_taken++; +} +#endif + +/** + * lpfc_adjust_pvt_pool_count - Adjust private pool count + * @phba: pointer to lpfc hba data structure. + * @hwqid: belong to which HWQ. + * + * This routine moves some XRIs from private to public pool when private pool + * is not busy. + **/ +void lpfc_adjust_pvt_pool_count(struct lpfc_hba *phba, u32 hwqid) +{ + struct lpfc_multixri_pool *multixri_pool; + u32 io_req_count; + u32 prev_io_req_count; + + multixri_pool = phba->sli4_hba.hdwq[hwqid].p_multixri_pool; + if (!multixri_pool) + return; + io_req_count = multixri_pool->io_req_count; + prev_io_req_count = multixri_pool->prev_io_req_count; + + if (prev_io_req_count != io_req_count) { + /* Private pool is busy */ + multixri_pool->prev_io_req_count = io_req_count; + } else { + /* Private pool is not busy. + * Move XRIs from private to public pool. + */ + lpfc_move_xri_pvt_to_pbl(phba, hwqid); + } +} + +/** + * lpfc_adjust_high_watermark - Adjust high watermark + * @phba: pointer to lpfc hba data structure. + * @hwqid: belong to which HWQ. + * + * This routine sets high watermark as number of outstanding XRIs, + * but make sure the new value is between xri_limit/2 and xri_limit. + **/ +void lpfc_adjust_high_watermark(struct lpfc_hba *phba, u32 hwqid) +{ + u32 new_watermark; + u32 watermark_max; + u32 watermark_min; + u32 xri_limit; + u32 txcmplq_cnt; + u32 abts_io_bufs; + struct lpfc_multixri_pool *multixri_pool; + struct lpfc_sli4_hdw_queue *qp; + + qp = &phba->sli4_hba.hdwq[hwqid]; + multixri_pool = qp->p_multixri_pool; + if (!multixri_pool) + return; + xri_limit = multixri_pool->xri_limit; + + watermark_max = xri_limit; + watermark_min = xri_limit / 2; + + txcmplq_cnt = qp->fcp_wq->pring->txcmplq_cnt; + abts_io_bufs = qp->abts_scsi_io_bufs; + if (qp->nvme_wq) { + txcmplq_cnt += qp->nvme_wq->pring->txcmplq_cnt; + abts_io_bufs += qp->abts_nvme_io_bufs; + } + + new_watermark = txcmplq_cnt + abts_io_bufs; + new_watermark = min(watermark_max, new_watermark); + new_watermark = max(watermark_min, new_watermark); + multixri_pool->pvt_pool.high_watermark = new_watermark; + +#ifdef LPFC_MXP_STAT + multixri_pool->stat_max_hwm = max(multixri_pool->stat_max_hwm, + new_watermark); +#endif +} + +/** + * lpfc_move_xri_pvt_to_pbl - Move some XRIs from private to public pool + * @phba: pointer to lpfc hba data structure. + * @hwqid: belong to which HWQ. + * + * This routine is called from hearbeat timer when pvt_pool is idle. + * All free XRIs are moved from private to public pool on hwqid with 2 steps. + * The first step moves (all - low_watermark) amount of XRIs. + * The second step moves the rest of XRIs. + **/ +void lpfc_move_xri_pvt_to_pbl(struct lpfc_hba *phba, u32 hwqid) +{ + struct lpfc_pbl_pool *pbl_pool; + struct lpfc_pvt_pool *pvt_pool; + struct lpfc_sli4_hdw_queue *qp; + struct lpfc_io_buf *lpfc_ncmd; + struct lpfc_io_buf *lpfc_ncmd_next; + unsigned long iflag; + struct list_head tmp_list; + u32 tmp_count; + + qp = &phba->sli4_hba.hdwq[hwqid]; + pbl_pool = &qp->p_multixri_pool->pbl_pool; + pvt_pool = &qp->p_multixri_pool->pvt_pool; + tmp_count = 0; + + lpfc_qp_spin_lock_irqsave(&pbl_pool->lock, iflag, qp, mv_to_pub_pool); + lpfc_qp_spin_lock(&pvt_pool->lock, qp, mv_from_pvt_pool); + + if (pvt_pool->count > pvt_pool->low_watermark) { + /* Step 1: move (all - low_watermark) from pvt_pool + * to pbl_pool + */ + + /* Move low watermark of bufs from pvt_pool to tmp_list */ + INIT_LIST_HEAD(&tmp_list); + list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next, + &pvt_pool->list, list) { + list_move_tail(&lpfc_ncmd->list, &tmp_list); + tmp_count++; + if (tmp_count >= pvt_pool->low_watermark) + break; + } + + /* Move all bufs from pvt_pool to pbl_pool */ + list_splice_init(&pvt_pool->list, &pbl_pool->list); + + /* Move all bufs from tmp_list to pvt_pool */ + list_splice(&tmp_list, &pvt_pool->list); + + pbl_pool->count += (pvt_pool->count - tmp_count); + pvt_pool->count = tmp_count; + } else { + /* Step 2: move the rest from pvt_pool to pbl_pool */ + list_splice_init(&pvt_pool->list, &pbl_pool->list); + pbl_pool->count += pvt_pool->count; + pvt_pool->count = 0; + } + + spin_unlock(&pvt_pool->lock); + spin_unlock_irqrestore(&pbl_pool->lock, iflag); +} + +/** + * _lpfc_move_xri_pbl_to_pvt - Move some XRIs from public to private pool + * @phba: pointer to lpfc hba data structure + * @pbl_pool: specified public free XRI pool + * @pvt_pool: specified private free XRI pool + * @count: number of XRIs to move + * + * This routine tries to move some free common bufs from the specified pbl_pool + * to the specified pvt_pool. It might move less than count XRIs if there's not + * enough in public pool. + * + * Return: + * true - if XRIs are successfully moved from the specified pbl_pool to the + * specified pvt_pool + * false - if the specified pbl_pool is empty or locked by someone else + **/ +static bool +_lpfc_move_xri_pbl_to_pvt(struct lpfc_hba *phba, struct lpfc_sli4_hdw_queue *qp, + struct lpfc_pbl_pool *pbl_pool, + struct lpfc_pvt_pool *pvt_pool, u32 count) +{ + struct lpfc_io_buf *lpfc_ncmd; + struct lpfc_io_buf *lpfc_ncmd_next; + unsigned long iflag; + int ret; + + ret = spin_trylock_irqsave(&pbl_pool->lock, iflag); + if (ret) { + if (pbl_pool->count) { + /* Move a batch of XRIs from public to private pool */ + lpfc_qp_spin_lock(&pvt_pool->lock, qp, mv_to_pvt_pool); + list_for_each_entry_safe(lpfc_ncmd, + lpfc_ncmd_next, + &pbl_pool->list, + list) { + list_move_tail(&lpfc_ncmd->list, + &pvt_pool->list); + pvt_pool->count++; + pbl_pool->count--; + count--; + if (count == 0) + break; + } + + spin_unlock(&pvt_pool->lock); + spin_unlock_irqrestore(&pbl_pool->lock, iflag); + return true; + } + spin_unlock_irqrestore(&pbl_pool->lock, iflag); + } + + return false; +} + +/** + * lpfc_move_xri_pbl_to_pvt - Move some XRIs from public to private pool + * @phba: pointer to lpfc hba data structure. + * @hwqid: belong to which HWQ. + * @count: number of XRIs to move + * + * This routine tries to find some free common bufs in one of public pools with + * Round Robin method. The search always starts from local hwqid, then the next + * HWQ which was found last time (rrb_next_hwqid). Once a public pool is found, + * a batch of free common bufs are moved to private pool on hwqid. + * It might move less than count XRIs if there's not enough in public pool. + **/ +void lpfc_move_xri_pbl_to_pvt(struct lpfc_hba *phba, u32 hwqid, u32 count) +{ + struct lpfc_multixri_pool *multixri_pool; + struct lpfc_multixri_pool *next_multixri_pool; + struct lpfc_pvt_pool *pvt_pool; + struct lpfc_pbl_pool *pbl_pool; + struct lpfc_sli4_hdw_queue *qp; + u32 next_hwqid; + u32 hwq_count; + int ret; + + qp = &phba->sli4_hba.hdwq[hwqid]; + multixri_pool = qp->p_multixri_pool; + pvt_pool = &multixri_pool->pvt_pool; + pbl_pool = &multixri_pool->pbl_pool; + + /* Check if local pbl_pool is available */ + ret = _lpfc_move_xri_pbl_to_pvt(phba, qp, pbl_pool, pvt_pool, count); + if (ret) { +#ifdef LPFC_MXP_STAT + multixri_pool->local_pbl_hit_count++; +#endif + return; + } + + hwq_count = phba->cfg_hdw_queue; + + /* Get the next hwqid which was found last time */ + next_hwqid = multixri_pool->rrb_next_hwqid; + + do { + /* Go to next hwq */ + next_hwqid = (next_hwqid + 1) % hwq_count; + + next_multixri_pool = + phba->sli4_hba.hdwq[next_hwqid].p_multixri_pool; + pbl_pool = &next_multixri_pool->pbl_pool; + + /* Check if the public free xri pool is available */ + ret = _lpfc_move_xri_pbl_to_pvt( + phba, qp, pbl_pool, pvt_pool, count); + + /* Exit while-loop if success or all hwqid are checked */ + } while (!ret && next_hwqid != multixri_pool->rrb_next_hwqid); + + /* Starting point for the next time */ + multixri_pool->rrb_next_hwqid = next_hwqid; + + if (!ret) { + /* stats: all public pools are empty*/ + multixri_pool->pbl_empty_count++; + } + +#ifdef LPFC_MXP_STAT + if (ret) { + if (next_hwqid == hwqid) + multixri_pool->local_pbl_hit_count++; + else + multixri_pool->other_pbl_hit_count++; + } +#endif +} + +/** + * lpfc_keep_pvt_pool_above_lowwm - Keep pvt_pool above low watermark + * @phba: pointer to lpfc hba data structure. + * @qp: belong to which HWQ. + * + * This routine get a batch of XRIs from pbl_pool if pvt_pool is less than + * low watermark. + **/ +void lpfc_keep_pvt_pool_above_lowwm(struct lpfc_hba *phba, u32 hwqid) +{ + struct lpfc_multixri_pool *multixri_pool; + struct lpfc_pvt_pool *pvt_pool; + + multixri_pool = phba->sli4_hba.hdwq[hwqid].p_multixri_pool; + pvt_pool = &multixri_pool->pvt_pool; + + if (pvt_pool->count < pvt_pool->low_watermark) + lpfc_move_xri_pbl_to_pvt(phba, hwqid, XRI_BATCH); +} + +/** + * lpfc_release_io_buf - Return one IO buf back to free pool + * @phba: pointer to lpfc hba data structure. + * @lpfc_ncmd: IO buf to be returned. + * @qp: belong to which HWQ. + * + * This routine returns one IO buf back to free pool. If this is an urgent IO, + * the IO buf is returned to expedite pool. If cfg_xri_rebalancing==1, + * the IO buf is returned to pbl_pool or pvt_pool based on watermark and + * xri_limit. If cfg_xri_rebalancing==0, the IO buf is returned to + * lpfc_io_buf_list_put. + **/ +void lpfc_release_io_buf(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_ncmd, + struct lpfc_sli4_hdw_queue *qp) +{ + unsigned long iflag; + struct lpfc_pbl_pool *pbl_pool; + struct lpfc_pvt_pool *pvt_pool; + struct lpfc_epd_pool *epd_pool; + u32 txcmplq_cnt; + u32 xri_owned; + u32 xri_limit; + u32 abts_io_bufs; + + /* MUST zero fields if buffer is reused by another protocol */ + lpfc_ncmd->nvmeCmd = NULL; + lpfc_ncmd->cur_iocbq.wqe_cmpl = NULL; + lpfc_ncmd->cur_iocbq.iocb_cmpl = NULL; + + if (phba->cfg_xri_rebalancing) { + if (lpfc_ncmd->expedite) { + /* Return to expedite pool */ + epd_pool = &phba->epd_pool; + spin_lock_irqsave(&epd_pool->lock, iflag); + list_add_tail(&lpfc_ncmd->list, &epd_pool->list); + epd_pool->count++; + spin_unlock_irqrestore(&epd_pool->lock, iflag); + return; + } + + /* Avoid invalid access if an IO sneaks in and is being rejected + * just _after_ xri pools are destroyed in lpfc_offline. + * Nothing much can be done at this point. + */ + if (!qp->p_multixri_pool) + return; + + pbl_pool = &qp->p_multixri_pool->pbl_pool; + pvt_pool = &qp->p_multixri_pool->pvt_pool; + + txcmplq_cnt = qp->fcp_wq->pring->txcmplq_cnt; + abts_io_bufs = qp->abts_scsi_io_bufs; + if (qp->nvme_wq) { + txcmplq_cnt += qp->nvme_wq->pring->txcmplq_cnt; + abts_io_bufs += qp->abts_nvme_io_bufs; + } + + xri_owned = pvt_pool->count + txcmplq_cnt + abts_io_bufs; + xri_limit = qp->p_multixri_pool->xri_limit; + +#ifdef LPFC_MXP_STAT + if (xri_owned <= xri_limit) + qp->p_multixri_pool->below_limit_count++; + else + qp->p_multixri_pool->above_limit_count++; +#endif + + /* XRI goes to either public or private free xri pool + * based on watermark and xri_limit + */ + if ((pvt_pool->count < pvt_pool->low_watermark) || + (xri_owned < xri_limit && + pvt_pool->count < pvt_pool->high_watermark)) { + lpfc_qp_spin_lock_irqsave(&pvt_pool->lock, iflag, + qp, free_pvt_pool); + list_add_tail(&lpfc_ncmd->list, + &pvt_pool->list); + pvt_pool->count++; + spin_unlock_irqrestore(&pvt_pool->lock, iflag); + } else { + lpfc_qp_spin_lock_irqsave(&pbl_pool->lock, iflag, + qp, free_pub_pool); + list_add_tail(&lpfc_ncmd->list, + &pbl_pool->list); + pbl_pool->count++; + spin_unlock_irqrestore(&pbl_pool->lock, iflag); + } + } else { + lpfc_qp_spin_lock_irqsave(&qp->io_buf_list_put_lock, iflag, + qp, free_xri); + list_add_tail(&lpfc_ncmd->list, + &qp->lpfc_io_buf_list_put); + qp->put_io_bufs++; + spin_unlock_irqrestore(&qp->io_buf_list_put_lock, + iflag); + } +} + +/** + * lpfc_get_io_buf_from_private_pool - Get one free IO buf from private pool + * @phba: pointer to lpfc hba data structure. + * @pvt_pool: pointer to private pool data structure. + * @ndlp: pointer to lpfc nodelist data structure. + * + * This routine tries to get one free IO buf from private pool. + * + * Return: + * pointer to one free IO buf - if private pool is not empty + * NULL - if private pool is empty + **/ +static struct lpfc_io_buf * +lpfc_get_io_buf_from_private_pool(struct lpfc_hba *phba, + struct lpfc_sli4_hdw_queue *qp, + struct lpfc_pvt_pool *pvt_pool, + struct lpfc_nodelist *ndlp) +{ + struct lpfc_io_buf *lpfc_ncmd; + struct lpfc_io_buf *lpfc_ncmd_next; + unsigned long iflag; + + lpfc_qp_spin_lock_irqsave(&pvt_pool->lock, iflag, qp, alloc_pvt_pool); + list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next, + &pvt_pool->list, list) { + if (lpfc_test_rrq_active( + phba, ndlp, lpfc_ncmd->cur_iocbq.sli4_lxritag)) + continue; + list_del(&lpfc_ncmd->list); + pvt_pool->count--; + spin_unlock_irqrestore(&pvt_pool->lock, iflag); + return lpfc_ncmd; + } + spin_unlock_irqrestore(&pvt_pool->lock, iflag); + + return NULL; +} + +/** + * lpfc_get_io_buf_from_expedite_pool - Get one free IO buf from expedite pool + * @phba: pointer to lpfc hba data structure. + * + * This routine tries to get one free IO buf from expedite pool. + * + * Return: + * pointer to one free IO buf - if expedite pool is not empty + * NULL - if expedite pool is empty + **/ +static struct lpfc_io_buf * +lpfc_get_io_buf_from_expedite_pool(struct lpfc_hba *phba) +{ + struct lpfc_io_buf *lpfc_ncmd; + struct lpfc_io_buf *lpfc_ncmd_next; + unsigned long iflag; + struct lpfc_epd_pool *epd_pool; + + epd_pool = &phba->epd_pool; + lpfc_ncmd = NULL; + + spin_lock_irqsave(&epd_pool->lock, iflag); + if (epd_pool->count > 0) { + list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next, + &epd_pool->list, list) { + list_del(&lpfc_ncmd->list); + epd_pool->count--; + break; + } + } + spin_unlock_irqrestore(&epd_pool->lock, iflag); + + return lpfc_ncmd; +} + +/** + * lpfc_get_io_buf_from_multixri_pools - Get one free IO bufs + * @phba: pointer to lpfc hba data structure. + * @ndlp: pointer to lpfc nodelist data structure. + * @hwqid: belong to which HWQ + * @expedite: 1 means this request is urgent. + * + * This routine will do the following actions and then return a pointer to + * one free IO buf. + * + * 1. If private free xri count is empty, move some XRIs from public to + * private pool. + * 2. Get one XRI from private free xri pool. + * 3. If we fail to get one from pvt_pool and this is an expedite request, + * get one free xri from expedite pool. + * + * Note: ndlp is only used on SCSI side for RRQ testing. + * The caller should pass NULL for ndlp on NVME side. + * + * Return: + * pointer to one free IO buf - if private pool is not empty + * NULL - if private pool is empty + **/ +static struct lpfc_io_buf * +lpfc_get_io_buf_from_multixri_pools(struct lpfc_hba *phba, + struct lpfc_nodelist *ndlp, + int hwqid, int expedite) +{ + struct lpfc_sli4_hdw_queue *qp; + struct lpfc_multixri_pool *multixri_pool; + struct lpfc_pvt_pool *pvt_pool; + struct lpfc_io_buf *lpfc_ncmd; + + qp = &phba->sli4_hba.hdwq[hwqid]; + lpfc_ncmd = NULL; + multixri_pool = qp->p_multixri_pool; + pvt_pool = &multixri_pool->pvt_pool; + multixri_pool->io_req_count++; + + /* If pvt_pool is empty, move some XRIs from public to private pool */ + if (pvt_pool->count == 0) + lpfc_move_xri_pbl_to_pvt(phba, hwqid, XRI_BATCH); + + /* Get one XRI from private free xri pool */ + lpfc_ncmd = lpfc_get_io_buf_from_private_pool(phba, qp, pvt_pool, ndlp); + + if (lpfc_ncmd) { + lpfc_ncmd->hdwq = qp; + lpfc_ncmd->hdwq_no = hwqid; + } else if (expedite) { + /* If we fail to get one from pvt_pool and this is an expedite + * request, get one free xri from expedite pool. + */ + lpfc_ncmd = lpfc_get_io_buf_from_expedite_pool(phba); + } + + return lpfc_ncmd; +} + +static inline struct lpfc_io_buf * +lpfc_io_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, int idx) +{ + struct lpfc_sli4_hdw_queue *qp; + struct lpfc_io_buf *lpfc_cmd, *lpfc_cmd_next; + + qp = &phba->sli4_hba.hdwq[idx]; + list_for_each_entry_safe(lpfc_cmd, lpfc_cmd_next, + &qp->lpfc_io_buf_list_get, list) { + if (lpfc_test_rrq_active(phba, ndlp, + lpfc_cmd->cur_iocbq.sli4_lxritag)) + continue; + + if (lpfc_cmd->flags & LPFC_SBUF_NOT_POSTED) + continue; + + list_del_init(&lpfc_cmd->list); + qp->get_io_bufs--; + lpfc_cmd->hdwq = qp; + lpfc_cmd->hdwq_no = idx; + return lpfc_cmd; + } + return NULL; +} + +/** + * lpfc_get_io_buf - Get one IO buffer from free pool + * @phba: The HBA for which this call is being executed. + * @ndlp: pointer to lpfc nodelist data structure. + * @hwqid: belong to which HWQ + * @expedite: 1 means this request is urgent. + * + * This routine gets one IO buffer from free pool. If cfg_xri_rebalancing==1, + * removes a IO buffer from multiXRI pools. If cfg_xri_rebalancing==0, removes + * a IO buffer from head of @hdwq io_buf_list and returns to caller. + * + * Note: ndlp is only used on SCSI side for RRQ testing. + * The caller should pass NULL for ndlp on NVME side. + * + * Return codes: + * NULL - Error + * Pointer to lpfc_io_buf - Success + **/ +struct lpfc_io_buf *lpfc_get_io_buf(struct lpfc_hba *phba, + struct lpfc_nodelist *ndlp, + u32 hwqid, int expedite) +{ + struct lpfc_sli4_hdw_queue *qp; + unsigned long iflag; + struct lpfc_io_buf *lpfc_cmd; + + qp = &phba->sli4_hba.hdwq[hwqid]; + lpfc_cmd = NULL; + + if (phba->cfg_xri_rebalancing) + lpfc_cmd = lpfc_get_io_buf_from_multixri_pools( + phba, ndlp, hwqid, expedite); + else { + lpfc_qp_spin_lock_irqsave(&qp->io_buf_list_get_lock, iflag, + qp, alloc_xri_get); + if (qp->get_io_bufs > LPFC_NVME_EXPEDITE_XRICNT || expedite) + lpfc_cmd = lpfc_io_buf(phba, ndlp, hwqid); + if (!lpfc_cmd) { + lpfc_qp_spin_lock(&qp->io_buf_list_put_lock, + qp, alloc_xri_put); + list_splice(&qp->lpfc_io_buf_list_put, + &qp->lpfc_io_buf_list_get); + qp->get_io_bufs += qp->put_io_bufs; + INIT_LIST_HEAD(&qp->lpfc_io_buf_list_put); + qp->put_io_bufs = 0; + spin_unlock(&qp->io_buf_list_put_lock); + if (qp->get_io_bufs > LPFC_NVME_EXPEDITE_XRICNT || + expedite) + lpfc_cmd = lpfc_io_buf(phba, ndlp, hwqid); + } + spin_unlock_irqrestore(&qp->io_buf_list_get_lock, iflag); + } + + return lpfc_cmd; +} diff --git a/drivers/scsi/lpfc/lpfc_sli.h b/drivers/scsi/lpfc/lpfc_sli.h index 7abb395bb64a..7a1a761efdd6 100644 --- a/drivers/scsi/lpfc/lpfc_sli.h +++ b/drivers/scsi/lpfc/lpfc_sli.h @@ -1,7 +1,7 @@ /******************************************************************* * This file is part of the Emulex Linux Device Driver for * * Fibre Channel Host Bus Adapters. * - * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term * + * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term * * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * * Copyright (C) 2004-2016 Emulex. All rights reserved. * * EMULEX and SLI are trademarks of Emulex. * @@ -20,6 +20,10 @@ * included with this package. * *******************************************************************/ +#if defined(CONFIG_DEBUG_FS) && !defined(CONFIG_SCSI_LPFC_DEBUG_FS) +#define CONFIG_SCSI_LPFC_DEBUG_FS +#endif + /* forward declaration for LPFC_IOCB_t's use */ struct lpfc_hba; struct lpfc_vport; @@ -33,6 +37,7 @@ typedef enum _lpfc_ctx_cmd { struct lpfc_cq_event { struct list_head list; + uint16_t hdwq; union { struct lpfc_mcqe mcqe_cmpl; struct lpfc_acqe_link acqe_link; @@ -351,3 +356,85 @@ struct lpfc_sli { #define LPFC_MBOX_SLI4_CONFIG_EXTENDED_TMO 300 /* Timeout for other flash-based outstanding mbox command (Seconds) */ #define LPFC_MBOX_TMO_FLASH_CMD 300 + +struct lpfc_io_buf { + /* Common fields */ + struct list_head list; + void *data; + dma_addr_t dma_handle; + dma_addr_t dma_phys_sgl; + struct sli4_sge *dma_sgl; + struct lpfc_iocbq cur_iocbq; + struct lpfc_sli4_hdw_queue *hdwq; + uint16_t hdwq_no; + uint16_t cpu; + + struct lpfc_nodelist *ndlp; + uint32_t timeout; + uint16_t flags; /* TBD convert exch_busy to flags */ +#define LPFC_SBUF_XBUSY 0x1 /* SLI4 hba reported XB on WCQE cmpl */ +#define LPFC_SBUF_BUMP_QDEPTH 0x2 /* bumped queue depth counter */ + /* External DIF device IO conversions */ +#define LPFC_SBUF_NORMAL_DIF 0x4 /* normal mode to insert/strip */ +#define LPFC_SBUF_PASS_DIF 0x8 /* insert/strip mode to passthru */ +#define LPFC_SBUF_NOT_POSTED 0x10 /* SGL failed post to FW. */ + uint16_t exch_busy; /* SLI4 hba reported XB on complete WCQE */ + uint16_t status; /* From IOCB Word 7- ulpStatus */ + uint32_t result; /* From IOCB Word 4. */ + + uint32_t seg_cnt; /* Number of scatter-gather segments returned by + * dma_map_sg. The driver needs this for calls + * to dma_unmap_sg. + */ + unsigned long start_time; + spinlock_t buf_lock; /* lock used in case of simultaneous abort */ + bool expedite; /* this is an expedite io_buf */ + + union { + /* SCSI specific fields */ + struct { + struct scsi_cmnd *pCmd; + struct lpfc_rport_data *rdata; + uint32_t prot_seg_cnt; /* seg_cnt's counterpart for + * protection data + */ + + /* + * data and dma_handle are the kernel virtual and bus + * address of the dma-able buffer containing the + * fcp_cmd, fcp_rsp and a scatter gather bde list that + * supports the sg_tablesize value. + */ + struct fcp_cmnd *fcp_cmnd; + struct fcp_rsp *fcp_rsp; + + wait_queue_head_t *waitq; + +#ifdef CONFIG_SCSI_LPFC_DEBUG_FS + /* Used to restore any changes to protection data for + * error injection + */ + void *prot_data_segment; + uint32_t prot_data; + uint32_t prot_data_type; +#define LPFC_INJERR_REFTAG 1 +#define LPFC_INJERR_APPTAG 2 +#define LPFC_INJERR_GUARD 3 +#endif + }; + + /* NVME specific fields */ + struct { + struct nvmefc_fcp_req *nvmeCmd; + uint16_t qidx; + +#ifdef CONFIG_SCSI_LPFC_DEBUG_FS + uint64_t ts_cmd_start; + uint64_t ts_last_cmd; + uint64_t ts_cmd_wqput; + uint64_t ts_isr_cmpl; + uint64_t ts_data_nvme; +#endif + }; + }; +}; diff --git a/drivers/scsi/lpfc/lpfc_sli4.h b/drivers/scsi/lpfc/lpfc_sli4.h index 6b2d2350e2c6..40c85091c805 100644 --- a/drivers/scsi/lpfc/lpfc_sli4.h +++ b/drivers/scsi/lpfc/lpfc_sli4.h @@ -1,7 +1,7 @@ /******************************************************************* * This file is part of the Emulex Linux Device Driver for * * Fibre Channel Host Bus Adapters. * - * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term * + * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term * * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * * Copyright (C) 2009-2016 Emulex. All rights reserved. * * EMULEX and SLI are trademarks of Emulex. * @@ -20,6 +20,10 @@ * included with this package. * *******************************************************************/ +#if defined(CONFIG_DEBUG_FS) && !defined(CONFIG_SCSI_LPFC_DEBUG_FS) +#define CONFIG_SCSI_LPFC_DEBUG_FS +#endif + #define LPFC_ACTIVE_MBOX_WAIT_CNT 100 #define LPFC_XRI_EXCH_BUSY_WAIT_TMO 10000 #define LPFC_XRI_EXCH_BUSY_WAIT_T1 10 @@ -36,14 +40,12 @@ #define LPFC_NEMBED_MBOX_SGL_CNT 254 /* Multi-queue arrangement for FCP EQ/CQ/WQ tuples */ -#define LPFC_HBA_IO_CHAN_MIN 0 -#define LPFC_HBA_IO_CHAN_MAX 32 -#define LPFC_FCP_IO_CHAN_DEF 4 -#define LPFC_NVME_IO_CHAN_DEF 0 - -/* Number of channels used for Flash Optimized Fabric (FOF) operations */ +#define LPFC_HBA_HDWQ_MIN 0 +#define LPFC_HBA_HDWQ_MAX 128 +#define LPFC_HBA_HDWQ_DEF 0 -#define LPFC_FOF_IO_CHAN_NUM 1 +/* Common buffer size to accomidate SCSI and NVME IO buffers */ +#define LPFC_COMMON_IO_BUF_SZ 768 /* * Provide the default FCF Record attributes used by the driver @@ -152,28 +154,58 @@ struct lpfc_queue { struct list_head child_list; struct list_head page_list; struct list_head sgl_list; + struct list_head cpu_list; uint32_t entry_count; /* Number of entries to support on the queue */ uint32_t entry_size; /* Size of each queue entry. */ - uint32_t entry_repost; /* Count of entries before doorbell is rung */ -#define LPFC_EQ_REPOST 8 -#define LPFC_MQ_REPOST 8 -#define LPFC_CQ_REPOST 64 -#define LPFC_RQ_REPOST 64 -#define LPFC_RELEASE_NOTIFICATION_INTERVAL 32 /* For WQs */ + uint32_t notify_interval; /* Queue Notification Interval + * For chip->host queues (EQ, CQ, RQ): + * specifies the interval (number of + * entries) where the doorbell is rung to + * notify the chip of entry consumption. + * For host->chip queues (WQ): + * specifies the interval (number of + * entries) where consumption CQE is + * requested to indicate WQ entries + * consumed by the chip. + * Not used on an MQ. + */ +#define LPFC_EQ_NOTIFY_INTRVL 16 +#define LPFC_CQ_NOTIFY_INTRVL 16 +#define LPFC_WQ_NOTIFY_INTRVL 16 +#define LPFC_RQ_NOTIFY_INTRVL 16 + uint32_t max_proc_limit; /* Queue Processing Limit + * For chip->host queues (EQ, CQ): + * specifies the maximum number of + * entries to be consumed in one + * processing iteration sequence. Queue + * will be rearmed after each iteration. + * Not used on an MQ, RQ or WQ. + */ +#define LPFC_EQ_MAX_PROC_LIMIT 256 +#define LPFC_CQ_MIN_PROC_LIMIT 64 +#define LPFC_CQ_MAX_PROC_LIMIT LPFC_CQE_EXP_COUNT // 4096 +#define LPFC_CQ_DEF_MAX_PROC_LIMIT LPFC_CQE_DEF_COUNT // 1024 +#define LPFC_CQ_MIN_THRESHOLD_TO_POLL 64 +#define LPFC_CQ_MAX_THRESHOLD_TO_POLL LPFC_CQ_DEF_MAX_PROC_LIMIT +#define LPFC_CQ_DEF_THRESHOLD_TO_POLL LPFC_CQ_DEF_MAX_PROC_LIMIT + uint32_t queue_claimed; /* indicates queue is being processed */ uint32_t queue_id; /* Queue ID assigned by the hardware */ uint32_t assoc_qid; /* Queue ID associated with, for CQ/WQ/MQ */ uint32_t host_index; /* The host's index for putting or getting */ uint32_t hba_index; /* The last known hba index for get or put */ + uint32_t q_mode; struct lpfc_sli_ring *pring; /* ptr to io ring associated with q */ struct lpfc_rqb *rqbp; /* ptr to RQ buffers */ - uint32_t q_mode; uint16_t page_count; /* Number of pages allocated for this queue */ uint16_t page_size; /* size of page allocated for this queue */ #define LPFC_EXPANDED_PAGE_SIZE 16384 #define LPFC_DEFAULT_PAGE_SIZE 4096 - uint16_t chann; /* IO channel this queue is associated with */ + uint16_t chann; /* Hardware Queue association WQ/CQ */ + /* CPU affinity for EQ */ +#define LPFC_FIND_BY_EQ 0 +#define LPFC_FIND_BY_HDWQ 1 uint8_t db_format; #define LPFC_DB_RING_FORMAT 0x01 #define LPFC_DB_LIST_FORMAT 0x02 @@ -212,10 +244,14 @@ struct lpfc_queue { #define RQ_buf_posted q_cnt_3 #define RQ_rcv_buf q_cnt_4 - struct work_struct irqwork; - struct work_struct spwork; + struct work_struct irqwork; + struct work_struct spwork; + struct delayed_work sched_irqwork; + struct delayed_work sched_spwork; uint64_t isr_timestamp; + uint16_t hdwq; + uint16_t last_cpu; /* most recent cpu */ uint8_t qe_valid; struct lpfc_queue *assoc_qp; union sli4_qe qe[1]; /* array to index entries (must be last) */ @@ -428,11 +464,6 @@ struct lpfc_hba_eq_hdl { uint32_t idx; char handler_name[LPFC_SLI4_HANDLER_NAME_SZ]; struct lpfc_hba *phba; - atomic_t hba_eq_in_use; - struct cpumask *cpumask; - /* CPU affinitsed to or 0xffffffff if multiple */ - uint32_t cpu; -#define LPFC_MULTI_CPU_AFFINITY 0xffffffff }; /*BB Credit recovery value*/ @@ -526,11 +557,165 @@ struct lpfc_vector_map_info { uint16_t phys_id; uint16_t core_id; uint16_t irq; - uint16_t channel_id; + uint16_t eq; + uint16_t hdwq; + uint16_t hyper; }; #define LPFC_VECTOR_MAP_EMPTY 0xffff +/* Multi-XRI pool */ +#define XRI_BATCH 8 + +struct lpfc_pbl_pool { + struct list_head list; + u32 count; + spinlock_t lock; /* lock for pbl_pool*/ +}; + +struct lpfc_pvt_pool { + u32 low_watermark; + u32 high_watermark; + + struct list_head list; + u32 count; + spinlock_t lock; /* lock for pvt_pool */ +}; + +struct lpfc_multixri_pool { + u32 xri_limit; + + /* Starting point when searching a pbl_pool with round-robin method */ + u32 rrb_next_hwqid; + + /* Used by lpfc_adjust_pvt_pool_count. + * io_req_count is incremented by 1 during IO submission. The heartbeat + * handler uses these two variables to determine if pvt_pool is idle or + * busy. + */ + u32 prev_io_req_count; + u32 io_req_count; + + /* statistics */ + u32 pbl_empty_count; +#ifdef LPFC_MXP_STAT + u32 above_limit_count; + u32 below_limit_count; + u32 local_pbl_hit_count; + u32 other_pbl_hit_count; + u32 stat_max_hwm; + +#define LPFC_MXP_SNAPSHOT_TAKEN 3 /* snapshot is taken at 3rd heartbeats */ + u32 stat_pbl_count; + u32 stat_pvt_count; + u32 stat_busy_count; + u32 stat_snapshot_taken; +#endif + + /* TODO: Separate pvt_pool into get and put list */ + struct lpfc_pbl_pool pbl_pool; /* Public free XRI pool */ + struct lpfc_pvt_pool pvt_pool; /* Private free XRI pool */ +}; + +struct lpfc_fc4_ctrl_stat { + u32 input_requests; + u32 output_requests; + u32 control_requests; + u32 io_cmpls; +}; + +#ifdef LPFC_HDWQ_LOCK_STAT +struct lpfc_lock_stat { + uint32_t alloc_xri_get; + uint32_t alloc_xri_put; + uint32_t free_xri; + uint32_t wq_access; + uint32_t alloc_pvt_pool; + uint32_t mv_from_pvt_pool; + uint32_t mv_to_pub_pool; + uint32_t mv_to_pvt_pool; + uint32_t free_pub_pool; + uint32_t free_pvt_pool; +}; +#endif + +struct lpfc_eq_intr_info { + struct list_head list; + uint32_t icnt; +}; + /* SLI4 HBA data structure entries */ +struct lpfc_sli4_hdw_queue { + /* Pointers to the constructed SLI4 queues */ + struct lpfc_queue *hba_eq; /* Event queues for HBA */ + struct lpfc_queue *fcp_cq; /* Fast-path FCP compl queue */ + struct lpfc_queue *nvme_cq; /* Fast-path NVME compl queue */ + struct lpfc_queue *fcp_wq; /* Fast-path FCP work queue */ + struct lpfc_queue *nvme_wq; /* Fast-path NVME work queue */ + uint16_t fcp_cq_map; + uint16_t nvme_cq_map; + + /* Keep track of IO buffers for this hardware queue */ + spinlock_t io_buf_list_get_lock; /* Common buf alloc list lock */ + struct list_head lpfc_io_buf_list_get; + spinlock_t io_buf_list_put_lock; /* Common buf free list lock */ + struct list_head lpfc_io_buf_list_put; + spinlock_t abts_scsi_buf_list_lock; /* list of aborted SCSI IOs */ + struct list_head lpfc_abts_scsi_buf_list; + spinlock_t abts_nvme_buf_list_lock; /* list of aborted NVME IOs */ + struct list_head lpfc_abts_nvme_buf_list; + uint32_t total_io_bufs; + uint32_t get_io_bufs; + uint32_t put_io_bufs; + uint32_t empty_io_bufs; + uint32_t abts_scsi_io_bufs; + uint32_t abts_nvme_io_bufs; + + /* Multi-XRI pool per HWQ */ + struct lpfc_multixri_pool *p_multixri_pool; + + /* FC-4 Stats counters */ + struct lpfc_fc4_ctrl_stat nvme_cstat; + struct lpfc_fc4_ctrl_stat scsi_cstat; +#ifdef LPFC_HDWQ_LOCK_STAT + struct lpfc_lock_stat lock_conflict; +#endif + +#ifdef CONFIG_SCSI_LPFC_DEBUG_FS +#define LPFC_CHECK_CPU_CNT 128 + uint32_t cpucheck_rcv_io[LPFC_CHECK_CPU_CNT]; + uint32_t cpucheck_xmt_io[LPFC_CHECK_CPU_CNT]; + uint32_t cpucheck_cmpl_io[LPFC_CHECK_CPU_CNT]; +#endif +}; + +#ifdef LPFC_HDWQ_LOCK_STAT +/* compile time trylock stats */ +#define lpfc_qp_spin_lock_irqsave(lock, flag, qp, lstat) \ + { \ + int only_once = 1; \ + while (spin_trylock_irqsave(lock, flag) == 0) { \ + if (only_once) { \ + only_once = 0; \ + qp->lock_conflict.lstat++; \ + } \ + } \ + } +#define lpfc_qp_spin_lock(lock, qp, lstat) \ + { \ + int only_once = 1; \ + while (spin_trylock(lock) == 0) { \ + if (only_once) { \ + only_once = 0; \ + qp->lock_conflict.lstat++; \ + } \ + } \ + } +#else +#define lpfc_qp_spin_lock_irqsave(lock, flag, qp, lstat) \ + spin_lock_irqsave(lock, flag) +#define lpfc_qp_spin_lock(lock, qp, lstat) spin_lock(lock) +#endif + struct lpfc_sli4_hba { void __iomem *conf_regs_memmap_p; /* Kernel memory mapped address for * config space registers @@ -599,21 +784,19 @@ struct lpfc_sli4_hba { struct lpfc_hba_eq_hdl *hba_eq_hdl; /* HBA per-WQ handle */ void (*sli4_eq_clr_intr)(struct lpfc_queue *q); - uint32_t (*sli4_eq_release)(struct lpfc_queue *q, bool arm); - uint32_t (*sli4_cq_release)(struct lpfc_queue *q, bool arm); + void (*sli4_write_eq_db)(struct lpfc_hba *phba, struct lpfc_queue *eq, + uint32_t count, bool arm); + void (*sli4_write_cq_db)(struct lpfc_hba *phba, struct lpfc_queue *cq, + uint32_t count, bool arm); /* Pointers to the constructed SLI4 queues */ - struct lpfc_queue **hba_eq; /* Event queues for HBA */ - struct lpfc_queue **fcp_cq; /* Fast-path FCP compl queue */ - struct lpfc_queue **nvme_cq; /* Fast-path NVME compl queue */ + struct lpfc_sli4_hdw_queue *hdwq; + struct list_head lpfc_wq_list; + + /* Pointers to the constructed SLI4 queues for NVMET */ struct lpfc_queue **nvmet_cqset; /* Fast-path NVMET CQ Set queues */ struct lpfc_queue **nvmet_mrq_hdr; /* Fast-path NVMET hdr MRQs */ struct lpfc_queue **nvmet_mrq_data; /* Fast-path NVMET data MRQs */ - struct lpfc_queue **fcp_wq; /* Fast-path FCP work queue */ - struct lpfc_queue **nvme_wq; /* Fast-path NVME work queue */ - uint16_t *fcp_cq_map; - uint16_t *nvme_cq_map; - struct list_head lpfc_wq_list; struct lpfc_queue *mbx_cq; /* Slow-path mailbox complete queue */ struct lpfc_queue *els_cq; /* Slow-path ELS response complete queue */ @@ -631,13 +814,7 @@ struct lpfc_sli4_hba { uint32_t ulp0_mode; /* ULP0 protocol mode */ uint32_t ulp1_mode; /* ULP1 protocol mode */ - struct lpfc_queue *fof_eq; /* Flash Optimized Fabric Event queue */ - /* Optimized Access Storage specific queues/structures */ - - struct lpfc_queue *oas_cq; /* OAS completion queue */ - struct lpfc_queue *oas_wq; /* OAS Work queue */ - struct lpfc_sli_ring *oas_ring; uint64_t oas_next_lun; uint8_t oas_next_tgt_wwpn[8]; uint8_t oas_next_vpt_wwpn[8]; @@ -663,22 +840,22 @@ struct lpfc_sli4_hba { uint16_t rpi_hdrs_in_use; /* must post rpi hdrs if set. */ uint16_t next_xri; /* last_xri - max_cfg_param.xri_base = used */ uint16_t next_rpi; - uint16_t nvme_xri_max; - uint16_t nvme_xri_cnt; - uint16_t nvme_xri_start; - uint16_t scsi_xri_max; - uint16_t scsi_xri_cnt; - uint16_t scsi_xri_start; + uint16_t io_xri_max; + uint16_t io_xri_cnt; + uint16_t io_xri_start; uint16_t els_xri_cnt; uint16_t nvmet_xri_cnt; uint16_t nvmet_io_wait_cnt; uint16_t nvmet_io_wait_total; + uint16_t cq_max; + struct lpfc_queue **cq_lookup; struct list_head lpfc_els_sgl_list; struct list_head lpfc_abts_els_sgl_list; + spinlock_t abts_scsi_buf_list_lock; /* list of aborted SCSI IOs */ + struct list_head lpfc_abts_scsi_buf_list; struct list_head lpfc_nvmet_sgl_list; + spinlock_t abts_nvmet_buf_list_lock; /* list of aborted NVMET IOs */ struct list_head lpfc_abts_nvmet_ctx_list; - struct list_head lpfc_abts_scsi_buf_list; - struct list_head lpfc_abts_nvme_buf_list; struct list_head lpfc_nvmet_io_wait_list; struct lpfc_nvmet_ctx_info *nvmet_ctx_info; struct lpfc_sglq **lpfc_sglq_active_list; @@ -707,17 +884,16 @@ struct lpfc_sli4_hba { #define LPFC_SLI4_PPNAME_NON 0 #define LPFC_SLI4_PPNAME_GET 1 struct lpfc_iov iov; - spinlock_t abts_nvme_buf_list_lock; /* list of aborted SCSI IOs */ - spinlock_t abts_scsi_buf_list_lock; /* list of aborted SCSI IOs */ spinlock_t sgl_list_lock; /* list of aborted els IOs */ spinlock_t nvmet_io_wait_lock; /* IOs waiting for ctx resources */ uint32_t physical_port; /* CPU to vector mapping information */ struct lpfc_vector_map_info *cpu_map; - uint16_t num_online_cpu; + uint16_t num_possible_cpu; uint16_t num_present_cpu; uint16_t curr_disp_cpu; + struct lpfc_eq_intr_info __percpu *eq_info; uint32_t conf_trunk; #define lpfc_conf_trunk_port0_WORD conf_trunk #define lpfc_conf_trunk_port0_SHIFT 0 @@ -818,12 +994,12 @@ struct lpfc_queue *lpfc_sli4_queue_alloc(struct lpfc_hba *, uint32_t, uint32_t, uint32_t); void lpfc_sli4_queue_free(struct lpfc_queue *); int lpfc_eq_create(struct lpfc_hba *, struct lpfc_queue *, uint32_t); -int lpfc_modify_hba_eq_delay(struct lpfc_hba *phba, uint32_t startq, - uint32_t numq, uint32_t imax); +void lpfc_modify_hba_eq_delay(struct lpfc_hba *phba, uint32_t startq, + uint32_t numq, uint32_t usdelay); int lpfc_cq_create(struct lpfc_hba *, struct lpfc_queue *, struct lpfc_queue *, uint32_t, uint32_t); int lpfc_cq_create_set(struct lpfc_hba *phba, struct lpfc_queue **cqp, - struct lpfc_queue **eqp, uint32_t type, + struct lpfc_sli4_hdw_queue *hdwq, uint32_t type, uint32_t subtype); int32_t lpfc_mq_create(struct lpfc_hba *, struct lpfc_queue *, struct lpfc_queue *, uint32_t); @@ -843,12 +1019,10 @@ int lpfc_rq_destroy(struct lpfc_hba *, struct lpfc_queue *, int lpfc_sli4_queue_setup(struct lpfc_hba *); void lpfc_sli4_queue_unset(struct lpfc_hba *); int lpfc_sli4_post_sgl(struct lpfc_hba *, dma_addr_t, dma_addr_t, uint16_t); -int lpfc_sli4_repost_scsi_sgl_list(struct lpfc_hba *); -int lpfc_repost_nvme_sgl_list(struct lpfc_hba *phba); +int lpfc_repost_io_sgl_list(struct lpfc_hba *phba); uint16_t lpfc_sli4_next_xritag(struct lpfc_hba *); void lpfc_sli4_free_xri(struct lpfc_hba *, int); int lpfc_sli4_post_async_mbox(struct lpfc_hba *); -int lpfc_sli4_post_scsi_sgl_block(struct lpfc_hba *, struct list_head *, int); struct lpfc_cq_event *__lpfc_sli4_cq_event_alloc(struct lpfc_hba *); struct lpfc_cq_event *lpfc_sli4_cq_event_alloc(struct lpfc_hba *); void __lpfc_sli4_cq_event_release(struct lpfc_hba *, struct lpfc_cq_event *); @@ -868,9 +1042,9 @@ int lpfc_sli4_resume_rpi(struct lpfc_nodelist *, void lpfc_sli4_fcp_xri_abort_event_proc(struct lpfc_hba *); void lpfc_sli4_els_xri_abort_event_proc(struct lpfc_hba *); void lpfc_sli4_fcp_xri_aborted(struct lpfc_hba *, - struct sli4_wcqe_xri_aborted *); + struct sli4_wcqe_xri_aborted *, int); void lpfc_sli4_nvme_xri_aborted(struct lpfc_hba *phba, - struct sli4_wcqe_xri_aborted *axri); + struct sli4_wcqe_xri_aborted *axri, int idx); void lpfc_sli4_nvmet_xri_aborted(struct lpfc_hba *phba, struct sli4_wcqe_xri_aborted *axri); void lpfc_sli4_els_xri_aborted(struct lpfc_hba *, @@ -884,11 +1058,15 @@ int lpfc_sli4_get_els_iocb_cnt(struct lpfc_hba *); int lpfc_sli4_get_iocb_cnt(struct lpfc_hba *phba); int lpfc_sli4_init_vpi(struct lpfc_vport *); inline void lpfc_sli4_eq_clr_intr(struct lpfc_queue *); -uint32_t lpfc_sli4_cq_release(struct lpfc_queue *, bool); -uint32_t lpfc_sli4_eq_release(struct lpfc_queue *, bool); +void lpfc_sli4_write_cq_db(struct lpfc_hba *phba, struct lpfc_queue *q, + uint32_t count, bool arm); +void lpfc_sli4_write_eq_db(struct lpfc_hba *phba, struct lpfc_queue *q, + uint32_t count, bool arm); inline void lpfc_sli4_if6_eq_clr_intr(struct lpfc_queue *q); -uint32_t lpfc_sli4_if6_cq_release(struct lpfc_queue *q, bool arm); -uint32_t lpfc_sli4_if6_eq_release(struct lpfc_queue *q, bool arm); +void lpfc_sli4_if6_write_cq_db(struct lpfc_hba *phba, struct lpfc_queue *q, + uint32_t count, bool arm); +void lpfc_sli4_if6_write_eq_db(struct lpfc_hba *phba, struct lpfc_queue *q, + uint32_t count, bool arm); void lpfc_sli4_fcfi_unreg(struct lpfc_hba *, uint16_t); int lpfc_sli4_fcf_scan_read_fcf_rec(struct lpfc_hba *, uint16_t); int lpfc_sli4_fcf_rr_read_fcf_rec(struct lpfc_hba *, uint16_t); diff --git a/drivers/scsi/lpfc/lpfc_version.h b/drivers/scsi/lpfc/lpfc_version.h index 3f4398ffb567..43fd693cf042 100644 --- a/drivers/scsi/lpfc/lpfc_version.h +++ b/drivers/scsi/lpfc/lpfc_version.h @@ -1,7 +1,7 @@ /******************************************************************* * This file is part of the Emulex Linux Device Driver for * * Fibre Channel Host Bus Adapters. * - * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term * + * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term * * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * * Copyright (C) 2004-2016 Emulex. All rights reserved. * * EMULEX and SLI are trademarks of Emulex. * @@ -20,7 +20,7 @@ * included with this package. * *******************************************************************/ -#define LPFC_DRIVER_VERSION "12.0.0.10" +#define LPFC_DRIVER_VERSION "12.2.0.0" #define LPFC_DRIVER_NAME "lpfc" /* Used for SLI 2/3 */ diff --git a/drivers/scsi/lpfc/lpfc_vport.c b/drivers/scsi/lpfc/lpfc_vport.c index 102a011ff6d4..343bc71d4615 100644 --- a/drivers/scsi/lpfc/lpfc_vport.c +++ b/drivers/scsi/lpfc/lpfc_vport.c @@ -1,7 +1,7 @@ /******************************************************************* * This file is part of the Emulex Linux Device Driver for * * Fibre Channel Host Bus Adapters. * - * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term * + * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term * * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * * Copyright (C) 2004-2016 Emulex. All rights reserved. * * EMULEX and SLI are trademarks of Emulex. * @@ -313,11 +313,11 @@ lpfc_vport_create(struct fc_vport *fc_vport, bool disable) goto error_out; } - /* NPIV is not supported if HBA has NVME enabled */ - if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { + /* NPIV is not supported if HBA has NVME Target enabled */ + if (phba->nvmet_support) { lpfc_printf_log(phba, KERN_ERR, LOG_VPORT, "3189 Create VPORT failed: " - "NPIV is not supported on NVME\n"); + "NPIV is not supported on NVME Target\n"); rc = VPORT_INVAL; goto error_out; } @@ -403,6 +403,9 @@ lpfc_vport_create(struct fc_vport *fc_vport, bool disable) /* Set the DFT_LUN_Q_DEPTH accordingly */ vport->cfg_lun_queue_depth = phba->pport->cfg_lun_queue_depth; + /* Only the physical port can support NVME for now */ + vport->cfg_enable_fc4_type = LPFC_ENABLE_FCP; + *(struct lpfc_vport **)fc_vport->dd_data = vport; vport->fc_vport = fc_vport; @@ -415,22 +418,6 @@ lpfc_vport_create(struct fc_vport *fc_vport, bool disable) vport->fdmi_port_mask = phba->pport->fdmi_port_mask; } - if ((phba->nvmet_support == 0) && - ((phba->cfg_enable_fc4_type == LPFC_ENABLE_BOTH) || - (phba->cfg_enable_fc4_type == LPFC_ENABLE_NVME))) { - /* Create NVME binding with nvme_fc_transport. This - * ensures the vport is initialized. - */ - rc = lpfc_nvme_create_localport(vport); - if (rc) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, - "6003 %s status x%x\n", - "NVME registration failed, ", - rc); - goto error_out; - } - } - /* * In SLI4, the vpi must be activated before it can be used * by the port. diff --git a/drivers/scsi/megaraid/megaraid_sas.h b/drivers/scsi/megaraid/megaraid_sas.h index 16536c41f0c5..6fd57f7f0b1e 100644 --- a/drivers/scsi/megaraid/megaraid_sas.h +++ b/drivers/scsi/megaraid/megaraid_sas.h @@ -33,8 +33,8 @@ /* * MegaRAID SAS Driver meta data */ -#define MEGASAS_VERSION "07.707.50.00-rc1" -#define MEGASAS_RELDATE "December 18, 2018" +#define MEGASAS_VERSION "07.707.51.00-rc1" +#define MEGASAS_RELDATE "February 7, 2019" /* * Device IDs @@ -790,6 +790,38 @@ struct MR_LD_TARGETID_LIST { u8 targetId[MAX_LOGICAL_DRIVES_EXT]; }; +struct MR_HOST_DEVICE_LIST_ENTRY { + struct { + union { + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u8 reserved:7; + u8 is_sys_pd:1; +#else + u8 is_sys_pd:1; + u8 reserved:7; +#endif + } bits; + u8 byte; + } u; + } flags; + u8 scsi_type; + __le16 target_id; + u8 reserved[4]; + __le64 sas_addr[2]; +} __packed; + +struct MR_HOST_DEVICE_LIST { + __le32 size; + __le32 count; + __le32 reserved[2]; + struct MR_HOST_DEVICE_LIST_ENTRY host_device_list[1]; +} __packed; + +#define HOST_DEVICE_LIST_SZ (sizeof(struct MR_HOST_DEVICE_LIST) + \ + (sizeof(struct MR_HOST_DEVICE_LIST_ENTRY) * \ + (MEGASAS_MAX_PD + MAX_LOGICAL_DRIVES_EXT - 1))) + /* * SAS controller properties @@ -870,13 +902,17 @@ struct megasas_ctrl_prop { u8 viewSpace; struct { #if defined(__BIG_ENDIAN_BITFIELD) - u16 reserved2:11; + u16 reserved3:9; + u16 enable_fw_dev_list:1; + u16 reserved2:1; u16 enable_snap_dump:1; u16 reserved1:4; #else u16 reserved1:4; u16 enable_snap_dump:1; - u16 reserved2:11; + u16 reserved2:1; + u16 enable_fw_dev_list:1; + u16 reserved3:9; #endif } on_off_properties2; }; @@ -1685,7 +1721,8 @@ union megasas_sgl_frame { typedef union _MFI_CAPABILITIES { struct { #if defined(__BIG_ENDIAN_BITFIELD) - u32 reserved:17; + u32 reserved:16; + u32 support_fw_exposed_dev_list:1; u32 support_nvme_passthru:1; u32 support_64bit_mode:1; u32 support_pd_map_target_id:1; @@ -1717,7 +1754,8 @@ typedef union _MFI_CAPABILITIES { u32 support_pd_map_target_id:1; u32 support_64bit_mode:1; u32 support_nvme_passthru:1; - u32 reserved:17; + u32 support_fw_exposed_dev_list:1; + u32 reserved:16; #endif } mfi_capabilities; __le32 reg; @@ -2202,6 +2240,9 @@ struct megasas_instance { struct MR_LD_TARGETID_LIST *ld_targetid_list_buf; dma_addr_t ld_targetid_list_buf_h; + struct MR_HOST_DEVICE_LIST *host_device_list_buf; + dma_addr_t host_device_list_buf_h; + struct MR_SNAPDUMP_PROPERTIES *snapdump_prop; dma_addr_t snapdump_prop_h; @@ -2337,6 +2378,7 @@ struct megasas_instance { u8 task_abort_tmo; u8 max_reset_tmo; u8 snapdump_wait_time; + u8 enable_fw_dev_list; }; struct MR_LD_VF_MAP { u32 size; diff --git a/drivers/scsi/megaraid/megaraid_sas_base.c b/drivers/scsi/megaraid/megaraid_sas_base.c index fcbff83c0097..dace907744a5 100644 --- a/drivers/scsi/megaraid/megaraid_sas_base.c +++ b/drivers/scsi/megaraid/megaraid_sas_base.c @@ -4188,6 +4188,7 @@ int megasas_alloc_cmds(struct megasas_instance *instance) if (megasas_create_frame_pool(instance)) { dev_printk(KERN_DEBUG, &instance->pdev->dev, "Error creating frame DMA pool\n"); megasas_free_cmds(instance); + return -ENOMEM; } return 0; @@ -4634,6 +4635,123 @@ megasas_ld_list_query(struct megasas_instance *instance, u8 query_type) return ret; } +/** + * dcmd.opcode - MR_DCMD_CTRL_DEVICE_LIST_GET + * dcmd.mbox - reserved + * dcmd.sge IN - ptr to return MR_HOST_DEVICE_LIST structure + * Desc: This DCMD will return the combined device list + * Status: MFI_STAT_OK - List returned successfully + * MFI_STAT_INVALID_CMD - Firmware support for the feature has been + * disabled + * @instance: Adapter soft state + * @is_probe: Driver probe check + * Return: 0 if DCMD succeeded + * non-zero if failed + */ +int +megasas_host_device_list_query(struct megasas_instance *instance, + bool is_probe) +{ + int ret, i, target_id; + struct megasas_cmd *cmd; + struct megasas_dcmd_frame *dcmd; + struct MR_HOST_DEVICE_LIST *ci; + u32 count; + dma_addr_t ci_h; + + ci = instance->host_device_list_buf; + ci_h = instance->host_device_list_buf_h; + + cmd = megasas_get_cmd(instance); + + if (!cmd) { + dev_warn(&instance->pdev->dev, + "%s: failed to get cmd\n", + __func__); + return -ENOMEM; + } + + dcmd = &cmd->frame->dcmd; + + memset(ci, 0, sizeof(*ci)); + memset(dcmd->mbox.b, 0, MFI_MBOX_SIZE); + + dcmd->mbox.b[0] = is_probe ? 0 : 1; + dcmd->cmd = MFI_CMD_DCMD; + dcmd->cmd_status = MFI_STAT_INVALID_STATUS; + dcmd->sge_count = 1; + dcmd->flags = MFI_FRAME_DIR_READ; + dcmd->timeout = 0; + dcmd->pad_0 = 0; + dcmd->data_xfer_len = cpu_to_le32(HOST_DEVICE_LIST_SZ); + dcmd->opcode = cpu_to_le32(MR_DCMD_CTRL_DEVICE_LIST_GET); + + megasas_set_dma_settings(instance, dcmd, ci_h, HOST_DEVICE_LIST_SZ); + + if (!instance->mask_interrupts) { + ret = megasas_issue_blocked_cmd(instance, cmd, + MFI_IO_TIMEOUT_SECS); + } else { + ret = megasas_issue_polled(instance, cmd); + cmd->flags |= DRV_DCMD_SKIP_REFIRE; + } + + switch (ret) { + case DCMD_SUCCESS: + /* Fill the internal pd_list and ld_ids array based on + * targetIds returned by FW + */ + count = le32_to_cpu(ci->count); + + memset(instance->local_pd_list, 0, + MEGASAS_MAX_PD * sizeof(struct megasas_pd_list)); + memset(instance->ld_ids, 0xff, MAX_LOGICAL_DRIVES_EXT); + for (i = 0; i < count; i++) { + target_id = le16_to_cpu(ci->host_device_list[i].target_id); + if (ci->host_device_list[i].flags.u.bits.is_sys_pd) { + instance->local_pd_list[target_id].tid = target_id; + instance->local_pd_list[target_id].driveType = + ci->host_device_list[i].scsi_type; + instance->local_pd_list[target_id].driveState = + MR_PD_STATE_SYSTEM; + } else { + instance->ld_ids[target_id] = target_id; + } + } + + memcpy(instance->pd_list, instance->local_pd_list, + sizeof(instance->pd_list)); + break; + + case DCMD_TIMEOUT: + switch (dcmd_timeout_ocr_possible(instance)) { + case INITIATE_OCR: + cmd->flags |= DRV_DCMD_SKIP_REFIRE; + megasas_reset_fusion(instance->host, + MFI_IO_TIMEOUT_OCR); + break; + case KILL_ADAPTER: + megaraid_sas_kill_hba(instance); + break; + case IGNORE_TIMEOUT: + dev_info(&instance->pdev->dev, "Ignore DCMD timeout: %s %d\n", + __func__, __LINE__); + break; + } + break; + case DCMD_FAILED: + dev_err(&instance->pdev->dev, + "%s: MR_DCMD_CTRL_DEVICE_LIST_GET failed\n", + __func__); + break; + } + + if (ret != DCMD_TIMEOUT) + megasas_return_cmd(instance, cmd); + + return ret; +} + /* * megasas_update_ext_vd_details : Update details w.r.t Extended VD * instance : Controller's instance @@ -4861,6 +4979,9 @@ megasas_get_ctrl_info(struct megasas_instance *instance) (ci->properties.on_off_properties2.enable_snap_dump ? MEGASAS_DEFAULT_SNAP_DUMP_WAIT_TIME : 0); + instance->enable_fw_dev_list = + ci->properties.on_off_properties2.enable_fw_dev_list; + dev_info(&instance->pdev->dev, "controller type\t: %s(%dMB)\n", instance->is_imr ? "iMR" : "MR", @@ -5320,6 +5441,40 @@ fallback: } /** + * megasas_get_device_list - Get the PD and LD device list from FW. + * @instance: Adapter soft state + * @return: Success or failure + * + * Issue DCMDs to Firmware to get the PD and LD list. + * Based on the FW support, driver sends the HOST_DEVICE_LIST or combination + * of PD_LIST/LD_LIST_QUERY DCMDs to get the device list. + */ +static +int megasas_get_device_list(struct megasas_instance *instance) +{ + memset(instance->pd_list, 0, + (MEGASAS_MAX_PD * sizeof(struct megasas_pd_list))); + memset(instance->ld_ids, 0xff, MEGASAS_MAX_LD_IDS); + + if (instance->enable_fw_dev_list) { + if (megasas_host_device_list_query(instance, true)) + return FAILED; + } else { + if (megasas_get_pd_list(instance) < 0) { + dev_err(&instance->pdev->dev, "failed to get PD list\n"); + return FAILED; + } + + if (megasas_ld_list_query(instance, + MR_LD_QUERY_TYPE_EXPOSED_TO_HOST)) { + dev_err(&instance->pdev->dev, "failed to get LD list\n"); + return FAILED; + } + } + + return SUCCESS; +} +/** * megasas_init_fw - Initializes the FW * @instance: Adapter soft state * @@ -5571,18 +5726,13 @@ static int megasas_init_fw(struct megasas_instance *instance) megasas_setup_jbod_map(instance); - /** for passthrough - * the following function will get the PD LIST. - */ - memset(instance->pd_list, 0, - (MEGASAS_MAX_PD * sizeof(struct megasas_pd_list))); - if (megasas_get_pd_list(instance) < 0) { - dev_err(&instance->pdev->dev, "failed to get PD list\n"); + if (megasas_get_device_list(instance) != SUCCESS) { + dev_err(&instance->pdev->dev, + "%s: megasas_get_device_list failed\n", + __func__); goto fail_get_ld_pd_list; } - memset(instance->ld_ids, 0xff, MEGASAS_MAX_LD_IDS); - /* stream detection initialization */ if (instance->adapter_type >= VENTURA_SERIES) { fusion->stream_detect_by_ld = @@ -5612,10 +5762,6 @@ static int megasas_init_fw(struct megasas_instance *instance) } } - if (megasas_ld_list_query(instance, - MR_LD_QUERY_TYPE_EXPOSED_TO_HOST)) - goto fail_get_ld_pd_list; - /* * Compute the max allowed sectors per IO: The controller info has two * limits on max sectors. Driver should use the minimum of these two. @@ -6424,6 +6570,18 @@ int megasas_alloc_ctrl_dma_buffers(struct megasas_instance *instance) if (!instance->snapdump_prop) dev_err(&pdev->dev, "Failed to allocate snapdump properties buffer\n"); + + instance->host_device_list_buf = dma_alloc_coherent(&pdev->dev, + HOST_DEVICE_LIST_SZ, + &instance->host_device_list_buf_h, + GFP_KERNEL); + + if (!instance->host_device_list_buf) { + dev_err(&pdev->dev, + "Failed to allocate targetid list buffer\n"); + return -ENOMEM; + } + } instance->pd_list_buf = @@ -6573,6 +6731,13 @@ void megasas_free_ctrl_dma_buffers(struct megasas_instance *instance) sizeof(struct MR_SNAPDUMP_PROPERTIES), instance->snapdump_prop, instance->snapdump_prop_h); + + if (instance->host_device_list_buf) + dma_free_coherent(&pdev->dev, + HOST_DEVICE_LIST_SZ, + instance->host_device_list_buf, + instance->host_device_list_buf_h); + } /* @@ -6746,7 +6911,9 @@ static int megasas_probe_one(struct pci_dev *pdev, /* * Trigger SCSI to scan our drives */ - scsi_scan_host(host); + if (!instance->enable_fw_dev_list || + (instance->host_device_list_buf->count > 0)) + scsi_scan_host(host); /* * Initiate AEN (Asynchronous Event Notification) @@ -7866,6 +8033,139 @@ static inline void megasas_remove_scsi_device(struct scsi_device *sdev) scsi_device_put(sdev); } +/** + * megasas_update_device_list - Update the PD and LD device list from FW + * after an AEN event notification + * @instance: Adapter soft state + * @event_type: Indicates type of event (PD or LD event) + * + * @return: Success or failure + * + * Issue DCMDs to Firmware to update the internal device list in driver. + * Based on the FW support, driver sends the HOST_DEVICE_LIST or combination + * of PD_LIST/LD_LIST_QUERY DCMDs to get the device list. + */ +static +int megasas_update_device_list(struct megasas_instance *instance, + int event_type) +{ + int dcmd_ret = DCMD_SUCCESS; + + if (instance->enable_fw_dev_list) { + dcmd_ret = megasas_host_device_list_query(instance, false); + if (dcmd_ret != DCMD_SUCCESS) + goto out; + } else { + if (event_type & SCAN_PD_CHANNEL) { + dcmd_ret = megasas_get_pd_list(instance); + + if (dcmd_ret != DCMD_SUCCESS) + goto out; + } + + if (event_type & SCAN_VD_CHANNEL) { + if (!instance->requestorId || + (instance->requestorId && + megasas_get_ld_vf_affiliation(instance, 0))) { + dcmd_ret = megasas_ld_list_query(instance, + MR_LD_QUERY_TYPE_EXPOSED_TO_HOST); + if (dcmd_ret != DCMD_SUCCESS) + goto out; + } + } + } + +out: + return dcmd_ret; +} + +/** + * megasas_add_remove_devices - Add/remove devices to SCSI mid-layer + * after an AEN event notification + * @instance: Adapter soft state + * @scan_type: Indicates type of devices (PD/LD) to add + * @return void + */ +static +void megasas_add_remove_devices(struct megasas_instance *instance, + int scan_type) +{ + int i, j; + u16 pd_index = 0; + u16 ld_index = 0; + u16 channel = 0, id = 0; + struct Scsi_Host *host; + struct scsi_device *sdev1; + struct MR_HOST_DEVICE_LIST *targetid_list = NULL; + struct MR_HOST_DEVICE_LIST_ENTRY *targetid_entry = NULL; + + host = instance->host; + + if (instance->enable_fw_dev_list) { + targetid_list = instance->host_device_list_buf; + for (i = 0; i < targetid_list->count; i++) { + targetid_entry = &targetid_list->host_device_list[i]; + if (targetid_entry->flags.u.bits.is_sys_pd) { + channel = le16_to_cpu(targetid_entry->target_id) / + MEGASAS_MAX_DEV_PER_CHANNEL; + id = le16_to_cpu(targetid_entry->target_id) % + MEGASAS_MAX_DEV_PER_CHANNEL; + } else { + channel = MEGASAS_MAX_PD_CHANNELS + + (le16_to_cpu(targetid_entry->target_id) / + MEGASAS_MAX_DEV_PER_CHANNEL); + id = le16_to_cpu(targetid_entry->target_id) % + MEGASAS_MAX_DEV_PER_CHANNEL; + } + sdev1 = scsi_device_lookup(host, channel, id, 0); + if (!sdev1) { + scsi_add_device(host, channel, id, 0); + } else { + scsi_device_put(sdev1); + } + } + } + + if (scan_type & SCAN_PD_CHANNEL) { + for (i = 0; i < MEGASAS_MAX_PD_CHANNELS; i++) { + for (j = 0; j < MEGASAS_MAX_DEV_PER_CHANNEL; j++) { + pd_index = i * MEGASAS_MAX_DEV_PER_CHANNEL + j; + sdev1 = scsi_device_lookup(host, i, j, 0); + if (instance->pd_list[pd_index].driveState == + MR_PD_STATE_SYSTEM) { + if (!sdev1) + scsi_add_device(host, i, j, 0); + else + scsi_device_put(sdev1); + } else { + if (sdev1) + megasas_remove_scsi_device(sdev1); + } + } + } + } + + if (scan_type & SCAN_VD_CHANNEL) { + for (i = 0; i < MEGASAS_MAX_LD_CHANNELS; i++) { + for (j = 0; j < MEGASAS_MAX_DEV_PER_CHANNEL; j++) { + ld_index = (i * MEGASAS_MAX_DEV_PER_CHANNEL) + j; + sdev1 = scsi_device_lookup(host, + MEGASAS_MAX_PD_CHANNELS + i, j, 0); + if (instance->ld_ids[ld_index] != 0xff) { + if (!sdev1) + scsi_add_device(host, MEGASAS_MAX_PD_CHANNELS + i, j, 0); + else + scsi_device_put(sdev1); + } else { + if (sdev1) + megasas_remove_scsi_device(sdev1); + } + } + } + } + +} + static void megasas_aen_polling(struct work_struct *work) { @@ -7873,11 +8173,7 @@ megasas_aen_polling(struct work_struct *work) container_of(work, struct megasas_aen_event, hotplug_work.work); struct megasas_instance *instance = ev->instance; union megasas_evt_class_locale class_locale; - struct Scsi_Host *host; - struct scsi_device *sdev1; - u16 pd_index = 0; - u16 ld_index = 0; - int i, j, doscan = 0; + int event_type = 0; u32 seq_num, wait_time = MEGASAS_RESET_WAIT_TIME; int error; u8 dcmd_ret = DCMD_SUCCESS; @@ -7896,7 +8192,6 @@ megasas_aen_polling(struct work_struct *work) mutex_lock(&instance->reset_mutex); instance->ev = NULL; - host = instance->host; if (instance->evt_detail) { megasas_decode_evt(instance); @@ -7904,40 +8199,20 @@ megasas_aen_polling(struct work_struct *work) case MR_EVT_PD_INSERTED: case MR_EVT_PD_REMOVED: - dcmd_ret = megasas_get_pd_list(instance); - if (dcmd_ret == DCMD_SUCCESS) - doscan = SCAN_PD_CHANNEL; + event_type = SCAN_PD_CHANNEL; break; case MR_EVT_LD_OFFLINE: case MR_EVT_CFG_CLEARED: case MR_EVT_LD_DELETED: case MR_EVT_LD_CREATED: - if (!instance->requestorId || - (instance->requestorId && megasas_get_ld_vf_affiliation(instance, 0))) - dcmd_ret = megasas_ld_list_query(instance, MR_LD_QUERY_TYPE_EXPOSED_TO_HOST); - - if (dcmd_ret == DCMD_SUCCESS) - doscan = SCAN_VD_CHANNEL; - + event_type = SCAN_VD_CHANNEL; break; case MR_EVT_CTRL_HOST_BUS_SCAN_REQUESTED: case MR_EVT_FOREIGN_CFG_IMPORTED: case MR_EVT_LD_STATE_CHANGE: - dcmd_ret = megasas_get_pd_list(instance); - - if (dcmd_ret != DCMD_SUCCESS) - break; - - if (!instance->requestorId || - (instance->requestorId && megasas_get_ld_vf_affiliation(instance, 0))) - dcmd_ret = megasas_ld_list_query(instance, MR_LD_QUERY_TYPE_EXPOSED_TO_HOST); - - if (dcmd_ret != DCMD_SUCCESS) - break; - - doscan = SCAN_VD_CHANNEL | SCAN_PD_CHANNEL; + event_type = SCAN_PD_CHANNEL | SCAN_VD_CHANNEL; dev_info(&instance->pdev->dev, "scanning for scsi%d...\n", instance->host->host_no); break; @@ -7953,7 +8228,7 @@ megasas_aen_polling(struct work_struct *work) } break; default: - doscan = 0; + event_type = 0; break; } } else { @@ -7963,44 +8238,13 @@ megasas_aen_polling(struct work_struct *work) return; } - mutex_unlock(&instance->reset_mutex); + if (event_type) + dcmd_ret = megasas_update_device_list(instance, event_type); - if (doscan & SCAN_PD_CHANNEL) { - for (i = 0; i < MEGASAS_MAX_PD_CHANNELS; i++) { - for (j = 0; j < MEGASAS_MAX_DEV_PER_CHANNEL; j++) { - pd_index = i*MEGASAS_MAX_DEV_PER_CHANNEL + j; - sdev1 = scsi_device_lookup(host, i, j, 0); - if (instance->pd_list[pd_index].driveState == - MR_PD_STATE_SYSTEM) { - if (!sdev1) - scsi_add_device(host, i, j, 0); - else - scsi_device_put(sdev1); - } else { - if (sdev1) - megasas_remove_scsi_device(sdev1); - } - } - } - } + mutex_unlock(&instance->reset_mutex); - if (doscan & SCAN_VD_CHANNEL) { - for (i = 0; i < MEGASAS_MAX_LD_CHANNELS; i++) { - for (j = 0; j < MEGASAS_MAX_DEV_PER_CHANNEL; j++) { - ld_index = (i * MEGASAS_MAX_DEV_PER_CHANNEL) + j; - sdev1 = scsi_device_lookup(host, MEGASAS_MAX_PD_CHANNELS + i, j, 0); - if (instance->ld_ids[ld_index] != 0xff) { - if (!sdev1) - scsi_add_device(host, MEGASAS_MAX_PD_CHANNELS + i, j, 0); - else - scsi_device_put(sdev1); - } else { - if (sdev1) - megasas_remove_scsi_device(sdev1); - } - } - } - } + if (event_type && dcmd_ret == DCMD_SUCCESS) + megasas_add_remove_devices(instance, event_type); if (dcmd_ret == DCMD_SUCCESS) seq_num = le32_to_cpu(instance->evt_detail->seq_num) + 1; diff --git a/drivers/scsi/megaraid/megaraid_sas_fusion.c b/drivers/scsi/megaraid/megaraid_sas_fusion.c index 647f48a28f85..1d17128030cd 100644 --- a/drivers/scsi/megaraid/megaraid_sas_fusion.c +++ b/drivers/scsi/megaraid/megaraid_sas_fusion.c @@ -937,11 +937,9 @@ wait_and_poll(struct megasas_instance *instance, struct megasas_cmd *cmd, { int i; struct megasas_header *frame_hdr = &cmd->frame->hdr; - struct fusion_context *fusion; u32 msecs = seconds * 1000; - fusion = instance->ctrl_context; /* * Wait for cmd_status to change */ @@ -1074,6 +1072,7 @@ megasas_ioc_init_fusion(struct megasas_instance *instance) drv_ops->mfi_capabilities.support_qd_throttling = 1; drv_ops->mfi_capabilities.support_pd_map_target_id = 1; drv_ops->mfi_capabilities.support_nvme_passthru = 1; + drv_ops->mfi_capabilities.support_fw_exposed_dev_list = 1; if (instance->consistent_mask_64bit) drv_ops->mfi_capabilities.support_64bit_mode = 1; @@ -1330,7 +1329,6 @@ megasas_sync_map_info(struct megasas_instance *instance) struct megasas_cmd *cmd; struct megasas_dcmd_frame *dcmd; u16 num_lds; - u32 size_sync_info; struct fusion_context *fusion; struct MR_LD_TARGET_SYNC *ci = NULL; struct MR_DRV_RAID_MAP_ALL *map; @@ -1359,8 +1357,6 @@ megasas_sync_map_info(struct megasas_instance *instance) dcmd = &cmd->frame->dcmd; - size_sync_info = sizeof(struct MR_LD_TARGET_SYNC) *num_lds; - memset(dcmd->mbox.b, 0, MFI_MBOX_SIZE); ci = (struct MR_LD_TARGET_SYNC *) @@ -1639,15 +1635,12 @@ static inline void megasas_free_ioc_init_cmd(struct megasas_instance *instance) u32 megasas_init_adapter_fusion(struct megasas_instance *instance) { - struct megasas_register_set __iomem *reg_set; struct fusion_context *fusion; u32 scratch_pad_1; int i = 0, count; fusion = instance->ctrl_context; - reg_set = instance->reg_set; - megasas_fusion_update_can_queue(instance, PROBE_CONTEXT); /* @@ -1926,7 +1919,6 @@ static bool megasas_is_prp_possible(struct megasas_instance *instance, struct scsi_cmnd *scmd, int sge_count) { - struct fusion_context *fusion; int i; u32 data_length = 0; struct scatterlist *sg_scmd; @@ -1935,7 +1927,6 @@ megasas_is_prp_possible(struct megasas_instance *instance, mr_nvme_pg_size = max_t(u32, instance->nvme_page_size, MR_DEFAULT_NVME_PAGE_SIZE); - fusion = instance->ctrl_context; data_length = scsi_bufflen(scmd); sg_scmd = scsi_sglist(scmd); @@ -2048,12 +2039,9 @@ megasas_make_prp_nvme(struct megasas_instance *instance, struct scsi_cmnd *scmd, u32 first_prp_len; bool build_prp = false; int data_len = scsi_bufflen(scmd); - struct fusion_context *fusion; u32 mr_nvme_pg_size = max_t(u32, instance->nvme_page_size, MR_DEFAULT_NVME_PAGE_SIZE); - fusion = instance->ctrl_context; - build_prp = megasas_is_prp_possible(instance, scmd, sge_count); if (!build_prp) @@ -2621,7 +2609,6 @@ megasas_build_ldio_fusion(struct megasas_instance *instance, u32 start_lba_lo, start_lba_hi, device_id, datalength = 0; u32 scsi_buff_len; struct MPI2_RAID_SCSI_IO_REQUEST *io_request; - union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc; struct IO_REQUEST_INFO io_info; struct fusion_context *fusion; struct MR_DRV_RAID_MAP_ALL *local_map_ptr; @@ -2644,8 +2631,6 @@ megasas_build_ldio_fusion(struct megasas_instance *instance, rctx->status = 0; rctx->ex_status = 0; - req_desc = (union MEGASAS_REQUEST_DESCRIPTOR_UNION *)cmd->request_desc; - start_lba_lo = 0; start_lba_hi = 0; fp_possible = false; @@ -3246,9 +3231,6 @@ megasas_build_and_issue_cmd_fusion(struct megasas_instance *instance, struct megasas_cmd_fusion *cmd, *r1_cmd = NULL; union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc; u32 index; - struct fusion_context *fusion; - - fusion = instance->ctrl_context; if ((megasas_cmd_type(scmd) == READ_WRITE_LDIO) && instance->ldio_threshold && @@ -4401,14 +4383,11 @@ int megasas_task_abort_fusion(struct scsi_cmnd *scmd) { struct megasas_instance *instance; u16 smid, devhandle; - struct fusion_context *fusion; int ret; struct MR_PRIV_DEVICE *mr_device_priv_data; mr_device_priv_data = scmd->device->hostdata; - instance = (struct megasas_instance *)scmd->device->host->hostdata; - fusion = instance->ctrl_context; scmd_printk(KERN_INFO, scmd, "task abort called for scmd(%p)\n", scmd); scsi_print_command(scmd); @@ -4428,7 +4407,6 @@ int megasas_task_abort_fusion(struct scsi_cmnd *scmd) goto out; } - if (!mr_device_priv_data->is_tm_capable) { ret = FAILED; goto out; @@ -4487,12 +4465,10 @@ int megasas_reset_target_fusion(struct scsi_cmnd *scmd) struct megasas_instance *instance; int ret = FAILED; u16 devhandle; - struct fusion_context *fusion; struct MR_PRIV_DEVICE *mr_device_priv_data; mr_device_priv_data = scmd->device->hostdata; instance = (struct megasas_instance *)scmd->device->host->hostdata; - fusion = instance->ctrl_context; sdev_printk(KERN_INFO, scmd->device, "target reset called for scmd(%p)\n", scmd); @@ -4512,7 +4488,6 @@ int megasas_reset_target_fusion(struct scsi_cmnd *scmd) goto out; } - if (!mr_device_priv_data->is_tm_capable) { ret = FAILED; goto out; diff --git a/drivers/scsi/megaraid/megaraid_sas_fusion.h b/drivers/scsi/megaraid/megaraid_sas_fusion.h index ca73c50fe723..1481bf029490 100644 --- a/drivers/scsi/megaraid/megaraid_sas_fusion.h +++ b/drivers/scsi/megaraid/megaraid_sas_fusion.h @@ -724,6 +724,7 @@ struct MPI2_IOC_INIT_REQUEST { #define MR_DCMD_LD_VF_MAP_GET_ALL_LDS_111 0x03200200 #define MR_DCMD_LD_VF_MAP_GET_ALL_LDS 0x03150200 #define MR_DCMD_CTRL_SNAPDUMP_GET_PROPERTIES 0x01200100 +#define MR_DCMD_CTRL_DEVICE_LIST_GET 0x01190600 struct MR_DEV_HANDLE_INFO { __le16 curDevHdl; diff --git a/drivers/scsi/mpt3sas/mpi/mpi2_cnfg.h b/drivers/scsi/mpt3sas/mpi/mpi2_cnfg.h index 398fa6fde960..a2f4a55c51be 100644 --- a/drivers/scsi/mpt3sas/mpi/mpi2_cnfg.h +++ b/drivers/scsi/mpt3sas/mpi/mpi2_cnfg.h @@ -548,7 +548,8 @@ typedef struct _MPI2_CONFIG_REPLY { #define MPI2_MFGPAGE_DEVID_SAS2308_1 (0x0086) #define MPI2_MFGPAGE_DEVID_SAS2308_2 (0x0087) #define MPI2_MFGPAGE_DEVID_SAS2308_3 (0x006E) -#define MPI2_MFGPAGE_DEVID_SAS2308_MPI_EP (0x02B0) +#define MPI2_MFGPAGE_DEVID_SWITCH_MPI_EP (0x02B0) +#define MPI2_MFGPAGE_DEVID_SWITCH_MPI_EP_1 (0x02B1) /*MPI v2.5 SAS products */ #define MPI25_MFGPAGE_DEVID_SAS3004 (0x0096) diff --git a/drivers/scsi/mpt3sas/mpt3sas_base.c b/drivers/scsi/mpt3sas/mpt3sas_base.c index 0a6cb8f0680c..e57774472e75 100644 --- a/drivers/scsi/mpt3sas/mpt3sas_base.c +++ b/drivers/scsi/mpt3sas/mpt3sas_base.c @@ -3563,6 +3563,7 @@ _base_display_OEMs_branding(struct MPT3SAS_ADAPTER *ioc) ioc->pdev->subsystem_device); break; } + break; case MPI2_MFGPAGE_DEVID_SAS2308_2: switch (ioc->pdev->subsystem_device) { case MPT2SAS_INTEL_RS25GB008_SSDID: @@ -3598,6 +3599,7 @@ _base_display_OEMs_branding(struct MPT3SAS_ADAPTER *ioc) ioc->pdev->subsystem_device); break; } + break; case MPI25_MFGPAGE_DEVID_SAS3008: switch (ioc->pdev->subsystem_device) { case MPT3SAS_INTEL_RMS3JC080_SSDID: @@ -3742,6 +3744,7 @@ _base_display_OEMs_branding(struct MPT3SAS_ADAPTER *ioc) ioc->pdev->subsystem_device); break; } + break; case MPI2_MFGPAGE_DEVID_SAS2308_2: switch (ioc->pdev->subsystem_device) { case MPT2SAS_HP_2_4_INTERNAL_SSDID: @@ -3765,6 +3768,7 @@ _base_display_OEMs_branding(struct MPT3SAS_ADAPTER *ioc) ioc->pdev->subsystem_device); break; } + break; default: ioc_info(ioc, "HP SAS HBA: Subsystem ID: 0x%X\n", ioc->pdev->subsystem_device); diff --git a/drivers/scsi/mpt3sas/mpt3sas_base.h b/drivers/scsi/mpt3sas/mpt3sas_base.h index 800351932cc3..19158cb59101 100644 --- a/drivers/scsi/mpt3sas/mpt3sas_base.h +++ b/drivers/scsi/mpt3sas/mpt3sas_base.h @@ -75,9 +75,9 @@ #define MPT3SAS_DRIVER_NAME "mpt3sas" #define MPT3SAS_AUTHOR "Avago Technologies <MPT-FusionLinux.pdl@avagotech.com>" #define MPT3SAS_DESCRIPTION "LSI MPT Fusion SAS 3.0 Device Driver" -#define MPT3SAS_DRIVER_VERSION "27.101.00.00" +#define MPT3SAS_DRIVER_VERSION "27.102.00.00" #define MPT3SAS_MAJOR_VERSION 27 -#define MPT3SAS_MINOR_VERSION 101 +#define MPT3SAS_MINOR_VERSION 102 #define MPT3SAS_BUILD_VERSION 0 #define MPT3SAS_RELEASE_VERSION 00 @@ -193,6 +193,9 @@ struct mpt3sas_nvme_cmd { #define SAS2_PCI_DEVICE_B0_REVISION (0x01) #define SAS3_PCI_DEVICE_C0_REVISION (0x02) +/* Atlas PCIe Switch Management Port */ +#define MPI26_ATLAS_PCIe_SWITCH_DEVID (0x00B2) + /* * Intel HBA branding */ diff --git a/drivers/scsi/mpt3sas/mpt3sas_scsih.c b/drivers/scsi/mpt3sas/mpt3sas_scsih.c index 6be39dc27103..8bb5b8f9f4d2 100644 --- a/drivers/scsi/mpt3sas/mpt3sas_scsih.c +++ b/drivers/scsi/mpt3sas/mpt3sas_scsih.c @@ -10256,7 +10256,8 @@ _scsih_determine_hba_mpi_version(struct pci_dev *pdev) case MPI2_MFGPAGE_DEVID_SAS2308_1: case MPI2_MFGPAGE_DEVID_SAS2308_2: case MPI2_MFGPAGE_DEVID_SAS2308_3: - case MPI2_MFGPAGE_DEVID_SAS2308_MPI_EP: + case MPI2_MFGPAGE_DEVID_SWITCH_MPI_EP: + case MPI2_MFGPAGE_DEVID_SWITCH_MPI_EP_1: return MPI2_VERSION; case MPI25_MFGPAGE_DEVID_SAS3004: case MPI25_MFGPAGE_DEVID_SAS3008: @@ -10282,6 +10283,7 @@ _scsih_determine_hba_mpi_version(struct pci_dev *pdev) case MPI26_MFGPAGE_DEVID_SAS3516_1: case MPI26_MFGPAGE_DEVID_SAS3416: case MPI26_MFGPAGE_DEVID_SAS3616: + case MPI26_ATLAS_PCIe_SWITCH_DEVID: case MPI26_MFGPAGE_DEVID_CFG_SEC_3916: case MPI26_MFGPAGE_DEVID_HARD_SEC_3916: case MPI26_MFGPAGE_DEVID_CFG_SEC_3816: @@ -10343,7 +10345,8 @@ _scsih_probe(struct pci_dev *pdev, const struct pci_device_id *id) ioc->is_warpdrive = 1; ioc->hide_ir_msg = 1; break; - case MPI2_MFGPAGE_DEVID_SAS2308_MPI_EP: + case MPI2_MFGPAGE_DEVID_SWITCH_MPI_EP: + case MPI2_MFGPAGE_DEVID_SWITCH_MPI_EP_1: ioc->is_mcpu_endpoint = 1; break; default: @@ -10371,6 +10374,7 @@ _scsih_probe(struct pci_dev *pdev, const struct pci_device_id *id) case MPI26_MFGPAGE_DEVID_SAS3516_1: case MPI26_MFGPAGE_DEVID_SAS3416: case MPI26_MFGPAGE_DEVID_SAS3616: + case MPI26_ATLAS_PCIe_SWITCH_DEVID: ioc->is_gen35_ioc = 1; break; case MPI26_MFGPAGE_DEVID_CFG_SEC_3816: @@ -10783,7 +10787,9 @@ static const struct pci_device_id mpt3sas_pci_table[] = { PCI_ANY_ID, PCI_ANY_ID }, { MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SAS2308_3, PCI_ANY_ID, PCI_ANY_ID }, - { MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SAS2308_MPI_EP, + { MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SWITCH_MPI_EP, + PCI_ANY_ID, PCI_ANY_ID }, + { MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SWITCH_MPI_EP_1, PCI_ANY_ID, PCI_ANY_ID }, /* SSS6200 */ { MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SSS6200, @@ -10849,6 +10855,10 @@ static const struct pci_device_id mpt3sas_pci_table[] = { { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_HARD_SEC_3916, PCI_ANY_ID, PCI_ANY_ID }, + /* Atlas PCIe Switch Management Port */ + { MPI2_MFGPAGE_VENDORID_LSI, MPI26_ATLAS_PCIe_SWITCH_DEVID, + PCI_ANY_ID, PCI_ANY_ID }, + /* Sea SI 0x00E5 Configurable Secure * 0x00E6 Hard Secure */ diff --git a/drivers/scsi/mvumi.c b/drivers/scsi/mvumi.c index 36f64205ecfa..3df02691a092 100644 --- a/drivers/scsi/mvumi.c +++ b/drivers/scsi/mvumi.c @@ -718,8 +718,8 @@ static int mvumi_host_reset(struct scsi_cmnd *scmd) mhba = (struct mvumi_hba *) scmd->device->host->hostdata; - scmd_printk(KERN_NOTICE, scmd, "RESET -%ld cmd=%x retries=%x\n", - scmd->serial_number, scmd->cmnd[0], scmd->retries); + scmd_printk(KERN_NOTICE, scmd, "RESET -%u cmd=%x retries=%x\n", + scmd->request->tag, scmd->cmnd[0], scmd->retries); return mhba->instancet->reset_host(mhba); } @@ -2104,7 +2104,6 @@ static int mvumi_queue_command(struct Scsi_Host *shost, unsigned long irq_flags; spin_lock_irqsave(shost->host_lock, irq_flags); - scsi_cmd_get_serial(shost, scmd); mhba = (struct mvumi_hba *) shost->hostdata; scmd->result = 0; diff --git a/drivers/scsi/nsp32.c b/drivers/scsi/nsp32.c index 00e3cbee55b8..da4d6e1106c4 100644 --- a/drivers/scsi/nsp32.c +++ b/drivers/scsi/nsp32.c @@ -2441,7 +2441,6 @@ static void nsp32_set_sync_entry(nsp32_hw_data *data, period = data->synct[entry].period_num; ackwidth = data->synct[entry].ackwidth; - offset = offset; sample_rate = data->synct[entry].sample_rate; target->syncreg = TO_SYNCREG(period, offset); diff --git a/drivers/scsi/osd/Kbuild b/drivers/scsi/osd/Kbuild deleted file mode 100644 index 58cecd45b0f5..000000000000 --- a/drivers/scsi/osd/Kbuild +++ /dev/null @@ -1,20 +0,0 @@ -# -# Kbuild for the OSD modules -# -# Copyright (C) 2008 Panasas Inc. All rights reserved. -# -# Authors: -# Boaz Harrosh <ooo@electrozaur.com> -# Benny Halevy <bhalevy@panasas.com> -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 -# - -# libosd.ko - osd-initiator library -libosd-y := osd_initiator.o -obj-$(CONFIG_SCSI_OSD_INITIATOR) += libosd.o - -# osd.ko - SCSI ULD and char-device -osd-y := osd_uld.o -obj-$(CONFIG_SCSI_OSD_ULD) += osd.o diff --git a/drivers/scsi/osd/Kconfig b/drivers/scsi/osd/Kconfig deleted file mode 100644 index 347cc5e33749..000000000000 --- a/drivers/scsi/osd/Kconfig +++ /dev/null @@ -1,49 +0,0 @@ -# -# Kernel configuration file for the OSD scsi protocol -# -# Copyright (C) 2008 Panasas Inc. All rights reserved. -# -# Authors: -# Boaz Harrosh <ooo@electrozaur.com> -# Benny Halevy <bhalevy@panasas.com> -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public version 2 License as -# published by the Free Software Foundation -# -config SCSI_OSD_INITIATOR - tristate "OSD-Initiator library" - depends on SCSI - help - Enable the OSD-Initiator library (libosd.ko). - NOTE: You must also select CRYPTO_SHA1 + CRYPTO_HMAC and their - dependencies - -config SCSI_OSD_ULD - tristate "OSD Upper Level driver" - depends on SCSI_OSD_INITIATOR - help - Build a SCSI upper layer driver that exports /dev/osdX devices - to user-mode for testing and controlling OSD devices. It is also - needed by exofs, for mounting an OSD based file system. - -config SCSI_OSD_DPRINT_SENSE - int "(0-2) When sense is returned, DEBUG print all sense descriptors" - default 1 - depends on SCSI_OSD_INITIATOR - help - When a CHECK_CONDITION status is returned from a target, and a - sense-buffer is retrieved, turning this on will dump a full - sense-decoding message. Setting to 2 will also print recoverable - errors that might be regularly returned for some filesystem - operations. - -config SCSI_OSD_DEBUG - bool "Compile All OSD modules with lots of DEBUG prints" - default n - depends on SCSI_OSD_INITIATOR - help - OSD Code is populated with lots of OSD_DEBUG(..) printouts to - dmesg. Enable this if you found a bug and you want to help us - track the problem (see also MAINTAINERS). Setting this will also - force SCSI_OSD_DPRINT_SENSE=2. diff --git a/drivers/scsi/osd/osd_debug.h b/drivers/scsi/osd/osd_debug.h deleted file mode 100644 index 26341261bb5c..000000000000 --- a/drivers/scsi/osd/osd_debug.h +++ /dev/null @@ -1,30 +0,0 @@ -/* - * osd_debug.h - Some kprintf macros - * - * Copyright (C) 2008 Panasas Inc. All rights reserved. - * - * Authors: - * Boaz Harrosh <ooo@electrozaur.com> - * Benny Halevy <bhalevy@panasas.com> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 - * - */ -#ifndef __OSD_DEBUG_H__ -#define __OSD_DEBUG_H__ - -#define OSD_ERR(fmt, a...) printk(KERN_ERR "osd: " fmt, ##a) -#define OSD_INFO(fmt, a...) printk(KERN_NOTICE "osd: " fmt, ##a) - -#ifdef CONFIG_SCSI_OSD_DEBUG -#define OSD_DEBUG(fmt, a...) \ - printk(KERN_NOTICE "osd @%s:%d: " fmt, __func__, __LINE__, ##a) -#else -#define OSD_DEBUG(fmt, a...) do {} while (0) -#endif - -/* u64 has problems with printk this will cast it to unsigned long long */ -#define _LLU(x) (unsigned long long)(x) - -#endif /* ndef __OSD_DEBUG_H__ */ diff --git a/drivers/scsi/osd/osd_initiator.c b/drivers/scsi/osd/osd_initiator.c deleted file mode 100644 index 60cf7c5eb880..000000000000 --- a/drivers/scsi/osd/osd_initiator.c +++ /dev/null @@ -1,2076 +0,0 @@ -/* - * osd_initiator - Main body of the osd initiator library. - * - * Note: The file does not contain the advanced security functionality which - * is only needed by the security_manager's initiators. - * - * Copyright (C) 2008 Panasas Inc. All rights reserved. - * - * Authors: - * Boaz Harrosh <ooo@electrozaur.com> - * Benny Halevy <bhalevy@panasas.com> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. Neither the name of the Panasas company nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED - * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR - * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include <linux/slab.h> -#include <linux/module.h> - -#include <scsi/osd_initiator.h> -#include <scsi/osd_sec.h> -#include <scsi/osd_attributes.h> -#include <scsi/osd_sense.h> - -#include <scsi/scsi_device.h> -#include <scsi/scsi_request.h> - -#include "osd_debug.h" - -#ifndef __unused -# define __unused __attribute__((unused)) -#endif - -enum { OSD_REQ_RETRIES = 1 }; - -MODULE_AUTHOR("Boaz Harrosh <ooo@electrozaur.com>"); -MODULE_DESCRIPTION("open-osd initiator library libosd.ko"); -MODULE_LICENSE("GPL"); - -static inline void build_test(void) -{ - /* structures were not packed */ - BUILD_BUG_ON(sizeof(struct osd_capability) != OSD_CAP_LEN); - BUILD_BUG_ON(sizeof(struct osdv2_cdb) != OSD_TOTAL_CDB_LEN); - BUILD_BUG_ON(sizeof(struct osdv1_cdb) != OSDv1_TOTAL_CDB_LEN); -} - -static const char *_osd_ver_desc(struct osd_request *or) -{ - return osd_req_is_ver1(or) ? "OSD1" : "OSD2"; -} - -#define ATTR_DEF_RI(id, len) ATTR_DEF(OSD_APAGE_ROOT_INFORMATION, id, len) - -static int _osd_get_print_system_info(struct osd_dev *od, - void *caps, struct osd_dev_info *odi) -{ - struct osd_request *or; - struct osd_attr get_attrs[] = { - ATTR_DEF_RI(OSD_ATTR_RI_VENDOR_IDENTIFICATION, 8), - ATTR_DEF_RI(OSD_ATTR_RI_PRODUCT_IDENTIFICATION, 16), - ATTR_DEF_RI(OSD_ATTR_RI_PRODUCT_MODEL, 32), - ATTR_DEF_RI(OSD_ATTR_RI_PRODUCT_REVISION_LEVEL, 4), - ATTR_DEF_RI(OSD_ATTR_RI_PRODUCT_SERIAL_NUMBER, 64 /*variable*/), - ATTR_DEF_RI(OSD_ATTR_RI_OSD_NAME, 64 /*variable*/), - ATTR_DEF_RI(OSD_ATTR_RI_TOTAL_CAPACITY, 8), - ATTR_DEF_RI(OSD_ATTR_RI_USED_CAPACITY, 8), - ATTR_DEF_RI(OSD_ATTR_RI_NUMBER_OF_PARTITIONS, 8), - ATTR_DEF_RI(OSD_ATTR_RI_CLOCK, 6), - /* IBM-OSD-SIM Has a bug with this one put it last */ - ATTR_DEF_RI(OSD_ATTR_RI_OSD_SYSTEM_ID, 20), - }; - void *iter = NULL, *pFirst; - int nelem = ARRAY_SIZE(get_attrs), a = 0; - int ret; - - or = osd_start_request(od); - if (!or) - return -ENOMEM; - - /* get attrs */ - osd_req_get_attributes(or, &osd_root_object); - osd_req_add_get_attr_list(or, get_attrs, ARRAY_SIZE(get_attrs)); - - ret = osd_finalize_request(or, 0, caps, NULL); - if (ret) - goto out; - - ret = osd_execute_request(or); - if (ret) { - OSD_ERR("Failed to detect %s => %d\n", _osd_ver_desc(or), ret); - goto out; - } - - osd_req_decode_get_attr_list(or, get_attrs, &nelem, &iter); - - OSD_INFO("Detected %s device\n", - _osd_ver_desc(or)); - - pFirst = get_attrs[a++].val_ptr; - OSD_INFO("VENDOR_IDENTIFICATION [%s]\n", - (char *)pFirst); - - pFirst = get_attrs[a++].val_ptr; - OSD_INFO("PRODUCT_IDENTIFICATION [%s]\n", - (char *)pFirst); - - pFirst = get_attrs[a++].val_ptr; - OSD_INFO("PRODUCT_MODEL [%s]\n", - (char *)pFirst); - - pFirst = get_attrs[a++].val_ptr; - OSD_INFO("PRODUCT_REVISION_LEVEL [%u]\n", - pFirst ? get_unaligned_be32(pFirst) : ~0U); - - pFirst = get_attrs[a++].val_ptr; - OSD_INFO("PRODUCT_SERIAL_NUMBER [%s]\n", - (char *)pFirst); - - odi->osdname_len = get_attrs[a].len; - /* Avoid NULL for memcmp optimization 0-length is good enough */ - odi->osdname = kzalloc(odi->osdname_len + 1, GFP_KERNEL); - if (!odi->osdname) { - ret = -ENOMEM; - goto out; - } - if (odi->osdname_len) - memcpy(odi->osdname, get_attrs[a].val_ptr, odi->osdname_len); - OSD_INFO("OSD_NAME [%s]\n", odi->osdname); - a++; - - pFirst = get_attrs[a++].val_ptr; - OSD_INFO("TOTAL_CAPACITY [0x%llx]\n", - pFirst ? _LLU(get_unaligned_be64(pFirst)) : ~0ULL); - - pFirst = get_attrs[a++].val_ptr; - OSD_INFO("USED_CAPACITY [0x%llx]\n", - pFirst ? _LLU(get_unaligned_be64(pFirst)) : ~0ULL); - - pFirst = get_attrs[a++].val_ptr; - OSD_INFO("NUMBER_OF_PARTITIONS [%llu]\n", - pFirst ? _LLU(get_unaligned_be64(pFirst)) : ~0ULL); - - if (a >= nelem) - goto out; - - /* FIXME: Where are the time utilities */ - pFirst = get_attrs[a++].val_ptr; - OSD_INFO("CLOCK [0x%6phN]\n", pFirst); - - if (a < nelem) { /* IBM-OSD-SIM bug, Might not have it */ - unsigned len = get_attrs[a].len; - char sid_dump[32*4 + 2]; /* 2nibbles+space+ASCII */ - - hex_dump_to_buffer(get_attrs[a].val_ptr, len, 32, 1, - sid_dump, sizeof(sid_dump), true); - OSD_INFO("OSD_SYSTEM_ID(%d)\n" - " [%s]\n", len, sid_dump); - - if (unlikely(len > sizeof(odi->systemid))) { - OSD_ERR("OSD Target error: OSD_SYSTEM_ID too long(%d). " - "device identification might not work\n", len); - len = sizeof(odi->systemid); - } - odi->systemid_len = len; - memcpy(odi->systemid, get_attrs[a].val_ptr, len); - a++; - } -out: - osd_end_request(or); - return ret; -} - -int osd_auto_detect_ver(struct osd_dev *od, - void *caps, struct osd_dev_info *odi) -{ - int ret; - - /* Auto-detect the osd version */ - ret = _osd_get_print_system_info(od, caps, odi); - if (ret) { - osd_dev_set_ver(od, OSD_VER1); - OSD_DEBUG("converting to OSD1\n"); - ret = _osd_get_print_system_info(od, caps, odi); - } - - return ret; -} -EXPORT_SYMBOL(osd_auto_detect_ver); - -static unsigned _osd_req_cdb_len(struct osd_request *or) -{ - return osd_req_is_ver1(or) ? OSDv1_TOTAL_CDB_LEN : OSD_TOTAL_CDB_LEN; -} - -static unsigned _osd_req_alist_elem_size(struct osd_request *or, unsigned len) -{ - return osd_req_is_ver1(or) ? - osdv1_attr_list_elem_size(len) : - osdv2_attr_list_elem_size(len); -} - -static void _osd_req_alist_elem_encode(struct osd_request *or, - void *attr_last, const struct osd_attr *oa) -{ - if (osd_req_is_ver1(or)) { - struct osdv1_attributes_list_element *attr = attr_last; - - attr->attr_page = cpu_to_be32(oa->attr_page); - attr->attr_id = cpu_to_be32(oa->attr_id); - attr->attr_bytes = cpu_to_be16(oa->len); - memcpy(attr->attr_val, oa->val_ptr, oa->len); - } else { - struct osdv2_attributes_list_element *attr = attr_last; - - attr->attr_page = cpu_to_be32(oa->attr_page); - attr->attr_id = cpu_to_be32(oa->attr_id); - attr->attr_bytes = cpu_to_be16(oa->len); - memcpy(attr->attr_val, oa->val_ptr, oa->len); - } -} - -static int _osd_req_alist_elem_decode(struct osd_request *or, - void *cur_p, struct osd_attr *oa, unsigned max_bytes) -{ - unsigned inc; - if (osd_req_is_ver1(or)) { - struct osdv1_attributes_list_element *attr = cur_p; - - if (max_bytes < sizeof(*attr)) - return -1; - - oa->len = be16_to_cpu(attr->attr_bytes); - inc = _osd_req_alist_elem_size(or, oa->len); - if (inc > max_bytes) - return -1; - - oa->attr_page = be32_to_cpu(attr->attr_page); - oa->attr_id = be32_to_cpu(attr->attr_id); - - /* OSD1: On empty attributes we return a pointer to 2 bytes - * of zeros. This keeps similar behaviour with OSD2. - * (See below) - */ - oa->val_ptr = likely(oa->len) ? attr->attr_val : - (u8 *)&attr->attr_bytes; - } else { - struct osdv2_attributes_list_element *attr = cur_p; - - if (max_bytes < sizeof(*attr)) - return -1; - - oa->len = be16_to_cpu(attr->attr_bytes); - inc = _osd_req_alist_elem_size(or, oa->len); - if (inc > max_bytes) - return -1; - - oa->attr_page = be32_to_cpu(attr->attr_page); - oa->attr_id = be32_to_cpu(attr->attr_id); - - /* OSD2: For convenience, on empty attributes, we return 8 bytes - * of zeros here. This keeps the same behaviour with OSD2r04, - * and is nice with null terminating ASCII fields. - * oa->val_ptr == NULL marks the end-of-list, or error. - */ - oa->val_ptr = likely(oa->len) ? attr->attr_val : attr->reserved; - } - return inc; -} - -static unsigned _osd_req_alist_size(struct osd_request *or, void *list_head) -{ - return osd_req_is_ver1(or) ? - osdv1_list_size(list_head) : - osdv2_list_size(list_head); -} - -static unsigned _osd_req_sizeof_alist_header(struct osd_request *or) -{ - return osd_req_is_ver1(or) ? - sizeof(struct osdv1_attributes_list_header) : - sizeof(struct osdv2_attributes_list_header); -} - -static void _osd_req_set_alist_type(struct osd_request *or, - void *list, int list_type) -{ - if (osd_req_is_ver1(or)) { - struct osdv1_attributes_list_header *attr_list = list; - - memset(attr_list, 0, sizeof(*attr_list)); - attr_list->type = list_type; - } else { - struct osdv2_attributes_list_header *attr_list = list; - - memset(attr_list, 0, sizeof(*attr_list)); - attr_list->type = list_type; - } -} - -static bool _osd_req_is_alist_type(struct osd_request *or, - void *list, int list_type) -{ - if (!list) - return false; - - if (osd_req_is_ver1(or)) { - struct osdv1_attributes_list_header *attr_list = list; - - return attr_list->type == list_type; - } else { - struct osdv2_attributes_list_header *attr_list = list; - - return attr_list->type == list_type; - } -} - -/* This is for List-objects not Attributes-Lists */ -static void _osd_req_encode_olist(struct osd_request *or, - struct osd_obj_id_list *list) -{ - struct osd_cdb_head *cdbh = osd_cdb_head(&or->cdb); - - if (osd_req_is_ver1(or)) { - cdbh->v1.list_identifier = list->list_identifier; - cdbh->v1.start_address = list->continuation_id; - } else { - cdbh->v2.list_identifier = list->list_identifier; - cdbh->v2.start_address = list->continuation_id; - } -} - -static osd_cdb_offset osd_req_encode_offset(struct osd_request *or, - u64 offset, unsigned *padding) -{ - return __osd_encode_offset(offset, padding, - osd_req_is_ver1(or) ? - OSDv1_OFFSET_MIN_SHIFT : OSD_OFFSET_MIN_SHIFT, - OSD_OFFSET_MAX_SHIFT); -} - -static struct osd_security_parameters * -_osd_req_sec_params(struct osd_request *or) -{ - struct osd_cdb *ocdb = &or->cdb; - - if (osd_req_is_ver1(or)) - return (struct osd_security_parameters *)&ocdb->v1.sec_params; - else - return (struct osd_security_parameters *)&ocdb->v2.sec_params; -} - -void osd_dev_init(struct osd_dev *osdd, struct scsi_device *scsi_device) -{ - memset(osdd, 0, sizeof(*osdd)); - osdd->scsi_device = scsi_device; - osdd->def_timeout = BLK_DEFAULT_SG_TIMEOUT; -#ifdef OSD_VER1_SUPPORT - osdd->version = OSD_VER2; -#endif - /* TODO: Allocate pools for osd_request attributes ... */ -} -EXPORT_SYMBOL(osd_dev_init); - -void osd_dev_fini(struct osd_dev *osdd) -{ - /* TODO: De-allocate pools */ - - osdd->scsi_device = NULL; -} -EXPORT_SYMBOL(osd_dev_fini); - -static struct osd_request *_osd_request_alloc(gfp_t gfp) -{ - struct osd_request *or; - - /* TODO: Use mempool with one saved request */ - or = kzalloc(sizeof(*or), gfp); - return or; -} - -static void _osd_request_free(struct osd_request *or) -{ - kfree(or); -} - -struct osd_request *osd_start_request(struct osd_dev *dev) -{ - struct osd_request *or; - - or = _osd_request_alloc(GFP_KERNEL); - if (!or) - return NULL; - - or->osd_dev = dev; - or->timeout = dev->def_timeout; - or->retries = OSD_REQ_RETRIES; - - return or; -} -EXPORT_SYMBOL(osd_start_request); - -static void _osd_free_seg(struct osd_request *or __unused, - struct _osd_req_data_segment *seg) -{ - if (!seg->buff || !seg->alloc_size) - return; - - kfree(seg->buff); - seg->buff = NULL; - seg->alloc_size = 0; -} - -static void _put_request(struct request *rq) -{ - /* - * If osd_finalize_request() was called but the request was not - * executed through the block layer, then we must release BIOs. - * TODO: Keep error code in or->async_error. Need to audit all - * code paths. - */ - if (unlikely(rq->bio)) - blk_mq_end_request(rq, BLK_STS_IOERR); - else - blk_put_request(rq); -} - -void osd_end_request(struct osd_request *or) -{ - struct request *rq = or->request; - - if (rq) { - if (rq->next_rq) { - _put_request(rq->next_rq); - rq->next_rq = NULL; - } - - _put_request(rq); - } - - _osd_free_seg(or, &or->get_attr); - _osd_free_seg(or, &or->enc_get_attr); - _osd_free_seg(or, &or->set_attr); - _osd_free_seg(or, &or->cdb_cont); - - _osd_request_free(or); -} -EXPORT_SYMBOL(osd_end_request); - -static void _set_error_resid(struct osd_request *or, struct request *req, - blk_status_t error) -{ - or->async_error = error; - or->req_errors = scsi_req(req)->result; - or->sense_len = scsi_req(req)->sense_len; - if (or->sense_len) - memcpy(or->sense, scsi_req(req)->sense, or->sense_len); - if (or->out.req) - or->out.residual = scsi_req(or->out.req)->resid_len; - if (or->in.req) - or->in.residual = scsi_req(or->in.req)->resid_len; -} - -int osd_execute_request(struct osd_request *or) -{ - blk_execute_rq(or->request->q, NULL, or->request, 0); - - if (scsi_req(or->request)->result) { - _set_error_resid(or, or->request, BLK_STS_IOERR); - return -EIO; - } - - _set_error_resid(or, or->request, BLK_STS_OK); - return 0; -} -EXPORT_SYMBOL(osd_execute_request); - -static void osd_request_async_done(struct request *req, blk_status_t error) -{ - struct osd_request *or = req->end_io_data; - - _set_error_resid(or, req, error); - if (req->next_rq) { - blk_put_request(req->next_rq); - req->next_rq = NULL; - } - - blk_put_request(req); - or->request = NULL; - or->in.req = NULL; - or->out.req = NULL; - - if (or->async_done) - or->async_done(or, or->async_private); - else - osd_end_request(or); -} - -int osd_execute_request_async(struct osd_request *or, - osd_req_done_fn *done, void *private) -{ - or->request->end_io_data = or; - or->async_private = private; - or->async_done = done; - - blk_execute_rq_nowait(or->request->q, NULL, or->request, 0, - osd_request_async_done); - return 0; -} -EXPORT_SYMBOL(osd_execute_request_async); - -u8 sg_out_pad_buffer[1 << OSDv1_OFFSET_MIN_SHIFT]; -u8 sg_in_pad_buffer[1 << OSDv1_OFFSET_MIN_SHIFT]; - -static int _osd_realloc_seg(struct osd_request *or, - struct _osd_req_data_segment *seg, unsigned max_bytes) -{ - void *buff; - - if (seg->alloc_size >= max_bytes) - return 0; - - buff = krealloc(seg->buff, max_bytes, GFP_KERNEL); - if (!buff) { - OSD_ERR("Failed to Realloc %d-bytes was-%d\n", max_bytes, - seg->alloc_size); - return -ENOMEM; - } - - memset(buff + seg->alloc_size, 0, max_bytes - seg->alloc_size); - seg->buff = buff; - seg->alloc_size = max_bytes; - return 0; -} - -static int _alloc_cdb_cont(struct osd_request *or, unsigned total_bytes) -{ - OSD_DEBUG("total_bytes=%d\n", total_bytes); - return _osd_realloc_seg(or, &or->cdb_cont, total_bytes); -} - -static int _alloc_set_attr_list(struct osd_request *or, - const struct osd_attr *oa, unsigned nelem, unsigned add_bytes) -{ - unsigned total_bytes = add_bytes; - - for (; nelem; --nelem, ++oa) - total_bytes += _osd_req_alist_elem_size(or, oa->len); - - OSD_DEBUG("total_bytes=%d\n", total_bytes); - return _osd_realloc_seg(or, &or->set_attr, total_bytes); -} - -static int _alloc_get_attr_desc(struct osd_request *or, unsigned max_bytes) -{ - OSD_DEBUG("total_bytes=%d\n", max_bytes); - return _osd_realloc_seg(or, &or->enc_get_attr, max_bytes); -} - -static int _alloc_get_attr_list(struct osd_request *or) -{ - OSD_DEBUG("total_bytes=%d\n", or->get_attr.total_bytes); - return _osd_realloc_seg(or, &or->get_attr, or->get_attr.total_bytes); -} - -/* - * Common to all OSD commands - */ - -static void _osdv1_req_encode_common(struct osd_request *or, - __be16 act, const struct osd_obj_id *obj, u64 offset, u64 len) -{ - struct osdv1_cdb *ocdb = &or->cdb.v1; - - /* - * For speed, the commands - * OSD_ACT_PERFORM_SCSI_COMMAND , V1 0x8F7E, V2 0x8F7C - * OSD_ACT_SCSI_TASK_MANAGEMENT , V1 0x8F7F, V2 0x8F7D - * are not supported here. Should pass zero and set after the call - */ - act &= cpu_to_be16(~0x0080); /* V1 action code */ - - OSD_DEBUG("OSDv1 execute opcode 0x%x\n", be16_to_cpu(act)); - - ocdb->h.varlen_cdb.opcode = VARIABLE_LENGTH_CMD; - ocdb->h.varlen_cdb.additional_cdb_length = OSD_ADDITIONAL_CDB_LENGTH; - ocdb->h.varlen_cdb.service_action = act; - - ocdb->h.partition = cpu_to_be64(obj->partition); - ocdb->h.object = cpu_to_be64(obj->id); - ocdb->h.v1.length = cpu_to_be64(len); - ocdb->h.v1.start_address = cpu_to_be64(offset); -} - -static void _osdv2_req_encode_common(struct osd_request *or, - __be16 act, const struct osd_obj_id *obj, u64 offset, u64 len) -{ - struct osdv2_cdb *ocdb = &or->cdb.v2; - - OSD_DEBUG("OSDv2 execute opcode 0x%x\n", be16_to_cpu(act)); - - ocdb->h.varlen_cdb.opcode = VARIABLE_LENGTH_CMD; - ocdb->h.varlen_cdb.additional_cdb_length = OSD_ADDITIONAL_CDB_LENGTH; - ocdb->h.varlen_cdb.service_action = act; - - ocdb->h.partition = cpu_to_be64(obj->partition); - ocdb->h.object = cpu_to_be64(obj->id); - ocdb->h.v2.length = cpu_to_be64(len); - ocdb->h.v2.start_address = cpu_to_be64(offset); -} - -static void _osd_req_encode_common(struct osd_request *or, - __be16 act, const struct osd_obj_id *obj, u64 offset, u64 len) -{ - if (osd_req_is_ver1(or)) - _osdv1_req_encode_common(or, act, obj, offset, len); - else - _osdv2_req_encode_common(or, act, obj, offset, len); -} - -/* - * Device commands - */ -/*TODO: void osd_req_set_master_seed_xchg(struct osd_request *, ...); */ -/*TODO: void osd_req_set_master_key(struct osd_request *, ...); */ - -void osd_req_format(struct osd_request *or, u64 tot_capacity) -{ - _osd_req_encode_common(or, OSD_ACT_FORMAT_OSD, &osd_root_object, 0, - tot_capacity); -} -EXPORT_SYMBOL(osd_req_format); - -int osd_req_list_dev_partitions(struct osd_request *or, - osd_id initial_id, struct osd_obj_id_list *list, unsigned nelem) -{ - return osd_req_list_partition_objects(or, 0, initial_id, list, nelem); -} -EXPORT_SYMBOL(osd_req_list_dev_partitions); - -static void _osd_req_encode_flush(struct osd_request *or, - enum osd_options_flush_scope_values op) -{ - struct osd_cdb_head *ocdb = osd_cdb_head(&or->cdb); - - ocdb->command_specific_options = op; -} - -void osd_req_flush_obsd(struct osd_request *or, - enum osd_options_flush_scope_values op) -{ - _osd_req_encode_common(or, OSD_ACT_FLUSH_OSD, &osd_root_object, 0, 0); - _osd_req_encode_flush(or, op); -} -EXPORT_SYMBOL(osd_req_flush_obsd); - -/*TODO: void osd_req_perform_scsi_command(struct osd_request *, - const u8 *cdb, ...); */ -/*TODO: void osd_req_task_management(struct osd_request *, ...); */ - -/* - * Partition commands - */ -static void _osd_req_encode_partition(struct osd_request *or, - __be16 act, osd_id partition) -{ - struct osd_obj_id par = { - .partition = partition, - .id = 0, - }; - - _osd_req_encode_common(or, act, &par, 0, 0); -} - -void osd_req_create_partition(struct osd_request *or, osd_id partition) -{ - _osd_req_encode_partition(or, OSD_ACT_CREATE_PARTITION, partition); -} -EXPORT_SYMBOL(osd_req_create_partition); - -void osd_req_remove_partition(struct osd_request *or, osd_id partition) -{ - _osd_req_encode_partition(or, OSD_ACT_REMOVE_PARTITION, partition); -} -EXPORT_SYMBOL(osd_req_remove_partition); - -/*TODO: void osd_req_set_partition_key(struct osd_request *, - osd_id partition, u8 new_key_id[OSD_CRYPTO_KEYID_SIZE], - u8 seed[OSD_CRYPTO_SEED_SIZE]); */ - -static int _osd_req_list_objects(struct osd_request *or, - __be16 action, const struct osd_obj_id *obj, osd_id initial_id, - struct osd_obj_id_list *list, unsigned nelem) -{ - struct request_queue *q = osd_request_queue(or->osd_dev); - u64 len = nelem * sizeof(osd_id) + sizeof(*list); - struct bio *bio; - - _osd_req_encode_common(or, action, obj, (u64)initial_id, len); - - if (list->list_identifier) - _osd_req_encode_olist(or, list); - - WARN_ON(or->in.bio); - bio = bio_map_kern(q, list, len, GFP_KERNEL); - if (IS_ERR(bio)) { - OSD_ERR("!!! Failed to allocate list_objects BIO\n"); - return PTR_ERR(bio); - } - - bio_set_op_attrs(bio, REQ_OP_READ, 0); - or->in.bio = bio; - or->in.total_bytes = bio->bi_iter.bi_size; - return 0; -} - -int osd_req_list_partition_collections(struct osd_request *or, - osd_id partition, osd_id initial_id, struct osd_obj_id_list *list, - unsigned nelem) -{ - struct osd_obj_id par = { - .partition = partition, - .id = 0, - }; - - return osd_req_list_collection_objects(or, &par, initial_id, list, - nelem); -} -EXPORT_SYMBOL(osd_req_list_partition_collections); - -int osd_req_list_partition_objects(struct osd_request *or, - osd_id partition, osd_id initial_id, struct osd_obj_id_list *list, - unsigned nelem) -{ - struct osd_obj_id par = { - .partition = partition, - .id = 0, - }; - - return _osd_req_list_objects(or, OSD_ACT_LIST, &par, initial_id, list, - nelem); -} -EXPORT_SYMBOL(osd_req_list_partition_objects); - -void osd_req_flush_partition(struct osd_request *or, - osd_id partition, enum osd_options_flush_scope_values op) -{ - _osd_req_encode_partition(or, OSD_ACT_FLUSH_PARTITION, partition); - _osd_req_encode_flush(or, op); -} -EXPORT_SYMBOL(osd_req_flush_partition); - -/* - * Collection commands - */ -/*TODO: void osd_req_create_collection(struct osd_request *, - const struct osd_obj_id *); */ -/*TODO: void osd_req_remove_collection(struct osd_request *, - const struct osd_obj_id *); */ - -int osd_req_list_collection_objects(struct osd_request *or, - const struct osd_obj_id *obj, osd_id initial_id, - struct osd_obj_id_list *list, unsigned nelem) -{ - return _osd_req_list_objects(or, OSD_ACT_LIST_COLLECTION, obj, - initial_id, list, nelem); -} -EXPORT_SYMBOL(osd_req_list_collection_objects); - -/*TODO: void query(struct osd_request *, ...); V2 */ - -void osd_req_flush_collection(struct osd_request *or, - const struct osd_obj_id *obj, enum osd_options_flush_scope_values op) -{ - _osd_req_encode_common(or, OSD_ACT_FLUSH_PARTITION, obj, 0, 0); - _osd_req_encode_flush(or, op); -} -EXPORT_SYMBOL(osd_req_flush_collection); - -/*TODO: void get_member_attrs(struct osd_request *, ...); V2 */ -/*TODO: void set_member_attrs(struct osd_request *, ...); V2 */ - -/* - * Object commands - */ -void osd_req_create_object(struct osd_request *or, struct osd_obj_id *obj) -{ - _osd_req_encode_common(or, OSD_ACT_CREATE, obj, 0, 0); -} -EXPORT_SYMBOL(osd_req_create_object); - -void osd_req_remove_object(struct osd_request *or, struct osd_obj_id *obj) -{ - _osd_req_encode_common(or, OSD_ACT_REMOVE, obj, 0, 0); -} -EXPORT_SYMBOL(osd_req_remove_object); - - -/*TODO: void osd_req_create_multi(struct osd_request *or, - struct osd_obj_id *first, struct osd_obj_id_list *list, unsigned nelem); -*/ - -void osd_req_write(struct osd_request *or, - const struct osd_obj_id *obj, u64 offset, - struct bio *bio, u64 len) -{ - _osd_req_encode_common(or, OSD_ACT_WRITE, obj, offset, len); - WARN_ON(or->out.bio || or->out.total_bytes); - WARN_ON(!op_is_write(bio_op(bio))); - or->out.bio = bio; - or->out.total_bytes = len; -} -EXPORT_SYMBOL(osd_req_write); - -int osd_req_write_kern(struct osd_request *or, - const struct osd_obj_id *obj, u64 offset, void* buff, u64 len) -{ - struct request_queue *req_q = osd_request_queue(or->osd_dev); - struct bio *bio = bio_map_kern(req_q, buff, len, GFP_KERNEL); - - if (IS_ERR(bio)) - return PTR_ERR(bio); - - bio_set_op_attrs(bio, REQ_OP_WRITE, 0); - osd_req_write(or, obj, offset, bio, len); - return 0; -} -EXPORT_SYMBOL(osd_req_write_kern); - -/*TODO: void osd_req_append(struct osd_request *, - const struct osd_obj_id *, struct bio *data_out); */ -/*TODO: void osd_req_create_write(struct osd_request *, - const struct osd_obj_id *, struct bio *data_out, u64 offset); */ -/*TODO: void osd_req_clear(struct osd_request *, - const struct osd_obj_id *, u64 offset, u64 len); */ -/*TODO: void osd_req_punch(struct osd_request *, - const struct osd_obj_id *, u64 offset, u64 len); V2 */ - -void osd_req_flush_object(struct osd_request *or, - const struct osd_obj_id *obj, enum osd_options_flush_scope_values op, - /*V2*/ u64 offset, /*V2*/ u64 len) -{ - if (unlikely(osd_req_is_ver1(or) && (offset || len))) { - OSD_DEBUG("OSD Ver1 flush on specific range ignored\n"); - offset = 0; - len = 0; - } - - _osd_req_encode_common(or, OSD_ACT_FLUSH, obj, offset, len); - _osd_req_encode_flush(or, op); -} -EXPORT_SYMBOL(osd_req_flush_object); - -void osd_req_read(struct osd_request *or, - const struct osd_obj_id *obj, u64 offset, - struct bio *bio, u64 len) -{ - _osd_req_encode_common(or, OSD_ACT_READ, obj, offset, len); - WARN_ON(or->in.bio || or->in.total_bytes); - WARN_ON(op_is_write(bio_op(bio))); - or->in.bio = bio; - or->in.total_bytes = len; -} -EXPORT_SYMBOL(osd_req_read); - -int osd_req_read_kern(struct osd_request *or, - const struct osd_obj_id *obj, u64 offset, void* buff, u64 len) -{ - struct request_queue *req_q = osd_request_queue(or->osd_dev); - struct bio *bio = bio_map_kern(req_q, buff, len, GFP_KERNEL); - - if (IS_ERR(bio)) - return PTR_ERR(bio); - - osd_req_read(or, obj, offset, bio, len); - return 0; -} -EXPORT_SYMBOL(osd_req_read_kern); - -static int _add_sg_continuation_descriptor(struct osd_request *or, - const struct osd_sg_entry *sglist, unsigned numentries, u64 *len) -{ - struct osd_sg_continuation_descriptor *oscd; - u32 oscd_size; - unsigned i; - int ret; - - oscd_size = sizeof(*oscd) + numentries * sizeof(oscd->entries[0]); - - if (!or->cdb_cont.total_bytes) { - /* First time, jump over the header, we will write to: - * cdb_cont.buff + cdb_cont.total_bytes - */ - or->cdb_cont.total_bytes = - sizeof(struct osd_continuation_segment_header); - } - - ret = _alloc_cdb_cont(or, or->cdb_cont.total_bytes + oscd_size); - if (unlikely(ret)) - return ret; - - oscd = or->cdb_cont.buff + or->cdb_cont.total_bytes; - oscd->hdr.type = cpu_to_be16(SCATTER_GATHER_LIST); - oscd->hdr.pad_length = 0; - oscd->hdr.length = cpu_to_be32(oscd_size - sizeof(*oscd)); - - *len = 0; - /* copy the sg entries and convert to network byte order */ - for (i = 0; i < numentries; i++) { - oscd->entries[i].offset = cpu_to_be64(sglist[i].offset); - oscd->entries[i].len = cpu_to_be64(sglist[i].len); - *len += sglist[i].len; - } - - or->cdb_cont.total_bytes += oscd_size; - OSD_DEBUG("total_bytes=%d oscd_size=%d numentries=%d\n", - or->cdb_cont.total_bytes, oscd_size, numentries); - return 0; -} - -static int _osd_req_finalize_cdb_cont(struct osd_request *or, const u8 *cap_key) -{ - struct request_queue *req_q = osd_request_queue(or->osd_dev); - struct bio *bio; - struct osd_cdb_head *cdbh = osd_cdb_head(&or->cdb); - struct osd_continuation_segment_header *cont_seg_hdr; - - if (!or->cdb_cont.total_bytes) - return 0; - - cont_seg_hdr = or->cdb_cont.buff; - cont_seg_hdr->format = CDB_CONTINUATION_FORMAT_V2; - cont_seg_hdr->service_action = cdbh->varlen_cdb.service_action; - - /* create a bio for continuation segment */ - bio = bio_map_kern(req_q, or->cdb_cont.buff, or->cdb_cont.total_bytes, - GFP_KERNEL); - if (IS_ERR(bio)) - return PTR_ERR(bio); - - bio_set_op_attrs(bio, REQ_OP_WRITE, 0); - - /* integrity check the continuation before the bio is linked - * with the other data segments since the continuation - * integrity is separate from the other data segments. - */ - osd_sec_sign_data(cont_seg_hdr->integrity_check, bio, cap_key); - - cdbh->v2.cdb_continuation_length = cpu_to_be32(or->cdb_cont.total_bytes); - - /* we can't use _req_append_segment, because we need to link in the - * continuation bio to the head of the bio list - the - * continuation segment (if it exists) is always the first segment in - * the out data buffer. - */ - bio->bi_next = or->out.bio; - or->out.bio = bio; - or->out.total_bytes += or->cdb_cont.total_bytes; - - return 0; -} - -/* osd_req_write_sg: Takes a @bio that points to the data out buffer and an - * @sglist that has the scatter gather entries. Scatter-gather enables a write - * of multiple none-contiguous areas of an object, in a single call. The extents - * may overlap and/or be in any order. The only constrain is that: - * total_bytes(sglist) >= total_bytes(bio) - */ -int osd_req_write_sg(struct osd_request *or, - const struct osd_obj_id *obj, struct bio *bio, - const struct osd_sg_entry *sglist, unsigned numentries) -{ - u64 len; - int ret = _add_sg_continuation_descriptor(or, sglist, numentries, &len); - - if (ret) - return ret; - osd_req_write(or, obj, 0, bio, len); - - return 0; -} -EXPORT_SYMBOL(osd_req_write_sg); - -/* osd_req_read_sg: Read multiple extents of an object into @bio - * See osd_req_write_sg - */ -int osd_req_read_sg(struct osd_request *or, - const struct osd_obj_id *obj, struct bio *bio, - const struct osd_sg_entry *sglist, unsigned numentries) -{ - u64 len; - u64 off; - int ret; - - if (numentries > 1) { - off = 0; - ret = _add_sg_continuation_descriptor(or, sglist, numentries, - &len); - if (ret) - return ret; - } else { - /* Optimize the case of single segment, read_sg is a - * bidi operation. - */ - len = sglist->len; - off = sglist->offset; - } - osd_req_read(or, obj, off, bio, len); - - return 0; -} -EXPORT_SYMBOL(osd_req_read_sg); - -/* SG-list write/read Kern API - * - * osd_req_{write,read}_sg_kern takes an array of @buff pointers and an array - * of sg_entries. @numentries indicates how many pointers and sg_entries there - * are. By requiring an array of buff pointers. This allows a caller to do a - * single write/read and scatter into multiple buffers. - * NOTE: Each buffer + len should not cross a page boundary. - */ -static struct bio *_create_sg_bios(struct osd_request *or, - void **buff, const struct osd_sg_entry *sglist, unsigned numentries) -{ - struct request_queue *q = osd_request_queue(or->osd_dev); - struct bio *bio; - unsigned i; - - bio = bio_kmalloc(GFP_KERNEL, numentries); - if (unlikely(!bio)) { - OSD_DEBUG("Failed to allocate BIO size=%u\n", numentries); - return ERR_PTR(-ENOMEM); - } - - for (i = 0; i < numentries; i++) { - unsigned offset = offset_in_page(buff[i]); - struct page *page = virt_to_page(buff[i]); - unsigned len = sglist[i].len; - unsigned added_len; - - BUG_ON(offset + len > PAGE_SIZE); - added_len = bio_add_pc_page(q, bio, page, len, offset); - if (unlikely(len != added_len)) { - OSD_DEBUG("bio_add_pc_page len(%d) != added_len(%d)\n", - len, added_len); - bio_put(bio); - return ERR_PTR(-ENOMEM); - } - } - - return bio; -} - -int osd_req_write_sg_kern(struct osd_request *or, - const struct osd_obj_id *obj, void **buff, - const struct osd_sg_entry *sglist, unsigned numentries) -{ - struct bio *bio = _create_sg_bios(or, buff, sglist, numentries); - if (IS_ERR(bio)) - return PTR_ERR(bio); - - bio_set_op_attrs(bio, REQ_OP_WRITE, 0); - osd_req_write_sg(or, obj, bio, sglist, numentries); - - return 0; -} -EXPORT_SYMBOL(osd_req_write_sg_kern); - -int osd_req_read_sg_kern(struct osd_request *or, - const struct osd_obj_id *obj, void **buff, - const struct osd_sg_entry *sglist, unsigned numentries) -{ - struct bio *bio = _create_sg_bios(or, buff, sglist, numentries); - if (IS_ERR(bio)) - return PTR_ERR(bio); - - osd_req_read_sg(or, obj, bio, sglist, numentries); - - return 0; -} -EXPORT_SYMBOL(osd_req_read_sg_kern); - - - -void osd_req_get_attributes(struct osd_request *or, - const struct osd_obj_id *obj) -{ - _osd_req_encode_common(or, OSD_ACT_GET_ATTRIBUTES, obj, 0, 0); -} -EXPORT_SYMBOL(osd_req_get_attributes); - -void osd_req_set_attributes(struct osd_request *or, - const struct osd_obj_id *obj) -{ - _osd_req_encode_common(or, OSD_ACT_SET_ATTRIBUTES, obj, 0, 0); -} -EXPORT_SYMBOL(osd_req_set_attributes); - -/* - * Attributes List-mode - */ - -int osd_req_add_set_attr_list(struct osd_request *or, - const struct osd_attr *oa, unsigned nelem) -{ - unsigned total_bytes = or->set_attr.total_bytes; - void *attr_last; - int ret; - - if (or->attributes_mode && - or->attributes_mode != OSD_CDB_GET_SET_ATTR_LISTS) { - WARN_ON(1); - return -EINVAL; - } - or->attributes_mode = OSD_CDB_GET_SET_ATTR_LISTS; - - if (!total_bytes) { /* first-time: allocate and put list header */ - total_bytes = _osd_req_sizeof_alist_header(or); - ret = _alloc_set_attr_list(or, oa, nelem, total_bytes); - if (ret) - return ret; - _osd_req_set_alist_type(or, or->set_attr.buff, - OSD_ATTR_LIST_SET_RETRIEVE); - } - attr_last = or->set_attr.buff + total_bytes; - - for (; nelem; --nelem) { - unsigned elem_size = _osd_req_alist_elem_size(or, oa->len); - - total_bytes += elem_size; - if (unlikely(or->set_attr.alloc_size < total_bytes)) { - or->set_attr.total_bytes = total_bytes - elem_size; - ret = _alloc_set_attr_list(or, oa, nelem, total_bytes); - if (ret) - return ret; - attr_last = - or->set_attr.buff + or->set_attr.total_bytes; - } - - _osd_req_alist_elem_encode(or, attr_last, oa); - - attr_last += elem_size; - ++oa; - } - - or->set_attr.total_bytes = total_bytes; - return 0; -} -EXPORT_SYMBOL(osd_req_add_set_attr_list); - -static int _req_append_segment(struct osd_request *or, - unsigned padding, struct _osd_req_data_segment *seg, - struct _osd_req_data_segment *last_seg, struct _osd_io_info *io) -{ - void *pad_buff; - int ret; - - if (padding) { - /* check if we can just add it to last buffer */ - if (last_seg && - (padding <= last_seg->alloc_size - last_seg->total_bytes)) - pad_buff = last_seg->buff + last_seg->total_bytes; - else - pad_buff = io->pad_buff; - - ret = blk_rq_map_kern(io->req->q, io->req, pad_buff, padding, - GFP_KERNEL); - if (ret) - return ret; - io->total_bytes += padding; - } - - ret = blk_rq_map_kern(io->req->q, io->req, seg->buff, seg->total_bytes, - GFP_KERNEL); - if (ret) - return ret; - - io->total_bytes += seg->total_bytes; - OSD_DEBUG("padding=%d buff=%p total_bytes=%d\n", padding, seg->buff, - seg->total_bytes); - return 0; -} - -static int _osd_req_finalize_set_attr_list(struct osd_request *or) -{ - struct osd_cdb_head *cdbh = osd_cdb_head(&or->cdb); - unsigned padding; - int ret; - - if (!or->set_attr.total_bytes) { - cdbh->attrs_list.set_attr_offset = OSD_OFFSET_UNUSED; - return 0; - } - - cdbh->attrs_list.set_attr_bytes = cpu_to_be32(or->set_attr.total_bytes); - cdbh->attrs_list.set_attr_offset = - osd_req_encode_offset(or, or->out.total_bytes, &padding); - - ret = _req_append_segment(or, padding, &or->set_attr, - or->out.last_seg, &or->out); - if (ret) - return ret; - - or->out.last_seg = &or->set_attr; - return 0; -} - -int osd_req_add_get_attr_list(struct osd_request *or, - const struct osd_attr *oa, unsigned nelem) -{ - unsigned total_bytes = or->enc_get_attr.total_bytes; - void *attr_last; - int ret; - - if (or->attributes_mode && - or->attributes_mode != OSD_CDB_GET_SET_ATTR_LISTS) { - WARN_ON(1); - return -EINVAL; - } - or->attributes_mode = OSD_CDB_GET_SET_ATTR_LISTS; - - /* first time calc data-in list header size */ - if (!or->get_attr.total_bytes) - or->get_attr.total_bytes = _osd_req_sizeof_alist_header(or); - - /* calc data-out info */ - if (!total_bytes) { /* first-time: allocate and put list header */ - unsigned max_bytes; - - total_bytes = _osd_req_sizeof_alist_header(or); - max_bytes = total_bytes + - nelem * sizeof(struct osd_attributes_list_attrid); - ret = _alloc_get_attr_desc(or, max_bytes); - if (ret) - return ret; - - _osd_req_set_alist_type(or, or->enc_get_attr.buff, - OSD_ATTR_LIST_GET); - } - attr_last = or->enc_get_attr.buff + total_bytes; - - for (; nelem; --nelem) { - struct osd_attributes_list_attrid *attrid; - const unsigned cur_size = sizeof(*attrid); - - total_bytes += cur_size; - if (unlikely(or->enc_get_attr.alloc_size < total_bytes)) { - or->enc_get_attr.total_bytes = total_bytes - cur_size; - ret = _alloc_get_attr_desc(or, - total_bytes + nelem * sizeof(*attrid)); - if (ret) - return ret; - attr_last = or->enc_get_attr.buff + - or->enc_get_attr.total_bytes; - } - - attrid = attr_last; - attrid->attr_page = cpu_to_be32(oa->attr_page); - attrid->attr_id = cpu_to_be32(oa->attr_id); - - attr_last += cur_size; - - /* calc data-in size */ - or->get_attr.total_bytes += - _osd_req_alist_elem_size(or, oa->len); - ++oa; - } - - or->enc_get_attr.total_bytes = total_bytes; - - OSD_DEBUG( - "get_attr.total_bytes=%u(%u) enc_get_attr.total_bytes=%u(%zu)\n", - or->get_attr.total_bytes, - or->get_attr.total_bytes - _osd_req_sizeof_alist_header(or), - or->enc_get_attr.total_bytes, - (or->enc_get_attr.total_bytes - _osd_req_sizeof_alist_header(or)) - / sizeof(struct osd_attributes_list_attrid)); - - return 0; -} -EXPORT_SYMBOL(osd_req_add_get_attr_list); - -static int _osd_req_finalize_get_attr_list(struct osd_request *or) -{ - struct osd_cdb_head *cdbh = osd_cdb_head(&or->cdb); - unsigned out_padding; - unsigned in_padding; - int ret; - - if (!or->enc_get_attr.total_bytes) { - cdbh->attrs_list.get_attr_desc_offset = OSD_OFFSET_UNUSED; - cdbh->attrs_list.get_attr_offset = OSD_OFFSET_UNUSED; - return 0; - } - - ret = _alloc_get_attr_list(or); - if (ret) - return ret; - - /* The out-going buffer info update */ - OSD_DEBUG("out-going\n"); - cdbh->attrs_list.get_attr_desc_bytes = - cpu_to_be32(or->enc_get_attr.total_bytes); - - cdbh->attrs_list.get_attr_desc_offset = - osd_req_encode_offset(or, or->out.total_bytes, &out_padding); - - ret = _req_append_segment(or, out_padding, &or->enc_get_attr, - or->out.last_seg, &or->out); - if (ret) - return ret; - or->out.last_seg = &or->enc_get_attr; - - /* The incoming buffer info update */ - OSD_DEBUG("in-coming\n"); - cdbh->attrs_list.get_attr_alloc_length = - cpu_to_be32(or->get_attr.total_bytes); - - cdbh->attrs_list.get_attr_offset = - osd_req_encode_offset(or, or->in.total_bytes, &in_padding); - - ret = _req_append_segment(or, in_padding, &or->get_attr, NULL, - &or->in); - if (ret) - return ret; - or->in.last_seg = &or->get_attr; - - return 0; -} - -int osd_req_decode_get_attr_list(struct osd_request *or, - struct osd_attr *oa, int *nelem, void **iterator) -{ - unsigned cur_bytes, returned_bytes; - int n; - const unsigned sizeof_attr_list = _osd_req_sizeof_alist_header(or); - void *cur_p; - - if (!_osd_req_is_alist_type(or, or->get_attr.buff, - OSD_ATTR_LIST_SET_RETRIEVE)) { - oa->attr_page = 0; - oa->attr_id = 0; - oa->val_ptr = NULL; - oa->len = 0; - *iterator = NULL; - return 0; - } - - if (*iterator) { - BUG_ON((*iterator < or->get_attr.buff) || - (or->get_attr.buff + or->get_attr.alloc_size < *iterator)); - cur_p = *iterator; - cur_bytes = (*iterator - or->get_attr.buff) - sizeof_attr_list; - returned_bytes = or->get_attr.total_bytes; - } else { /* first time decode the list header */ - cur_bytes = sizeof_attr_list; - returned_bytes = _osd_req_alist_size(or, or->get_attr.buff) + - sizeof_attr_list; - - cur_p = or->get_attr.buff + sizeof_attr_list; - - if (returned_bytes > or->get_attr.alloc_size) { - OSD_DEBUG("target report: space was not big enough! " - "Allocate=%u Needed=%u\n", - or->get_attr.alloc_size, - returned_bytes + sizeof_attr_list); - - returned_bytes = - or->get_attr.alloc_size - sizeof_attr_list; - } - or->get_attr.total_bytes = returned_bytes; - } - - for (n = 0; (n < *nelem) && (cur_bytes < returned_bytes); ++n) { - int inc = _osd_req_alist_elem_decode(or, cur_p, oa, - returned_bytes - cur_bytes); - - if (inc < 0) { - OSD_ERR("BAD FOOD from target. list not valid!" - "c=%d r=%d n=%d\n", - cur_bytes, returned_bytes, n); - oa->val_ptr = NULL; - cur_bytes = returned_bytes; /* break the caller loop */ - break; - } - - cur_bytes += inc; - cur_p += inc; - ++oa; - } - - *iterator = (returned_bytes - cur_bytes) ? cur_p : NULL; - *nelem = n; - return returned_bytes - cur_bytes; -} -EXPORT_SYMBOL(osd_req_decode_get_attr_list); - -/* - * Attributes Page-mode - */ - -int osd_req_add_get_attr_page(struct osd_request *or, - u32 page_id, void *attar_page, unsigned max_page_len, - const struct osd_attr *set_one_attr) -{ - struct osd_cdb_head *cdbh = osd_cdb_head(&or->cdb); - - if (or->attributes_mode && - or->attributes_mode != OSD_CDB_GET_ATTR_PAGE_SET_ONE) { - WARN_ON(1); - return -EINVAL; - } - or->attributes_mode = OSD_CDB_GET_ATTR_PAGE_SET_ONE; - - or->get_attr.buff = attar_page; - or->get_attr.total_bytes = max_page_len; - - cdbh->attrs_page.get_attr_page = cpu_to_be32(page_id); - cdbh->attrs_page.get_attr_alloc_length = cpu_to_be32(max_page_len); - - if (!set_one_attr || !set_one_attr->attr_page) - return 0; /* The set is optional */ - - or->set_attr.buff = set_one_attr->val_ptr; - or->set_attr.total_bytes = set_one_attr->len; - - cdbh->attrs_page.set_attr_page = cpu_to_be32(set_one_attr->attr_page); - cdbh->attrs_page.set_attr_id = cpu_to_be32(set_one_attr->attr_id); - cdbh->attrs_page.set_attr_length = cpu_to_be32(set_one_attr->len); - return 0; -} -EXPORT_SYMBOL(osd_req_add_get_attr_page); - -static int _osd_req_finalize_attr_page(struct osd_request *or) -{ - struct osd_cdb_head *cdbh = osd_cdb_head(&or->cdb); - unsigned in_padding, out_padding; - int ret; - - /* returned page */ - cdbh->attrs_page.get_attr_offset = - osd_req_encode_offset(or, or->in.total_bytes, &in_padding); - - ret = _req_append_segment(or, in_padding, &or->get_attr, NULL, - &or->in); - if (ret) - return ret; - - if (or->set_attr.total_bytes == 0) - return 0; - - /* set one value */ - cdbh->attrs_page.set_attr_offset = - osd_req_encode_offset(or, or->out.total_bytes, &out_padding); - - ret = _req_append_segment(or, out_padding, &or->set_attr, NULL, - &or->out); - return ret; -} - -static inline void osd_sec_parms_set_out_offset(bool is_v1, - struct osd_security_parameters *sec_parms, osd_cdb_offset offset) -{ - if (is_v1) - sec_parms->v1.data_out_integrity_check_offset = offset; - else - sec_parms->v2.data_out_integrity_check_offset = offset; -} - -static inline void osd_sec_parms_set_in_offset(bool is_v1, - struct osd_security_parameters *sec_parms, osd_cdb_offset offset) -{ - if (is_v1) - sec_parms->v1.data_in_integrity_check_offset = offset; - else - sec_parms->v2.data_in_integrity_check_offset = offset; -} - -static int _osd_req_finalize_data_integrity(struct osd_request *or, - bool has_in, bool has_out, struct bio *out_data_bio, u64 out_data_bytes, - const u8 *cap_key) -{ - struct osd_security_parameters *sec_parms = _osd_req_sec_params(or); - int ret; - - if (!osd_is_sec_alldata(sec_parms)) - return 0; - - if (has_out) { - struct _osd_req_data_segment seg = { - .buff = &or->out_data_integ, - .total_bytes = sizeof(or->out_data_integ), - }; - unsigned pad; - - or->out_data_integ.data_bytes = cpu_to_be64(out_data_bytes); - or->out_data_integ.set_attributes_bytes = cpu_to_be64( - or->set_attr.total_bytes); - or->out_data_integ.get_attributes_bytes = cpu_to_be64( - or->enc_get_attr.total_bytes); - - osd_sec_parms_set_out_offset(osd_req_is_ver1(or), sec_parms, - osd_req_encode_offset(or, or->out.total_bytes, &pad)); - - ret = _req_append_segment(or, pad, &seg, or->out.last_seg, - &or->out); - if (ret) - return ret; - or->out.last_seg = NULL; - - /* they are now all chained to request sign them all together */ - osd_sec_sign_data(&or->out_data_integ, out_data_bio, - cap_key); - } - - if (has_in) { - struct _osd_req_data_segment seg = { - .buff = &or->in_data_integ, - .total_bytes = sizeof(or->in_data_integ), - }; - unsigned pad; - - osd_sec_parms_set_in_offset(osd_req_is_ver1(or), sec_parms, - osd_req_encode_offset(or, or->in.total_bytes, &pad)); - - ret = _req_append_segment(or, pad, &seg, or->in.last_seg, - &or->in); - if (ret) - return ret; - - or->in.last_seg = NULL; - } - - return 0; -} - -/* - * osd_finalize_request and helpers - */ -static struct request *_make_request(struct request_queue *q, bool has_write, - struct _osd_io_info *oii) -{ - struct request *req; - struct bio *bio = oii->bio; - int ret; - - req = blk_get_request(q, has_write ? REQ_OP_SCSI_OUT : REQ_OP_SCSI_IN, - 0); - if (IS_ERR(req)) - return req; - - for_each_bio(bio) { - struct bio *bounce_bio = bio; - - ret = blk_rq_append_bio(req, &bounce_bio); - if (ret) - return ERR_PTR(ret); - } - - return req; -} - -static int _init_blk_request(struct osd_request *or, - bool has_in, bool has_out) -{ - struct scsi_device *scsi_device = or->osd_dev->scsi_device; - struct request_queue *q = scsi_device->request_queue; - struct request *req; - int ret; - - req = _make_request(q, has_out, has_out ? &or->out : &or->in); - if (IS_ERR(req)) { - ret = PTR_ERR(req); - goto out; - } - - or->request = req; - req->rq_flags |= RQF_QUIET; - - req->timeout = or->timeout; - scsi_req(req)->retries = or->retries; - - if (has_out) { - or->out.req = req; - if (has_in) { - /* allocate bidi request */ - req = _make_request(q, false, &or->in); - if (IS_ERR(req)) { - OSD_DEBUG("blk_get_request for bidi failed\n"); - ret = PTR_ERR(req); - goto out; - } - or->in.req = or->request->next_rq = req; - } - } else if (has_in) - or->in.req = req; - - ret = 0; -out: - OSD_DEBUG("or=%p has_in=%d has_out=%d => %d, %p\n", - or, has_in, has_out, ret, or->request); - return ret; -} - -int osd_finalize_request(struct osd_request *or, - u8 options, const void *cap, const u8 *cap_key) -{ - struct osd_cdb_head *cdbh = osd_cdb_head(&or->cdb); - bool has_in, has_out; - /* Save for data_integrity without the cdb_continuation */ - struct bio *out_data_bio = or->out.bio; - u64 out_data_bytes = or->out.total_bytes; - int ret; - - if (options & OSD_REQ_FUA) - cdbh->options |= OSD_CDB_FUA; - - if (options & OSD_REQ_DPO) - cdbh->options |= OSD_CDB_DPO; - - if (options & OSD_REQ_BYPASS_TIMESTAMPS) - cdbh->timestamp_control = OSD_CDB_BYPASS_TIMESTAMPS; - - osd_set_caps(&or->cdb, cap); - - has_in = or->in.bio || or->get_attr.total_bytes; - has_out = or->out.bio || or->cdb_cont.total_bytes || - or->set_attr.total_bytes || or->enc_get_attr.total_bytes; - - ret = _osd_req_finalize_cdb_cont(or, cap_key); - if (ret) { - OSD_DEBUG("_osd_req_finalize_cdb_cont failed\n"); - return ret; - } - ret = _init_blk_request(or, has_in, has_out); - if (ret) { - OSD_DEBUG("_init_blk_request failed\n"); - return ret; - } - - or->out.pad_buff = sg_out_pad_buffer; - or->in.pad_buff = sg_in_pad_buffer; - - if (!or->attributes_mode) - or->attributes_mode = OSD_CDB_GET_SET_ATTR_LISTS; - cdbh->command_specific_options |= or->attributes_mode; - if (or->attributes_mode == OSD_CDB_GET_ATTR_PAGE_SET_ONE) { - ret = _osd_req_finalize_attr_page(or); - if (ret) { - OSD_DEBUG("_osd_req_finalize_attr_page failed\n"); - return ret; - } - } else { - /* TODO: I think that for the GET_ATTR command these 2 should - * be reversed to keep them in execution order (for embedded - * targets with low memory footprint) - */ - ret = _osd_req_finalize_set_attr_list(or); - if (ret) { - OSD_DEBUG("_osd_req_finalize_set_attr_list failed\n"); - return ret; - } - - ret = _osd_req_finalize_get_attr_list(or); - if (ret) { - OSD_DEBUG("_osd_req_finalize_get_attr_list failed\n"); - return ret; - } - } - - ret = _osd_req_finalize_data_integrity(or, has_in, has_out, - out_data_bio, out_data_bytes, - cap_key); - if (ret) - return ret; - - osd_sec_sign_cdb(&or->cdb, cap_key); - - scsi_req(or->request)->cmd = or->cdb.buff; - scsi_req(or->request)->cmd_len = _osd_req_cdb_len(or); - - return 0; -} -EXPORT_SYMBOL(osd_finalize_request); - -static bool _is_osd_security_code(int code) -{ - return (code == osd_security_audit_value_frozen) || - (code == osd_security_working_key_frozen) || - (code == osd_nonce_not_unique) || - (code == osd_nonce_timestamp_out_of_range) || - (code == osd_invalid_dataout_buffer_integrity_check_value); -} - -#define OSD_SENSE_PRINT1(fmt, a...) \ - do { \ - if (__cur_sense_need_output) \ - OSD_ERR(fmt, ##a); \ - } while (0) - -#define OSD_SENSE_PRINT2(fmt, a...) OSD_SENSE_PRINT1(" " fmt, ##a) - -int osd_req_decode_sense_full(struct osd_request *or, - struct osd_sense_info *osi, bool silent, - struct osd_obj_id *bad_obj_list __unused, int max_obj __unused, - struct osd_attr *bad_attr_list, int max_attr) -{ - int sense_len, original_sense_len; - struct osd_sense_info local_osi; - struct scsi_sense_descriptor_based *ssdb; - void *cur_descriptor; -#if (CONFIG_SCSI_OSD_DPRINT_SENSE == 0) - const bool __cur_sense_need_output = false; -#else - bool __cur_sense_need_output = !silent; -#endif - int ret; - - if (likely(!or->req_errors)) - return 0; - - osi = osi ? : &local_osi; - memset(osi, 0, sizeof(*osi)); - - ssdb = (typeof(ssdb))or->sense; - sense_len = or->sense_len; - if ((sense_len < (int)sizeof(*ssdb) || !ssdb->sense_key)) { - OSD_ERR("Block-layer returned error(0x%x) but " - "sense_len(%u) || key(%d) is empty\n", - or->req_errors, sense_len, ssdb->sense_key); - goto analyze; - } - - if ((ssdb->response_code != 0x72) && (ssdb->response_code != 0x73)) { - OSD_ERR("Unrecognized scsi sense: rcode=%x length=%d\n", - ssdb->response_code, sense_len); - goto analyze; - } - - osi->key = ssdb->sense_key; - osi->additional_code = be16_to_cpu(ssdb->additional_sense_code); - original_sense_len = ssdb->additional_sense_length + 8; - -#if (CONFIG_SCSI_OSD_DPRINT_SENSE == 1) - if (__cur_sense_need_output) - __cur_sense_need_output = (osi->key > scsi_sk_recovered_error); -#endif - OSD_SENSE_PRINT1("Main Sense information key=0x%x length(%d, %d) " - "additional_code=0x%x async_error=%d errors=0x%x\n", - osi->key, original_sense_len, sense_len, - osi->additional_code, or->async_error, - or->req_errors); - - if (original_sense_len < sense_len) - sense_len = original_sense_len; - - cur_descriptor = ssdb->ssd; - sense_len -= sizeof(*ssdb); - while (sense_len > 0) { - struct scsi_sense_descriptor *ssd = cur_descriptor; - int cur_len = ssd->additional_length + 2; - - sense_len -= cur_len; - - if (sense_len < 0) - break; /* sense was truncated */ - - switch (ssd->descriptor_type) { - case scsi_sense_information: - case scsi_sense_command_specific_information: - { - struct scsi_sense_command_specific_data_descriptor - *sscd = cur_descriptor; - - osi->command_info = - get_unaligned_be64(&sscd->information) ; - OSD_SENSE_PRINT2( - "command_specific_information 0x%llx \n", - _LLU(osi->command_info)); - break; - } - case scsi_sense_key_specific: - { - struct scsi_sense_key_specific_data_descriptor - *ssks = cur_descriptor; - - osi->sense_info = get_unaligned_be16(&ssks->value); - OSD_SENSE_PRINT2( - "sense_key_specific_information %u" - "sksv_cd_bpv_bp (0x%x)\n", - osi->sense_info, ssks->sksv_cd_bpv_bp); - break; - } - case osd_sense_object_identification: - { /*FIXME: Keep first not last, Store in array*/ - struct osd_sense_identification_data_descriptor - *osidd = cur_descriptor; - - osi->not_initiated_command_functions = - le32_to_cpu(osidd->not_initiated_functions); - osi->completed_command_functions = - le32_to_cpu(osidd->completed_functions); - osi->obj.partition = be64_to_cpu(osidd->partition_id); - osi->obj.id = be64_to_cpu(osidd->object_id); - OSD_SENSE_PRINT2( - "object_identification pid=0x%llx oid=0x%llx\n", - _LLU(osi->obj.partition), _LLU(osi->obj.id)); - OSD_SENSE_PRINT2( - "not_initiated_bits(%x) " - "completed_command_bits(%x)\n", - osi->not_initiated_command_functions, - osi->completed_command_functions); - break; - } - case osd_sense_response_integrity_check: - { - struct osd_sense_response_integrity_check_descriptor - *d = cur_descriptor; - /* 2nibbles+space+ASCII */ - char dump[sizeof(d->integrity_check_value) * 4 + 2]; - - hex_dump_to_buffer(d->integrity_check_value, - sizeof(d->integrity_check_value), - 32, 1, dump, sizeof(dump), true); - OSD_SENSE_PRINT2("response_integrity [%s]\n", dump); - } - case osd_sense_attribute_identification: - { - struct osd_sense_attributes_data_descriptor - *osadd = cur_descriptor; - unsigned len = min(cur_len, sense_len); - struct osd_sense_attr *pattr = osadd->sense_attrs; - - while (len >= sizeof(*pattr)) { - u32 attr_page = be32_to_cpu(pattr->attr_page); - u32 attr_id = be32_to_cpu(pattr->attr_id); - - if (!osi->attr.attr_page) { - osi->attr.attr_page = attr_page; - osi->attr.attr_id = attr_id; - } - - if (bad_attr_list && max_attr) { - bad_attr_list->attr_page = attr_page; - bad_attr_list->attr_id = attr_id; - bad_attr_list++; - max_attr--; - } - - len -= sizeof(*pattr); - OSD_SENSE_PRINT2( - "osd_sense_attribute_identification" - "attr_page=0x%x attr_id=0x%x\n", - attr_page, attr_id); - } - } - /*These are not legal for OSD*/ - case scsi_sense_field_replaceable_unit: - OSD_SENSE_PRINT2("scsi_sense_field_replaceable_unit\n"); - break; - case scsi_sense_stream_commands: - OSD_SENSE_PRINT2("scsi_sense_stream_commands\n"); - break; - case scsi_sense_block_commands: - OSD_SENSE_PRINT2("scsi_sense_block_commands\n"); - break; - case scsi_sense_ata_return: - OSD_SENSE_PRINT2("scsi_sense_ata_return\n"); - break; - default: - if (ssd->descriptor_type <= scsi_sense_Reserved_last) - OSD_SENSE_PRINT2( - "scsi_sense Reserved descriptor (0x%x)", - ssd->descriptor_type); - else - OSD_SENSE_PRINT2( - "scsi_sense Vendor descriptor (0x%x)", - ssd->descriptor_type); - } - - cur_descriptor += cur_len; - } - -analyze: - if (!osi->key) { - /* scsi sense is Empty, the request was never issued to target - * linux return code might tell us what happened. - */ - if (or->async_error == BLK_STS_RESOURCE) - osi->osd_err_pri = OSD_ERR_PRI_RESOURCE; - else - osi->osd_err_pri = OSD_ERR_PRI_UNREACHABLE; - ret = or->async_error; - } else if (osi->key <= scsi_sk_recovered_error) { - osi->osd_err_pri = 0; - ret = 0; - } else if (osi->additional_code == scsi_invalid_field_in_cdb) { - if (osi->cdb_field_offset == OSD_CFO_STARTING_BYTE) { - osi->osd_err_pri = OSD_ERR_PRI_CLEAR_PAGES; - ret = -EFAULT; /* caller should recover from this */ - } else if (osi->cdb_field_offset == OSD_CFO_OBJECT_ID) { - osi->osd_err_pri = OSD_ERR_PRI_NOT_FOUND; - ret = -ENOENT; - } else if (osi->cdb_field_offset == OSD_CFO_PERMISSIONS) { - osi->osd_err_pri = OSD_ERR_PRI_NO_ACCESS; - ret = -EACCES; - } else { - osi->osd_err_pri = OSD_ERR_PRI_BAD_CRED; - ret = -EINVAL; - } - } else if (osi->additional_code == osd_quota_error) { - osi->osd_err_pri = OSD_ERR_PRI_NO_SPACE; - ret = -ENOSPC; - } else if (_is_osd_security_code(osi->additional_code)) { - osi->osd_err_pri = OSD_ERR_PRI_BAD_CRED; - ret = -EINVAL; - } else { - osi->osd_err_pri = OSD_ERR_PRI_EIO; - ret = -EIO; - } - - if (!or->out.residual) - or->out.residual = or->out.total_bytes; - if (!or->in.residual) - or->in.residual = or->in.total_bytes; - - return ret; -} -EXPORT_SYMBOL(osd_req_decode_sense_full); - -/* - * Implementation of osd_sec.h API - * TODO: Move to a separate osd_sec.c file at a later stage. - */ - -enum { OSD_SEC_CAP_V1_ALL_CAPS = - OSD_SEC_CAP_APPEND | OSD_SEC_CAP_OBJ_MGMT | OSD_SEC_CAP_REMOVE | - OSD_SEC_CAP_CREATE | OSD_SEC_CAP_SET_ATTR | OSD_SEC_CAP_GET_ATTR | - OSD_SEC_CAP_WRITE | OSD_SEC_CAP_READ | OSD_SEC_CAP_POL_SEC | - OSD_SEC_CAP_GLOBAL | OSD_SEC_CAP_DEV_MGMT -}; - -enum { OSD_SEC_CAP_V2_ALL_CAPS = - OSD_SEC_CAP_V1_ALL_CAPS | OSD_SEC_CAP_QUERY | OSD_SEC_CAP_M_OBJECT -}; - -void osd_sec_init_nosec_doall_caps(void *caps, - const struct osd_obj_id *obj, bool is_collection, const bool is_v1) -{ - struct osd_capability *cap = caps; - u8 type; - u8 descriptor_type; - - if (likely(obj->id)) { - if (unlikely(is_collection)) { - type = OSD_SEC_OBJ_COLLECTION; - descriptor_type = is_v1 ? OSD_SEC_OBJ_DESC_OBJ : - OSD_SEC_OBJ_DESC_COL; - } else { - type = OSD_SEC_OBJ_USER; - descriptor_type = OSD_SEC_OBJ_DESC_OBJ; - } - WARN_ON(!obj->partition); - } else { - type = obj->partition ? OSD_SEC_OBJ_PARTITION : - OSD_SEC_OBJ_ROOT; - descriptor_type = OSD_SEC_OBJ_DESC_PAR; - } - - memset(cap, 0, sizeof(*cap)); - - cap->h.format = OSD_SEC_CAP_FORMAT_VER1; - cap->h.integrity_algorithm__key_version = 0; /* MAKE_BYTE(0, 0); */ - cap->h.security_method = OSD_SEC_NOSEC; -/* cap->expiration_time; - cap->AUDIT[30-10]; - cap->discriminator[42-30]; - cap->object_created_time; */ - cap->h.object_type = type; - osd_sec_set_caps(&cap->h, OSD_SEC_CAP_V1_ALL_CAPS); - cap->h.object_descriptor_type = descriptor_type; - cap->od.obj_desc.policy_access_tag = 0; - cap->od.obj_desc.allowed_partition_id = cpu_to_be64(obj->partition); - cap->od.obj_desc.allowed_object_id = cpu_to_be64(obj->id); -} -EXPORT_SYMBOL(osd_sec_init_nosec_doall_caps); - -/* FIXME: Extract version from caps pointer. - * Also Pete's target only supports caps from OSDv1 for now - */ -void osd_set_caps(struct osd_cdb *cdb, const void *caps) -{ - /* NOTE: They start at same address */ - memcpy(&cdb->v1.caps, caps, OSDv1_CAP_LEN); -} - -bool osd_is_sec_alldata(struct osd_security_parameters *sec_parms __unused) -{ - return false; -} - -void osd_sec_sign_cdb(struct osd_cdb *ocdb __unused, const u8 *cap_key __unused) -{ -} - -void osd_sec_sign_data(void *data_integ __unused, - struct bio *bio __unused, const u8 *cap_key __unused) -{ -} - -/* - * Declared in osd_protocol.h - * 4.12.5 Data-In and Data-Out buffer offsets - * byte offset = mantissa * (2^(exponent+8)) - * Returns the smallest allowed encoded offset that contains given @offset - * The actual encoded offset returned is @offset + *@padding. - */ -osd_cdb_offset __osd_encode_offset( - u64 offset, unsigned *padding, int min_shift, int max_shift) -{ - u64 try_offset = -1, mod, align; - osd_cdb_offset be32_offset; - int shift; - - *padding = 0; - if (!offset) - return 0; - - for (shift = min_shift; shift < max_shift; ++shift) { - try_offset = offset >> shift; - if (try_offset < (1 << OSD_OFFSET_MAX_BITS)) - break; - } - - BUG_ON(shift == max_shift); - - align = 1 << shift; - mod = offset & (align - 1); - if (mod) { - *padding = align - mod; - try_offset += 1; - } - - try_offset |= ((shift - 8) & 0xf) << 28; - be32_offset = cpu_to_be32((u32)try_offset); - - OSD_DEBUG("offset=%llu mantissa=%llu exp=%d encoded=%x pad=%d\n", - _LLU(offset), _LLU(try_offset & 0x0FFFFFFF), shift, - be32_offset, *padding); - return be32_offset; -} diff --git a/drivers/scsi/osd/osd_uld.c b/drivers/scsi/osd/osd_uld.c deleted file mode 100644 index eaf36ccf58db..000000000000 --- a/drivers/scsi/osd/osd_uld.c +++ /dev/null @@ -1,571 +0,0 @@ -/* - * osd_uld.c - OSD Upper Layer Driver - * - * A Linux driver module that registers as a SCSI ULD and probes - * for OSD type SCSI devices. - * It's main function is to export osd devices to in-kernel users like - * osdfs and pNFS-objects-LD. It also provides one ioctl for running - * in Kernel tests. - * - * Copyright (C) 2008 Panasas Inc. All rights reserved. - * - * Authors: - * Boaz Harrosh <ooo@electrozaur.com> - * Benny Halevy <bhalevy@panasas.com> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. Neither the name of the Panasas company nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED - * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR - * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include <linux/namei.h> -#include <linux/cdev.h> -#include <linux/fs.h> -#include <linux/module.h> -#include <linux/device.h> -#include <linux/idr.h> -#include <linux/major.h> -#include <linux/file.h> -#include <linux/slab.h> - -#include <scsi/scsi.h> -#include <scsi/scsi_driver.h> -#include <scsi/scsi_device.h> -#include <scsi/scsi_ioctl.h> - -#include <scsi/osd_initiator.h> -#include <scsi/osd_sec.h> - -#include "osd_debug.h" - -#ifndef TYPE_OSD -# define TYPE_OSD 0x11 -#endif - -#ifndef SCSI_OSD_MAJOR -# define SCSI_OSD_MAJOR 260 -#endif -#define SCSI_OSD_MAX_MINOR MINORMASK - -static const char osd_name[] = "osd"; -static const char *osd_version_string = "open-osd 0.2.1"; - -MODULE_AUTHOR("Boaz Harrosh <ooo@electrozaur.com>"); -MODULE_DESCRIPTION("open-osd Upper-Layer-Driver osd.ko"); -MODULE_LICENSE("GPL"); -MODULE_ALIAS_CHARDEV_MAJOR(SCSI_OSD_MAJOR); -MODULE_ALIAS_SCSI_DEVICE(TYPE_OSD); - -struct osd_uld_device { - int minor; - struct device class_dev; - struct cdev cdev; - struct osd_dev od; - struct osd_dev_info odi; - struct gendisk *disk; -}; - -struct osd_dev_handle { - struct osd_dev od; - struct file *file; - struct osd_uld_device *oud; -} ; - -static DEFINE_IDA(osd_minor_ida); - -/* - * scsi sysfs attribute operations - */ -static ssize_t osdname_show(struct device *dev, struct device_attribute *attr, - char *buf) -{ - struct osd_uld_device *ould = container_of(dev, struct osd_uld_device, - class_dev); - return sprintf(buf, "%s\n", ould->odi.osdname); -} -static DEVICE_ATTR_RO(osdname); - -static ssize_t systemid_show(struct device *dev, struct device_attribute *attr, - char *buf) -{ - struct osd_uld_device *ould = container_of(dev, struct osd_uld_device, - class_dev); - - memcpy(buf, ould->odi.systemid, ould->odi.systemid_len); - return ould->odi.systemid_len; -} -static DEVICE_ATTR_RO(systemid); - -static struct attribute *osd_uld_attrs[] = { - &dev_attr_osdname.attr, - &dev_attr_systemid.attr, - NULL, -}; -ATTRIBUTE_GROUPS(osd_uld); - -static struct class osd_uld_class = { - .owner = THIS_MODULE, - .name = "scsi_osd", - .dev_groups = osd_uld_groups, -}; - -/* - * Char Device operations - */ - -static int osd_uld_open(struct inode *inode, struct file *file) -{ - struct osd_uld_device *oud = container_of(inode->i_cdev, - struct osd_uld_device, cdev); - - get_device(&oud->class_dev); - /* cache osd_uld_device on file handle */ - file->private_data = oud; - OSD_DEBUG("osd_uld_open %p\n", oud); - return 0; -} - -static int osd_uld_release(struct inode *inode, struct file *file) -{ - struct osd_uld_device *oud = file->private_data; - - OSD_DEBUG("osd_uld_release %p\n", file->private_data); - file->private_data = NULL; - put_device(&oud->class_dev); - return 0; -} - -/* FIXME: Only one vector for now */ -unsigned g_test_ioctl; -do_test_fn *g_do_test; - -int osduld_register_test(unsigned ioctl, do_test_fn *do_test) -{ - if (g_test_ioctl) - return -EINVAL; - - g_test_ioctl = ioctl; - g_do_test = do_test; - return 0; -} -EXPORT_SYMBOL(osduld_register_test); - -void osduld_unregister_test(unsigned ioctl) -{ - if (ioctl == g_test_ioctl) { - g_test_ioctl = 0; - g_do_test = NULL; - } -} -EXPORT_SYMBOL(osduld_unregister_test); - -static do_test_fn *_find_ioctl(unsigned cmd) -{ - if (g_test_ioctl == cmd) - return g_do_test; - else - return NULL; -} - -static long osd_uld_ioctl(struct file *file, unsigned int cmd, - unsigned long arg) -{ - struct osd_uld_device *oud = file->private_data; - int ret; - do_test_fn *do_test; - - do_test = _find_ioctl(cmd); - if (do_test) - ret = do_test(&oud->od, cmd, arg); - else { - OSD_ERR("Unknown ioctl %d: osd_uld_device=%p\n", cmd, oud); - ret = -ENOIOCTLCMD; - } - return ret; -} - -static const struct file_operations osd_fops = { - .owner = THIS_MODULE, - .open = osd_uld_open, - .release = osd_uld_release, - .unlocked_ioctl = osd_uld_ioctl, - .llseek = noop_llseek, -}; - -struct osd_dev *osduld_path_lookup(const char *name) -{ - struct osd_uld_device *oud; - struct osd_dev_handle *odh; - struct file *file; - int error; - - if (!name || !*name) { - OSD_ERR("Mount with !path || !*path\n"); - return ERR_PTR(-EINVAL); - } - - odh = kzalloc(sizeof(*odh), GFP_KERNEL); - if (unlikely(!odh)) - return ERR_PTR(-ENOMEM); - - file = filp_open(name, O_RDWR, 0); - if (IS_ERR(file)) { - error = PTR_ERR(file); - goto free_od; - } - - if (file->f_op != &osd_fops){ - error = -EINVAL; - goto close_file; - } - - oud = file->private_data; - - odh->od = oud->od; - odh->file = file; - odh->oud = oud; - - return &odh->od; - -close_file: - fput(file); -free_od: - kfree(odh); - return ERR_PTR(error); -} -EXPORT_SYMBOL(osduld_path_lookup); - -static inline bool _the_same_or_null(const u8 *a1, unsigned a1_len, - const u8 *a2, unsigned a2_len) -{ - if (!a2_len) /* User string is Empty means don't care */ - return true; - - if (a1_len != a2_len) - return false; - - return 0 == memcmp(a1, a2, a1_len); -} - -static int _match_odi(struct device *dev, const void *find_data) -{ - struct osd_uld_device *oud = container_of(dev, struct osd_uld_device, - class_dev); - const struct osd_dev_info *odi = find_data; - - if (_the_same_or_null(oud->odi.systemid, oud->odi.systemid_len, - odi->systemid, odi->systemid_len) && - _the_same_or_null(oud->odi.osdname, oud->odi.osdname_len, - odi->osdname, odi->osdname_len)) { - OSD_DEBUG("found device sysid_len=%d osdname=%d\n", - odi->systemid_len, odi->osdname_len); - return 1; - } else { - return 0; - } -} - -/* osduld_info_lookup - Loop through all devices, return the requested osd_dev. - * - * if @odi->systemid_len and/or @odi->osdname_len are zero, they act as a don't - * care. .e.g if they're both zero /dev/osd0 is returned. - */ -struct osd_dev *osduld_info_lookup(const struct osd_dev_info *odi) -{ - struct device *dev = class_find_device(&osd_uld_class, NULL, odi, _match_odi); - if (likely(dev)) { - struct osd_dev_handle *odh = kzalloc(sizeof(*odh), GFP_KERNEL); - struct osd_uld_device *oud = container_of(dev, - struct osd_uld_device, class_dev); - - if (unlikely(!odh)) { - put_device(dev); - return ERR_PTR(-ENOMEM); - } - - odh->od = oud->od; - odh->oud = oud; - - return &odh->od; - } - - return ERR_PTR(-ENODEV); -} -EXPORT_SYMBOL(osduld_info_lookup); - -void osduld_put_device(struct osd_dev *od) -{ - if (od && !IS_ERR(od)) { - struct osd_dev_handle *odh = - container_of(od, struct osd_dev_handle, od); - struct osd_uld_device *oud = odh->oud; - - BUG_ON(od->scsi_device != oud->od.scsi_device); - - /* If scsi has released the device (logout), and exofs has last - * reference on oud it will be freed by above osd_uld_release - * within fput below. But this will oops in cdev_release which - * is called after the fops->release. A get_/put_ pair makes - * sure we have a cdev for the duration of fput - */ - if (odh->file) { - get_device(&oud->class_dev); - fput(odh->file); - } - put_device(&oud->class_dev); - kfree(odh); - } -} -EXPORT_SYMBOL(osduld_put_device); - -const struct osd_dev_info *osduld_device_info(struct osd_dev *od) -{ - struct osd_dev_handle *odh = - container_of(od, struct osd_dev_handle, od); - return &odh->oud->odi; -} -EXPORT_SYMBOL(osduld_device_info); - -bool osduld_device_same(struct osd_dev *od, const struct osd_dev_info *odi) -{ - struct osd_dev_handle *odh = - container_of(od, struct osd_dev_handle, od); - struct osd_uld_device *oud = odh->oud; - - return (oud->odi.systemid_len == odi->systemid_len) && - _the_same_or_null(oud->odi.systemid, oud->odi.systemid_len, - odi->systemid, odi->systemid_len) && - (oud->odi.osdname_len == odi->osdname_len) && - _the_same_or_null(oud->odi.osdname, oud->odi.osdname_len, - odi->osdname, odi->osdname_len); -} -EXPORT_SYMBOL(osduld_device_same); - -/* - * Scsi Device operations - */ - -static int __detect_osd(struct osd_uld_device *oud) -{ - struct scsi_device *scsi_device = oud->od.scsi_device; - struct scsi_sense_hdr sense_hdr; - char caps[OSD_CAP_LEN]; - int error; - - /* sending a test_unit_ready as first command seems to be needed - * by some targets - */ - OSD_DEBUG("start scsi_test_unit_ready %p %p %p\n", - oud, scsi_device, scsi_device->request_queue); - error = scsi_test_unit_ready(scsi_device, 10*HZ, 5, &sense_hdr); - if (error) - OSD_ERR("warning: scsi_test_unit_ready failed\n"); - - osd_sec_init_nosec_doall_caps(caps, &osd_root_object, false, true); - if (osd_auto_detect_ver(&oud->od, caps, &oud->odi)) - return -ENODEV; - - return 0; -} - -static void __remove(struct device *dev) -{ - struct osd_uld_device *oud = container_of(dev, struct osd_uld_device, - class_dev); - struct scsi_device *scsi_device = oud->od.scsi_device; - - kfree(oud->odi.osdname); - - osd_dev_fini(&oud->od); - scsi_device_put(scsi_device); - - OSD_INFO("osd_remove %s\n", - oud->disk ? oud->disk->disk_name : NULL); - - if (oud->disk) - put_disk(oud->disk); - - kfree(oud); -} - -static int osd_probe(struct device *dev) -{ - struct scsi_device *scsi_device = to_scsi_device(dev); - struct gendisk *disk; - struct osd_uld_device *oud; - int minor; - int error; - - if (scsi_device->type != TYPE_OSD) - return -ENODEV; - - minor = ida_alloc_max(&osd_minor_ida, SCSI_OSD_MAX_MINOR, GFP_KERNEL); - if (minor == -ENOSPC) - return -EBUSY; - if (minor < 0) - return -ENODEV; - - error = -ENOMEM; - oud = kzalloc(sizeof(*oud), GFP_KERNEL); - if (NULL == oud) - goto err_retract_minor; - - /* class device member */ - device_initialize(&oud->class_dev); - dev_set_drvdata(dev, oud); - oud->minor = minor; - oud->class_dev.devt = MKDEV(SCSI_OSD_MAJOR, oud->minor); - oud->class_dev.class = &osd_uld_class; - oud->class_dev.parent = dev; - oud->class_dev.release = __remove; - - /* hold one more reference to the scsi_device that will get released - * in __release, in case a logout is happening while fs is mounted - */ - if (scsi_device_get(scsi_device)) - goto err_retract_minor; - osd_dev_init(&oud->od, scsi_device); - - /* allocate a disk and set it up */ - /* FIXME: do we need this since sg has already done that */ - disk = alloc_disk(1); - if (!disk) { - OSD_ERR("alloc_disk failed\n"); - goto err_free_osd; - } - disk->major = SCSI_OSD_MAJOR; - disk->first_minor = oud->minor; - sprintf(disk->disk_name, "osd%d", oud->minor); - oud->disk = disk; - - /* Detect the OSD Version */ - error = __detect_osd(oud); - if (error) { - OSD_ERR("osd detection failed, non-compatible OSD device\n"); - goto err_free_osd; - } - - /* init the char-device for communication with user-mode */ - cdev_init(&oud->cdev, &osd_fops); - oud->cdev.owner = THIS_MODULE; - - error = dev_set_name(&oud->class_dev, "%s", disk->disk_name); - if (error) { - OSD_ERR("dev_set_name failed => %d\n", error); - goto err_free_osd; - } - - error = cdev_device_add(&oud->cdev, &oud->class_dev); - if (error) { - OSD_ERR("device_register failed => %d\n", error); - goto err_free_osd; - } - - OSD_INFO("osd_probe %s\n", disk->disk_name); - return 0; - -err_free_osd: - put_device(&oud->class_dev); -err_retract_minor: - ida_free(&osd_minor_ida, minor); - return error; -} - -static int osd_remove(struct device *dev) -{ - struct scsi_device *scsi_device = to_scsi_device(dev); - struct osd_uld_device *oud = dev_get_drvdata(dev); - - if (oud->od.scsi_device != scsi_device) { - OSD_ERR("Half cooked osd-device %p, || %p!=%p", - dev, oud->od.scsi_device, scsi_device); - } - - cdev_device_del(&oud->cdev, &oud->class_dev); - ida_free(&osd_minor_ida, oud->minor); - put_device(&oud->class_dev); - - return 0; -} - -/* - * Global driver and scsi registration - */ - -static struct scsi_driver osd_driver = { - .gendrv = { - .name = osd_name, - .owner = THIS_MODULE, - .probe = osd_probe, - .remove = osd_remove, - } -}; - -static int __init osd_uld_init(void) -{ - int err; - - err = class_register(&osd_uld_class); - if (err) { - OSD_ERR("Unable to register sysfs class => %d\n", err); - return err; - } - - err = register_chrdev_region(MKDEV(SCSI_OSD_MAJOR, 0), - SCSI_OSD_MAX_MINOR, osd_name); - if (err) { - OSD_ERR("Unable to register major %d for osd ULD => %d\n", - SCSI_OSD_MAJOR, err); - goto err_out; - } - - err = scsi_register_driver(&osd_driver.gendrv); - if (err) { - OSD_ERR("scsi_register_driver failed => %d\n", err); - goto err_out_chrdev; - } - - OSD_INFO("LOADED %s\n", osd_version_string); - return 0; - -err_out_chrdev: - unregister_chrdev_region(MKDEV(SCSI_OSD_MAJOR, 0), SCSI_OSD_MAX_MINOR); -err_out: - class_unregister(&osd_uld_class); - return err; -} - -static void __exit osd_uld_exit(void) -{ - scsi_unregister_driver(&osd_driver.gendrv); - unregister_chrdev_region(MKDEV(SCSI_OSD_MAJOR, 0), SCSI_OSD_MAX_MINOR); - class_unregister(&osd_uld_class); - OSD_INFO("UNLOADED %s\n", osd_version_string); -} - -module_init(osd_uld_init); -module_exit(osd_uld_exit); diff --git a/drivers/scsi/osst.c b/drivers/scsi/osst.c index 664c1238a87f..be3c73ebbfde 100644 --- a/drivers/scsi/osst.c +++ b/drivers/scsi/osst.c @@ -139,7 +139,7 @@ static int debugging = 1; #define OSST_TIMEOUT (200 * HZ) #define OSST_LONG_TIMEOUT (1800 * HZ) -#define TAPE_NR(x) (iminor(x) & ~(-1 << ST_MODE_SHIFT)) +#define TAPE_NR(x) (iminor(x) & ((1 << ST_MODE_SHIFT)-1)) #define TAPE_MODE(x) ((iminor(x) & ST_MODE_MASK) >> ST_MODE_SHIFT) #define TAPE_REWIND(x) ((iminor(x) & 0x80) == 0) #define TAPE_IS_RAW(x) (TAPE_MODE(x) & (ST_NBR_MODES >> 1)) diff --git a/drivers/scsi/pcmcia/Makefile b/drivers/scsi/pcmcia/Makefile index faa87a4b2d2b..a5a24dd44e7e 100644 --- a/drivers/scsi/pcmcia/Makefile +++ b/drivers/scsi/pcmcia/Makefile @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 -ccflags-y := -Idrivers/scsi +ccflags-y := -I $(srctree)/drivers/scsi # 16-bit client drivers obj-$(CONFIG_PCMCIA_QLOGIC) += qlogic_cs.o diff --git a/drivers/scsi/pcmcia/nsp_cs.c b/drivers/scsi/pcmcia/nsp_cs.c index 1bd6825a4f14..a81748e6e8fb 100644 --- a/drivers/scsi/pcmcia/nsp_cs.c +++ b/drivers/scsi/pcmcia/nsp_cs.c @@ -1134,7 +1134,8 @@ static irqreturn_t nspintr(int irq, void *dev_id) //*sync_neg = SYNC_NOT_YET; - if ((tmpSC->SCp.Message == MSG_COMMAND_COMPLETE)) { /* all command complete and return status */ + /* all command complete and return status */ + if (tmpSC->SCp.Message == MSG_COMMAND_COMPLETE) { tmpSC->result = (DID_OK << 16) | ((tmpSC->SCp.Message & 0xff) << 8) | ((tmpSC->SCp.Status & 0xff) << 0); diff --git a/drivers/scsi/qedf/qedf_debugfs.c b/drivers/scsi/qedf/qedf_debugfs.c index c29c162a494f..a32d8ee4666e 100644 --- a/drivers/scsi/qedf/qedf_debugfs.c +++ b/drivers/scsi/qedf/qedf_debugfs.c @@ -27,30 +27,19 @@ qedf_dbg_host_init(struct qedf_dbg_ctx *qedf, const struct file_operations *fops) { char host_dirname[32]; - struct dentry *file_dentry = NULL; QEDF_INFO(qedf, QEDF_LOG_DEBUGFS, "Creating debugfs host node\n"); /* create pf dir */ sprintf(host_dirname, "host%u", qedf->host_no); qedf->bdf_dentry = debugfs_create_dir(host_dirname, qedf_dbg_root); - if (!qedf->bdf_dentry) - return; /* create debugfs files */ while (dops) { if (!(dops->name)) break; - file_dentry = debugfs_create_file(dops->name, 0600, - qedf->bdf_dentry, qedf, - fops); - if (!file_dentry) { - QEDF_INFO(qedf, QEDF_LOG_DEBUGFS, - "Debugfs entry %s creation failed\n", - dops->name); - debugfs_remove_recursive(qedf->bdf_dentry); - return; - } + debugfs_create_file(dops->name, 0600, qedf->bdf_dentry, qedf, + fops); dops++; fops++; } @@ -80,9 +69,6 @@ qedf_dbg_init(char *drv_name) /* create qed dir in root of debugfs. NULL means debugfs root */ qedf_dbg_root = debugfs_create_dir(drv_name, NULL); - if (!qedf_dbg_root) - QEDF_INFO(NULL, QEDF_LOG_DEBUGFS, "Init of debugfs " - "failed\n"); } /** diff --git a/drivers/scsi/qedf/qedf_io.c b/drivers/scsi/qedf/qedf_io.c index 6bbc38b1b465..6ca583bdde23 100644 --- a/drivers/scsi/qedf/qedf_io.c +++ b/drivers/scsi/qedf/qedf_io.c @@ -1128,12 +1128,6 @@ void qedf_scsi_completion(struct qedf_ctx *qedf, struct fcoe_cqe *cqe, return; } - if (!sc_cmd->request->special) { - QEDF_WARN(&(qedf->dbg_ctx), "request->special is NULL so " - "request not valid, sc_cmd=%p.\n", sc_cmd); - return; - } - if (!sc_cmd->request->q) { QEDF_WARN(&(qedf->dbg_ctx), "request->q is NULL so request " "is not valid, sc_cmd=%p.\n", sc_cmd); diff --git a/drivers/scsi/qedf/qedf_main.c b/drivers/scsi/qedf/qedf_main.c index 9bbc19fc190b..9f9431a4cc0e 100644 --- a/drivers/scsi/qedf/qedf_main.c +++ b/drivers/scsi/qedf/qedf_main.c @@ -1418,7 +1418,7 @@ static struct libfc_function_template qedf_lport_template = { static void qedf_fcoe_ctlr_setup(struct qedf_ctx *qedf) { - fcoe_ctlr_init(&qedf->ctlr, FIP_ST_AUTO); + fcoe_ctlr_init(&qedf->ctlr, FIP_MODE_AUTO); qedf->ctlr.send = qedf_fip_send; qedf->ctlr.get_src_addr = qedf_get_src_mac; diff --git a/drivers/scsi/qedi/qedi_debugfs.c b/drivers/scsi/qedi/qedi_debugfs.c index fd914ca4149a..5667e4752e2e 100644 --- a/drivers/scsi/qedi/qedi_debugfs.c +++ b/drivers/scsi/qedi/qedi_debugfs.c @@ -23,27 +23,16 @@ qedi_dbg_host_init(struct qedi_dbg_ctx *qedi, const struct file_operations *fops) { char host_dirname[32]; - struct dentry *file_dentry = NULL; sprintf(host_dirname, "host%u", qedi->host_no); qedi->bdf_dentry = debugfs_create_dir(host_dirname, qedi_dbg_root); - if (!qedi->bdf_dentry) - return; while (dops) { if (!(dops->name)) break; - file_dentry = debugfs_create_file(dops->name, 0600, - qedi->bdf_dentry, qedi, - fops); - if (!file_dentry) { - QEDI_INFO(qedi, QEDI_LOG_DEBUGFS, - "Debugfs entry %s creation failed\n", - dops->name); - debugfs_remove_recursive(qedi->bdf_dentry); - return; - } + debugfs_create_file(dops->name, 0600, qedi->bdf_dentry, qedi, + fops); dops++; fops++; } @@ -60,8 +49,6 @@ void qedi_dbg_init(char *drv_name) { qedi_dbg_root = debugfs_create_dir(drv_name, NULL); - if (!qedi_dbg_root) - QEDI_INFO(NULL, QEDI_LOG_DEBUGFS, "Init of debugfs failed\n"); } void diff --git a/drivers/scsi/qedi/qedi_fw.c b/drivers/scsi/qedi/qedi_fw.c index 25d763ae5d5a..e2a995a6e8e7 100644 --- a/drivers/scsi/qedi/qedi_fw.c +++ b/drivers/scsi/qedi/qedi_fw.c @@ -616,13 +616,6 @@ static void qedi_scsi_completion(struct qedi_ctx *qedi, goto error; } - if (!sc_cmd->request->special) { - QEDI_WARN(&qedi->dbg_ctx, - "request->special is NULL so request not valid, sc_cmd=%p.\n", - sc_cmd); - goto error; - } - if (!sc_cmd->request->q) { QEDI_WARN(&qedi->dbg_ctx, "request->q is NULL so request is not valid, sc_cmd=%p.\n", diff --git a/drivers/scsi/qla2xxx/qla_attr.c b/drivers/scsi/qla2xxx/qla_attr.c index ac504a1ff0ff..2eb1ae721a7d 100644 --- a/drivers/scsi/qla2xxx/qla_attr.c +++ b/drivers/scsi/qla2xxx/qla_attr.c @@ -543,6 +543,9 @@ qla2x00_sysfs_write_vpd(struct file *filp, struct kobject *kobj, if (unlikely(pci_channel_offline(ha->pdev))) return 0; + if (qla2x00_chip_is_down(vha)) + return 0; + if (!capable(CAP_SYS_ADMIN) || off != 0 || count != ha->vpd_size || !ha->isp_ops->write_nvram) return 0; @@ -1002,7 +1005,7 @@ qla2x00_free_sysfs_attr(scsi_qla_host_t *vha, bool stop_beacon) /* Scsi_Host attributes. */ static ssize_t -qla2x00_drvr_version_show(struct device *dev, +qla2x00_driver_version_show(struct device *dev, struct device_attribute *attr, char *buf) { return scnprintf(buf, PAGE_SIZE, "%s\n", qla2x00_version_str); @@ -1632,6 +1635,92 @@ qla2x00_max_speed_sup_show(struct device *dev, struct device_attribute *attr, ha->max_speed_sup ? "32Gps" : "16Gps"); } +static ssize_t +qla2x00_port_speed_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct scsi_qla_host *vha = shost_priv(dev_to_shost(dev)); + ulong type, speed; + int oldspeed, rval; + int mode = QLA_SET_DATA_RATE_LR; + struct qla_hw_data *ha = vha->hw; + + if (!IS_QLA27XX(vha->hw)) { + ql_log(ql_log_warn, vha, 0x70d8, + "Speed setting not supported \n"); + return -EINVAL; + } + + rval = kstrtol(buf, 10, &type); + speed = type; + if (type == 40 || type == 80 || type == 160 || + type == 320) { + ql_dbg(ql_dbg_user, vha, 0x70d9, + "Setting will be affected after a loss of sync\n"); + type = type/10; + mode = QLA_SET_DATA_RATE_NOLR; + } + + oldspeed = ha->set_data_rate; + + switch (type) { + case 0: + ha->set_data_rate = PORT_SPEED_AUTO; + break; + case 4: + ha->set_data_rate = PORT_SPEED_4GB; + break; + case 8: + ha->set_data_rate = PORT_SPEED_8GB; + break; + case 16: + ha->set_data_rate = PORT_SPEED_16GB; + break; + case 32: + ha->set_data_rate = PORT_SPEED_32GB; + break; + default: + ql_log(ql_log_warn, vha, 0x1199, + "Unrecognized speed setting:%lx. Setting Autoneg\n", + speed); + ha->set_data_rate = PORT_SPEED_AUTO; + } + + if (qla2x00_chip_is_down(vha) || (oldspeed == ha->set_data_rate)) + return -EINVAL; + + ql_log(ql_log_info, vha, 0x70da, + "Setting speed to %lx Gbps \n", type); + + rval = qla2x00_set_data_rate(vha, mode); + if (rval != QLA_SUCCESS) + return -EIO; + + return strlen(buf); +} + +static ssize_t +qla2x00_port_speed_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct scsi_qla_host *vha = shost_priv(dev_to_shost(dev)); + struct qla_hw_data *ha = vha->hw; + ssize_t rval; + char *spd[7] = {"0", "0", "0", "4", "8", "16", "32"}; + + rval = qla2x00_get_data_rate(vha); + if (rval != QLA_SUCCESS) { + ql_log(ql_log_warn, vha, 0x70db, + "Unable to get port speed rval:%zd\n", rval); + return -EINVAL; + } + + ql_log(ql_log_info, vha, 0x70d6, + "port speed:%d\n", ha->link_data_rate); + + return scnprintf(buf, PAGE_SIZE, "%s\n", spd[ha->link_data_rate]); +} + /* ----- */ static ssize_t @@ -2059,7 +2148,21 @@ ql2xiniexchg_store(struct device *dev, struct device_attribute *attr, return strlen(buf); } -static DEVICE_ATTR(driver_version, S_IRUGO, qla2x00_drvr_version_show, NULL); +static ssize_t +qla2x00_dif_bundle_statistics_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + scsi_qla_host_t *vha = shost_priv(class_to_shost(dev)); + struct qla_hw_data *ha = vha->hw; + + return scnprintf(buf, PAGE_SIZE, + "cross=%llu read=%llu write=%llu kalloc=%llu dma_alloc=%llu unusable=%u\n", + ha->dif_bundle_crossed_pages, ha->dif_bundle_reads, + ha->dif_bundle_writes, ha->dif_bundle_kallocs, + ha->dif_bundle_dma_allocs, ha->pool.unusable.count); +} + +static DEVICE_ATTR(driver_version, S_IRUGO, qla2x00_driver_version_show, NULL); static DEVICE_ATTR(fw_version, S_IRUGO, qla2x00_fw_version_show, NULL); static DEVICE_ATTR(serial_num, S_IRUGO, qla2x00_serial_num_show, NULL); static DEVICE_ATTR(isp_name, S_IRUGO, qla2x00_isp_name_show, NULL); @@ -2112,6 +2215,10 @@ static DEVICE_ATTR(zio_threshold, 0644, static DEVICE_ATTR_RW(qlini_mode); static DEVICE_ATTR_RW(ql2xexchoffld); static DEVICE_ATTR_RW(ql2xiniexchg); +static DEVICE_ATTR(dif_bundle_statistics, 0444, + qla2x00_dif_bundle_statistics_show, NULL); +static DEVICE_ATTR(port_speed, 0644, qla2x00_port_speed_show, + qla2x00_port_speed_store); struct device_attribute *qla2x00_host_attrs[] = { @@ -2150,6 +2257,8 @@ struct device_attribute *qla2x00_host_attrs[] = { &dev_attr_min_link_speed, &dev_attr_max_speed_sup, &dev_attr_zio_threshold, + &dev_attr_dif_bundle_statistics, + &dev_attr_port_speed, NULL, /* reserve for qlini_mode */ NULL, /* reserve for ql2xiniexchg */ NULL, /* reserve for ql2xexchoffld */ diff --git a/drivers/scsi/qla2xxx/qla_def.h b/drivers/scsi/qla2xxx/qla_def.h index d1fc4958222a..3d46975a5e5c 100644 --- a/drivers/scsi/qla2xxx/qla_def.h +++ b/drivers/scsi/qla2xxx/qla_def.h @@ -314,6 +314,7 @@ struct srb_cmd { #define SRB_CRC_PROT_DMA_VALID BIT_4 /* DIF: prot DMA valid */ #define SRB_CRC_CTX_DSD_VALID BIT_5 /* DIF: dsd_list valid */ #define SRB_WAKEUP_ON_COMP BIT_6 +#define SRB_DIF_BUNDL_DMA_VALID BIT_7 /* DIF: DMA list valid */ /* To identify if a srb is of T10-CRC type. @sp => srb_t pointer */ #define IS_PROT_IO(sp) (sp->flags & SRB_CRC_CTX_DSD_VALID) @@ -1892,6 +1893,13 @@ struct crc_context { /* List of DMA context transfers */ struct list_head dsd_list; + /* List of DIF Bundling context DMA address */ + struct list_head ldif_dsd_list; + u8 no_ldif_dsd; + + struct list_head ldif_dma_hndl_list; + u32 dif_bundl_len; + u8 no_dif_bundl; /* This structure should not exceed 512 bytes */ }; @@ -2359,7 +2367,9 @@ typedef struct fc_port { #define NVME_PRLI_SP_INITIATOR BIT_5 #define NVME_PRLI_SP_TARGET BIT_4 #define NVME_PRLI_SP_DISCOVERY BIT_3 +#define NVME_PRLI_SP_FIRST_BURST BIT_0 uint8_t nvme_flag; + uint32_t nvme_first_burst_size; #define NVME_FLAG_REGISTERED 4 #define NVME_FLAG_DELETING 2 #define NVME_FLAG_RESETTING 1 @@ -3688,12 +3698,14 @@ struct qla_hw_data { #define PORT_SPEED_UNKNOWN 0xFFFF #define PORT_SPEED_1GB 0x00 #define PORT_SPEED_2GB 0x01 +#define PORT_SPEED_AUTO 0x02 #define PORT_SPEED_4GB 0x03 #define PORT_SPEED_8GB 0x04 #define PORT_SPEED_16GB 0x05 #define PORT_SPEED_32GB 0x06 #define PORT_SPEED_10GB 0x13 uint16_t link_data_rate; /* F/W operating speed */ + uint16_t set_data_rate; /* Set by user */ uint8_t current_topology; uint8_t prev_topology; @@ -3958,6 +3970,10 @@ struct qla_hw_data { uint16_t fw_subminor_version; uint16_t fw_attributes; uint16_t fw_attributes_h; +#define FW_ATTR_H_NVME_FBURST BIT_1 +#define FW_ATTR_H_NVME BIT_10 +#define FW_ATTR_H_NVME_UPDATED BIT_14 + uint16_t fw_attributes_ext[2]; uint32_t fw_memory_size; uint32_t fw_transfer_size; @@ -4184,12 +4200,32 @@ struct qla_hw_data { uint16_t min_link_speed; uint16_t max_speed_sup; + /* DMA pool for the DIF bundling buffers */ + struct dma_pool *dif_bundl_pool; + #define DIF_BUNDLING_DMA_POOL_SIZE 1024 + struct { + struct { + struct list_head head; + uint count; + } good; + struct { + struct list_head head; + uint count; + } unusable; + } pool; + + unsigned long long dif_bundle_crossed_pages; + unsigned long long dif_bundle_reads; + unsigned long long dif_bundle_writes; + unsigned long long dif_bundle_kallocs; + unsigned long long dif_bundle_dma_allocs; + atomic_t nvme_active_aen_cnt; uint16_t nvme_last_rptd_aen; /* Last recorded aen count */ atomic_t zio_threshold; uint16_t last_zio_threshold; -#define DEFAULT_ZIO_THRESHOLD 64 +#define DEFAULT_ZIO_THRESHOLD 5 }; #define FW_ABILITY_MAX_SPEED_MASK 0xFUL @@ -4198,6 +4234,10 @@ struct qla_hw_data { #define FW_ABILITY_MAX_SPEED(ha) \ (ha->fw_ability_mask & FW_ABILITY_MAX_SPEED_MASK) +#define QLA_GET_DATA_RATE 0 +#define QLA_SET_DATA_RATE_NOLR 1 +#define QLA_SET_DATA_RATE_LR 2 /* Set speed and initiate LR */ + /* * Qlogic scsi host structure */ @@ -4229,6 +4269,7 @@ typedef struct scsi_qla_host { uint32_t qpairs_req_created:1; uint32_t qpairs_rsp_created:1; uint32_t nvme_enabled:1; + uint32_t nvme_first_burst:1; } flags; atomic_t loop_state; diff --git a/drivers/scsi/qla2xxx/qla_dfs.c b/drivers/scsi/qla2xxx/qla_dfs.c index 0b190082aa8d..ead17288e2a7 100644 --- a/drivers/scsi/qla2xxx/qla_dfs.c +++ b/drivers/scsi/qla2xxx/qla_dfs.c @@ -446,11 +446,6 @@ qla2x00_dfs_setup(scsi_qla_host_t *vha) atomic_set(&qla2x00_dfs_root_count, 0); qla2x00_dfs_root = debugfs_create_dir(QLA2XXX_DRIVER_NAME, NULL); - if (!qla2x00_dfs_root) { - ql_log(ql_log_warn, vha, 0x00f7, - "Unable to create debugfs root directory.\n"); - goto out; - } create_dir: if (ha->dfs_dir) @@ -458,64 +453,28 @@ create_dir: mutex_init(&ha->fce_mutex); ha->dfs_dir = debugfs_create_dir(vha->host_str, qla2x00_dfs_root); - if (!ha->dfs_dir) { - ql_log(ql_log_warn, vha, 0x00f8, - "Unable to create debugfs ha directory.\n"); - goto out; - } atomic_inc(&qla2x00_dfs_root_count); create_nodes: ha->dfs_fw_resource_cnt = debugfs_create_file("fw_resource_count", S_IRUSR, ha->dfs_dir, vha, &dfs_fw_resource_cnt_ops); - if (!ha->dfs_fw_resource_cnt) { - ql_log(ql_log_warn, vha, 0x00fd, - "Unable to create debugFS fw_resource_count node.\n"); - goto out; - } ha->dfs_tgt_counters = debugfs_create_file("tgt_counters", S_IRUSR, ha->dfs_dir, vha, &dfs_tgt_counters_ops); - if (!ha->dfs_tgt_counters) { - ql_log(ql_log_warn, vha, 0xd301, - "Unable to create debugFS tgt_counters node.\n"); - goto out; - } ha->tgt.dfs_tgt_port_database = debugfs_create_file("tgt_port_database", S_IRUSR, ha->dfs_dir, vha, &dfs_tgt_port_database_ops); - if (!ha->tgt.dfs_tgt_port_database) { - ql_log(ql_log_warn, vha, 0xd03f, - "Unable to create debugFS tgt_port_database node.\n"); - goto out; - } ha->dfs_fce = debugfs_create_file("fce", S_IRUSR, ha->dfs_dir, vha, &dfs_fce_ops); - if (!ha->dfs_fce) { - ql_log(ql_log_warn, vha, 0x00f9, - "Unable to create debugfs fce node.\n"); - goto out; - } ha->tgt.dfs_tgt_sess = debugfs_create_file("tgt_sess", S_IRUSR, ha->dfs_dir, vha, &dfs_tgt_sess_ops); - if (!ha->tgt.dfs_tgt_sess) { - ql_log(ql_log_warn, vha, 0xd040, - "Unable to create debugFS tgt_sess node.\n"); - goto out; - } - if (IS_QLA27XX(ha) || IS_QLA83XX(ha)) { + if (IS_QLA27XX(ha) || IS_QLA83XX(ha)) ha->tgt.dfs_naqp = debugfs_create_file("naqp", 0400, ha->dfs_dir, vha, &dfs_naqp_ops); - if (!ha->tgt.dfs_naqp) { - ql_log(ql_log_warn, vha, 0xd011, - "Unable to create debugFS naqp node.\n"); - goto out; - } - } out: return 0; } diff --git a/drivers/scsi/qla2xxx/qla_gbl.h b/drivers/scsi/qla2xxx/qla_gbl.h index 3673fcdb033a..4eefe69ca807 100644 --- a/drivers/scsi/qla2xxx/qla_gbl.h +++ b/drivers/scsi/qla2xxx/qla_gbl.h @@ -160,6 +160,7 @@ extern int ql2xautodetectsfp; extern int ql2xenablemsix; extern int qla2xuseresexchforels; extern int ql2xexlogins; +extern int ql2xdifbundlinginternalbuffers; extern int qla2x00_loop_reset(scsi_qla_host_t *); extern void qla2x00_abort_all_cmds(scsi_qla_host_t *, int); @@ -269,8 +270,8 @@ extern void qla24xx_build_scsi_iocbs(srb_t *, struct cmd_type_7 *, uint16_t, struct req_que *); extern int qla2x00_start_scsi(srb_t *sp); extern int qla24xx_start_scsi(srb_t *sp); -int qla2x00_marker(struct scsi_qla_host *, struct req_que *, struct rsp_que *, - uint16_t, uint64_t, uint8_t); +int qla2x00_marker(struct scsi_qla_host *, struct qla_qpair *, + uint16_t, uint64_t, uint8_t); extern int qla2x00_start_sp(srb_t *); extern int qla24xx_dif_start_scsi(srb_t *); extern int qla2x00_start_bidir(srb_t *, struct scsi_qla_host *, uint32_t); @@ -285,7 +286,7 @@ extern int qla24xx_walk_and_build_sglist_no_difb(struct qla_hw_data *, srb_t *, extern int qla24xx_walk_and_build_sglist(struct qla_hw_data *, srb_t *, uint32_t *, uint16_t, struct qla_tc_param *); extern int qla24xx_walk_and_build_prot_sglist(struct qla_hw_data *, srb_t *, - uint32_t *, uint16_t, struct qla_tc_param *); + uint32_t *, uint16_t, struct qla_tgt_cmd *); extern int qla24xx_get_one_block_sg(uint32_t, struct qla2_sgx *, uint32_t *); extern int qla24xx_configure_prot_mode(srb_t *, uint16_t *); extern int qla24xx_build_scsi_crc_2_iocbs(srb_t *, @@ -898,5 +899,6 @@ void qlt_update_host_map(struct scsi_qla_host *, port_id_t); void qlt_remove_target_resources(struct qla_hw_data *); void qlt_clr_qp_table(struct scsi_qla_host *vha); void qlt_set_mode(struct scsi_qla_host *); +int qla2x00_set_data_rate(scsi_qla_host_t *vha, uint16_t mode); #endif /* _QLA_GBL_H */ diff --git a/drivers/scsi/qla2xxx/qla_gs.c b/drivers/scsi/qla2xxx/qla_gs.c index cbc3bc49d4d1..c6fdad12428e 100644 --- a/drivers/scsi/qla2xxx/qla_gs.c +++ b/drivers/scsi/qla2xxx/qla_gs.c @@ -657,15 +657,16 @@ static int qla_async_rftid(scsi_qla_host_t *vha, port_id_t *d_id) sp->u.iocb_cmd.timeout = qla2x00_async_iocb_timeout; sp->done = qla2x00_async_sns_sp_done; + ql_dbg(ql_dbg_disc, vha, 0xffff, + "Async-%s - hdl=%x portid %06x.\n", + sp->name, sp->handle, d_id->b24); + rval = qla2x00_start_sp(sp); if (rval != QLA_SUCCESS) { ql_dbg(ql_dbg_disc, vha, 0x2043, "RFT_ID issue IOCB failed (%d).\n", rval); goto done_free_sp; } - ql_dbg(ql_dbg_disc, vha, 0xffff, - "Async-%s - hdl=%x portid %06x.\n", - sp->name, sp->handle, d_id->b24); return rval; done_free_sp: sp->free(sp); @@ -752,6 +753,10 @@ static int qla_async_rffid(scsi_qla_host_t *vha, port_id_t *d_id, sp->u.iocb_cmd.timeout = qla2x00_async_iocb_timeout; sp->done = qla2x00_async_sns_sp_done; + ql_dbg(ql_dbg_disc, vha, 0xffff, + "Async-%s - hdl=%x portid %06x feature %x type %x.\n", + sp->name, sp->handle, d_id->b24, fc4feature, fc4type); + rval = qla2x00_start_sp(sp); if (rval != QLA_SUCCESS) { ql_dbg(ql_dbg_disc, vha, 0x2047, @@ -759,9 +764,6 @@ static int qla_async_rffid(scsi_qla_host_t *vha, port_id_t *d_id, goto done_free_sp; } - ql_dbg(ql_dbg_disc, vha, 0xffff, - "Async-%s - hdl=%x portid %06x feature %x type %x.\n", - sp->name, sp->handle, d_id->b24, fc4feature, fc4type); return rval; done_free_sp: @@ -844,15 +846,16 @@ static int qla_async_rnnid(scsi_qla_host_t *vha, port_id_t *d_id, sp->u.iocb_cmd.timeout = qla2x00_async_iocb_timeout; sp->done = qla2x00_async_sns_sp_done; + ql_dbg(ql_dbg_disc, vha, 0xffff, + "Async-%s - hdl=%x portid %06x\n", + sp->name, sp->handle, d_id->b24); + rval = qla2x00_start_sp(sp); if (rval != QLA_SUCCESS) { ql_dbg(ql_dbg_disc, vha, 0x204d, "RNN_ID issue IOCB failed (%d).\n", rval); goto done_free_sp; } - ql_dbg(ql_dbg_disc, vha, 0xffff, - "Async-%s - hdl=%x portid %06x\n", - sp->name, sp->handle, d_id->b24); return rval; @@ -957,15 +960,16 @@ static int qla_async_rsnn_nn(scsi_qla_host_t *vha) sp->u.iocb_cmd.timeout = qla2x00_async_iocb_timeout; sp->done = qla2x00_async_sns_sp_done; + ql_dbg(ql_dbg_disc, vha, 0xffff, + "Async-%s - hdl=%x.\n", + sp->name, sp->handle); + rval = qla2x00_start_sp(sp); if (rval != QLA_SUCCESS) { ql_dbg(ql_dbg_disc, vha, 0x2043, "RFT_ID issue IOCB failed (%d).\n", rval); goto done_free_sp; } - ql_dbg(ql_dbg_disc, vha, 0xffff, - "Async-%s - hdl=%x.\n", - sp->name, sp->handle); return rval; @@ -3578,14 +3582,14 @@ int qla24xx_async_gffid(scsi_qla_host_t *vha, fc_port_t *fcport) sp->done = qla24xx_async_gffid_sp_done; - rval = qla2x00_start_sp(sp); - if (rval != QLA_SUCCESS) - goto done_free_sp; - ql_dbg(ql_dbg_disc, vha, 0x2132, "Async-%s hdl=%x %8phC.\n", sp->name, sp->handle, fcport->port_name); + rval = qla2x00_start_sp(sp); + if (rval != QLA_SUCCESS) + goto done_free_sp; + return rval; done_free_sp: sp->free(sp); @@ -4067,6 +4071,10 @@ static int qla24xx_async_gnnft(scsi_qla_host_t *vha, struct srb *sp, sp->done = qla2x00_async_gpnft_gnnft_sp_done; + ql_dbg(ql_dbg_disc, vha, 0xffff, + "Async-%s hdl=%x FC4Type %x.\n", sp->name, + sp->handle, ct_req->req.gpn_ft.port_type); + rval = qla2x00_start_sp(sp); if (rval != QLA_SUCCESS) { spin_lock_irqsave(&vha->work_lock, flags); @@ -4075,9 +4083,6 @@ static int qla24xx_async_gnnft(scsi_qla_host_t *vha, struct srb *sp, goto done_free_sp; } - ql_dbg(ql_dbg_disc, vha, 0xffff, - "Async-%s hdl=%x FC4Type %x.\n", sp->name, - sp->handle, ct_req->req.gpn_ft.port_type); return rval; done_free_sp: @@ -4158,7 +4163,8 @@ int qla24xx_async_gpnft(scsi_qla_host_t *vha, u8 fc4_type, srb_t *sp) spin_lock_irqsave(&vha->work_lock, flags); vha->scan.scan_flags &= ~SF_SCANNING; spin_unlock_irqrestore(&vha->work_lock, flags); - goto done_free_sp; + qla2x00_rel_sp(sp); + return rval; } sp->u.iocb_cmd.u.ctarg.req_size = GPN_FT_REQ_SIZE; @@ -4177,7 +4183,13 @@ int qla24xx_async_gpnft(scsi_qla_host_t *vha, u8 fc4_type, srb_t *sp) spin_lock_irqsave(&vha->work_lock, flags); vha->scan.scan_flags &= ~SF_SCANNING; spin_unlock_irqrestore(&vha->work_lock, flags); - goto done_free_sp; + dma_free_coherent(&vha->hw->pdev->dev, + sp->u.iocb_cmd.u.ctarg.req_allocated_size, + sp->u.iocb_cmd.u.ctarg.req, + sp->u.iocb_cmd.u.ctarg.req_dma); + sp->u.iocb_cmd.u.ctarg.req = NULL; + qla2x00_rel_sp(sp); + return rval; } sp->u.iocb_cmd.u.ctarg.rsp_size = rspsz; @@ -4214,6 +4226,10 @@ int qla24xx_async_gpnft(scsi_qla_host_t *vha, u8 fc4_type, srb_t *sp) sp->done = qla2x00_async_gpnft_gnnft_sp_done; + ql_dbg(ql_dbg_disc, vha, 0xffff, + "Async-%s hdl=%x FC4Type %x.\n", sp->name, + sp->handle, ct_req->req.gpn_ft.port_type); + rval = qla2x00_start_sp(sp); if (rval != QLA_SUCCESS) { spin_lock_irqsave(&vha->work_lock, flags); @@ -4222,9 +4238,6 @@ int qla24xx_async_gpnft(scsi_qla_host_t *vha, u8 fc4_type, srb_t *sp) goto done_free_sp; } - ql_dbg(ql_dbg_disc, vha, 0xffff, - "Async-%s hdl=%x FC4Type %x.\n", sp->name, - sp->handle, ct_req->req.gpn_ft.port_type); return rval; done_free_sp: @@ -4345,13 +4358,14 @@ int qla24xx_async_gnnid(scsi_qla_host_t *vha, fc_port_t *fcport) sp->done = qla2x00_async_gnnid_sp_done; - rval = qla2x00_start_sp(sp); - if (rval != QLA_SUCCESS) - goto done_free_sp; ql_dbg(ql_dbg_disc, vha, 0xffff, "Async-%s - %8phC hdl=%x loopid=%x portid %06x.\n", sp->name, fcport->port_name, sp->handle, fcport->loop_id, fcport->d_id.b24); + + rval = qla2x00_start_sp(sp); + if (rval != QLA_SUCCESS) + goto done_free_sp; return rval; done_free_sp: @@ -4475,14 +4489,15 @@ int qla24xx_async_gfpnid(scsi_qla_host_t *vha, fc_port_t *fcport) sp->done = qla2x00_async_gfpnid_sp_done; - rval = qla2x00_start_sp(sp); - if (rval != QLA_SUCCESS) - goto done_free_sp; - ql_dbg(ql_dbg_disc, vha, 0xffff, "Async-%s - %8phC hdl=%x loopid=%x portid %06x.\n", sp->name, fcport->port_name, sp->handle, fcport->loop_id, fcport->d_id.b24); + + rval = qla2x00_start_sp(sp); + if (rval != QLA_SUCCESS) + goto done_free_sp; + return rval; done_free_sp: diff --git a/drivers/scsi/qla2xxx/qla_init.c b/drivers/scsi/qla2xxx/qla_init.c index 8d1acc802a67..420045155ba0 100644 --- a/drivers/scsi/qla2xxx/qla_init.c +++ b/drivers/scsi/qla2xxx/qla_init.c @@ -366,14 +366,16 @@ qla2x00_async_prlo(struct scsi_qla_host *vha, fc_port_t *fcport) qla2x00_init_timer(sp, qla2x00_get_async_timeout(vha) + 2); sp->done = qla2x00_async_prlo_sp_done; - rval = qla2x00_start_sp(sp); - if (rval != QLA_SUCCESS) - goto done_free_sp; ql_dbg(ql_dbg_disc, vha, 0x2070, "Async-prlo - hdl=%x loop-id=%x portid=%02x%02x%02x.\n", sp->handle, fcport->loop_id, fcport->d_id.b.domain, fcport->d_id.b.area, fcport->d_id.b.al_pa); + + rval = qla2x00_start_sp(sp); + if (rval != QLA_SUCCESS) + goto done_free_sp; + return rval; done_free_sp: @@ -471,9 +473,11 @@ qla2x00_async_adisc(struct scsi_qla_host *vha, fc_port_t *fcport, { srb_t *sp; struct srb_iocb *lio; - int rval; + int rval = QLA_FUNCTION_FAILED; + + if (!vha->flags.online || (fcport->flags & FCF_ASYNC_SENT)) + return rval; - rval = QLA_FUNCTION_FAILED; fcport->flags |= FCF_ASYNC_SENT; sp = qla2x00_get_sp(vha, fcport, GFP_KERNEL); if (!sp) @@ -644,11 +648,14 @@ static void qla24xx_handle_gnl_done_event(scsi_qla_host_t *vha, break; case DSC_LS_PORT_UNAVAIL: default: - if (fcport->loop_id != FC_NO_LOOP_ID) - qla2x00_clear_loop_id(fcport); - - fcport->loop_id = loop_id; - fcport->fw_login_state = DSC_LS_PORT_UNAVAIL; + if (fcport->loop_id == FC_NO_LOOP_ID) { + qla2x00_find_new_loop_id(vha, fcport); + fcport->fw_login_state = + DSC_LS_PORT_UNAVAIL; + } + ql_dbg(ql_dbg_disc, vha, 0x20e5, + "%s %d %8phC\n", __func__, __LINE__, + fcport->port_name); qla24xx_fcport_handle_login(vha, fcport); break; } @@ -931,14 +938,14 @@ int qla24xx_async_gnl(struct scsi_qla_host *vha, fc_port_t *fcport) sp->done = qla24xx_async_gnl_sp_done; - rval = qla2x00_start_sp(sp); - if (rval != QLA_SUCCESS) - goto done_free_sp; - ql_dbg(ql_dbg_disc, vha, 0x20da, "Async-%s - OUT WWPN %8phC hndl %x\n", sp->name, fcport->port_name, sp->handle); + rval = qla2x00_start_sp(sp); + if (rval != QLA_SUCCESS) + goto done_free_sp; + return rval; done_free_sp: @@ -1072,6 +1079,11 @@ qla24xx_async_prli(struct scsi_qla_host *vha, fc_port_t *fcport) if (fcport->fc4f_nvme) lio->u.logio.flags |= SRB_LOGIN_NVME_PRLI; + ql_dbg(ql_dbg_disc, vha, 0x211b, + "Async-prli - %8phC hdl=%x, loopid=%x portid=%06x retries=%d %s.\n", + fcport->port_name, sp->handle, fcport->loop_id, fcport->d_id.b24, + fcport->login_retry, fcport->fc4f_nvme ? "nvme" : "fc"); + rval = qla2x00_start_sp(sp); if (rval != QLA_SUCCESS) { fcport->flags |= FCF_LOGIN_NEEDED; @@ -1079,11 +1091,6 @@ qla24xx_async_prli(struct scsi_qla_host *vha, fc_port_t *fcport) goto done_free_sp; } - ql_dbg(ql_dbg_disc, vha, 0x211b, - "Async-prli - %8phC hdl=%x, loopid=%x portid=%06x retries=%d %s.\n", - fcport->port_name, sp->handle, fcport->loop_id, fcport->d_id.b24, - fcport->login_retry, fcport->fc4f_nvme ? "nvme" : "fc"); - return rval; done_free_sp: @@ -1471,29 +1478,6 @@ int qla24xx_fcport_handle_login(struct scsi_qla_host *vha, fc_port_t *fcport) return 0; } -static -void qla24xx_handle_rscn_event(fc_port_t *fcport, struct event_arg *ea) -{ - fcport->rscn_gen++; - - ql_dbg(ql_dbg_disc, fcport->vha, 0x210c, - "%s %8phC DS %d LS %d\n", - __func__, fcport->port_name, fcport->disc_state, - fcport->fw_login_state); - - if (fcport->flags & FCF_ASYNC_SENT) - return; - - switch (fcport->disc_state) { - case DSC_DELETED: - case DSC_LOGIN_COMPLETE: - qla24xx_post_gpnid_work(fcport->vha, &ea->id); - break; - default: - break; - } -} - int qla24xx_post_newsess_work(struct scsi_qla_host *vha, port_id_t *id, u8 *port_name, u8 *node_name, void *pla, u8 fc4_type) { @@ -1560,8 +1544,6 @@ static void qla_handle_els_plogi_done(scsi_qla_host_t *vha, void qla2x00_fcport_event_handler(scsi_qla_host_t *vha, struct event_arg *ea) { - fc_port_t *f, *tf; - uint32_t id = 0, mask, rid; fc_port_t *fcport; switch (ea->event) { @@ -1574,10 +1556,6 @@ void qla2x00_fcport_event_handler(scsi_qla_host_t *vha, struct event_arg *ea) case FCME_RSCN: if (test_bit(UNLOADING, &vha->dpc_flags)) return; - switch (ea->id.b.rsvd_1) { - case RSCN_PORT_ADDR: -#define BIGSCAN 1 -#if defined BIGSCAN & BIGSCAN > 0 { unsigned long flags; fcport = qla2x00_find_fcport_by_nportid @@ -1596,59 +1574,6 @@ void qla2x00_fcport_event_handler(scsi_qla_host_t *vha, struct event_arg *ea) } spin_unlock_irqrestore(&vha->work_lock, flags); } -#else - { - int rc; - fcport = qla2x00_find_fcport_by_nportid(vha, &ea->id, 1); - if (!fcport) { - /* cable moved */ - rc = qla24xx_post_gpnid_work(vha, &ea->id); - if (rc) { - ql_log(ql_log_warn, vha, 0xd044, - "RSCN GPNID work failed %06x\n", - ea->id.b24); - } - } else { - ea->fcport = fcport; - fcport->scan_needed = 1; - qla24xx_handle_rscn_event(fcport, ea); - } - } -#endif - break; - case RSCN_AREA_ADDR: - case RSCN_DOM_ADDR: - if (ea->id.b.rsvd_1 == RSCN_AREA_ADDR) { - mask = 0xffff00; - ql_dbg(ql_dbg_async, vha, 0x5044, - "RSCN: Area 0x%06x was affected\n", - ea->id.b24); - } else { - mask = 0xff0000; - ql_dbg(ql_dbg_async, vha, 0x507a, - "RSCN: Domain 0x%06x was affected\n", - ea->id.b24); - } - - rid = ea->id.b24 & mask; - list_for_each_entry_safe(f, tf, &vha->vp_fcports, - list) { - id = f->d_id.b24 & mask; - if (rid == id) { - ea->fcport = f; - qla24xx_handle_rscn_event(f, ea); - } - } - break; - case RSCN_FAB_ADDR: - default: - ql_log(ql_log_warn, vha, 0xd045, - "RSCN: Fabric was affected. Addr format %d\n", - ea->id.b.rsvd_1); - qla2x00_mark_all_devices_lost(vha, 1); - set_bit(LOOP_RESYNC_NEEDED, &vha->dpc_flags); - set_bit(LOCAL_LOOP_UPDATE, &vha->dpc_flags); - } break; case FCME_GNL_DONE: qla24xx_handle_gnl_done_event(vha, ea); @@ -1709,11 +1634,7 @@ void qla_rscn_replay(fc_port_t *fcport) ea.event = FCME_RSCN; ea.id = fcport->d_id; ea.id.b.rsvd_1 = RSCN_PORT_ADDR; -#if defined BIGSCAN & BIGSCAN > 0 qla2x00_fcport_event_handler(fcport->vha, &ea); -#else - qla24xx_post_gpnid_work(fcport->vha, &ea.id); -#endif } } @@ -1784,8 +1705,8 @@ qla2x00_async_tm_cmd(fc_port_t *fcport, uint32_t flags, uint32_t lun, lun = (uint16_t)tm_iocb->u.tmf.lun; /* Issue Marker IOCB */ - qla2x00_marker(vha, vha->hw->req_q_map[0], - vha->hw->rsp_q_map[0], fcport->loop_id, lun, + qla2x00_marker(vha, vha->hw->base_qpair, + fcport->loop_id, lun, flags == TCF_LUN_RESET ? MK_SYNC_ID_LUN : MK_SYNC_ID); } @@ -1829,7 +1750,7 @@ qla24xx_async_abort_cmd(srb_t *cmd_sp, bool wait) int rval = QLA_FUNCTION_FAILED; sp = qla2xxx_get_qpair_sp(cmd_sp->vha, cmd_sp->qpair, cmd_sp->fcport, - GFP_KERNEL); + GFP_ATOMIC); if (!sp) goto done; @@ -1912,6 +1833,12 @@ qla24xx_handle_prli_done_event(struct scsi_qla_host *vha, struct event_arg *ea) ea->fcport->chip_reset = vha->hw->base_qpair->chip_reset; ea->fcport->logout_on_delete = 1; + ea->fcport->nvme_prli_service_param = ea->iop[0]; + if (ea->iop[0] & NVME_PRLI_SP_FIRST_BURST) + ea->fcport->nvme_first_burst_size = + (ea->iop[1] & 0xffff) * 512; + else + ea->fcport->nvme_first_burst_size = 0; qla24xx_post_gpdb_work(vha, ea->fcport, 0); break; default: @@ -3955,8 +3882,17 @@ qla24xx_config_rings(struct scsi_qla_host *vha) WRT_REG_DWORD(®->isp24.rsp_q_in, 0); WRT_REG_DWORD(®->isp24.rsp_q_out, 0); } + qlt_24xx_config_rings(vha); + /* If the user has configured the speed, set it here */ + if (ha->set_data_rate) { + ql_dbg(ql_dbg_init, vha, 0x00fd, + "Speed set by user : %s Gbps \n", + qla2x00_get_link_speed_str(ha, ha->set_data_rate)); + icb->firmware_options_3 = (ha->set_data_rate << 13); + } + /* PCI posting */ RD_REG_DWORD(&ioreg->hccr); } @@ -4755,6 +4691,16 @@ qla2x00_alloc_fcport(scsi_qla_host_t *vha, gfp_t flags) if (!fcport) return NULL; + fcport->ct_desc.ct_sns = dma_alloc_coherent(&vha->hw->pdev->dev, + sizeof(struct ct_sns_pkt), &fcport->ct_desc.ct_sns_dma, + flags); + if (!fcport->ct_desc.ct_sns) { + ql_log(ql_log_warn, vha, 0xd049, + "Failed to allocate ct_sns request.\n"); + kfree(fcport); + return NULL; + } + /* Setup fcport template structure. */ fcport->vha = vha; fcport->port_type = FCT_UNKNOWN; @@ -4763,13 +4709,11 @@ qla2x00_alloc_fcport(scsi_qla_host_t *vha, gfp_t flags) fcport->supported_classes = FC_COS_UNSPECIFIED; fcport->fp_speed = PORT_SPEED_UNKNOWN; - fcport->ct_desc.ct_sns = dma_alloc_coherent(&vha->hw->pdev->dev, - sizeof(struct ct_sns_pkt), &fcport->ct_desc.ct_sns_dma, - flags); fcport->disc_state = DSC_DELETED; fcport->fw_login_state = DSC_LS_PORT_UNAVAIL; fcport->deleted = QLA_SESS_DELETED; fcport->login_retry = vha->hw->login_retry_count; + fcport->chip_reset = vha->hw->base_qpair->chip_reset; fcport->logout_on_delete = 1; if (!fcport->ct_desc.ct_sns) { @@ -4778,6 +4722,7 @@ qla2x00_alloc_fcport(scsi_qla_host_t *vha, gfp_t flags) kfree(fcport); fcport = NULL; } + INIT_WORK(&fcport->del_work, qla24xx_delete_sess_fn); INIT_WORK(&fcport->reg_work, qla_register_fcport_fn); INIT_LIST_HEAD(&fcport->gnl_entry); @@ -5046,11 +4991,6 @@ qla2x00_configure_local_loop(scsi_qla_host_t *vha) if ((domain & 0xf0) == 0xf0) continue; - /* Bypass if not same domain and area of adapter. */ - if (area && domain && - (area != vha->d_id.b.area || domain != vha->d_id.b.domain)) - continue; - /* Bypass invalid local loop ID. */ if (loop_id > LAST_LOCAL_LOOP_ID) continue; @@ -6101,11 +6041,6 @@ qla2x00_loop_resync(scsi_qla_host_t *vha) { int rval = QLA_SUCCESS; uint32_t wait_time; - struct req_que *req; - struct rsp_que *rsp; - - req = vha->req; - rsp = req->rsp; clear_bit(ISP_ABORT_RETRY, &vha->dpc_flags); if (vha->flags.online) { @@ -6118,8 +6053,8 @@ qla2x00_loop_resync(scsi_qla_host_t *vha) * Issue a marker after FW becomes * ready. */ - qla2x00_marker(vha, req, rsp, 0, 0, - MK_SYNC_ALL); + qla2x00_marker(vha, vha->hw->base_qpair, + 0, 0, MK_SYNC_ALL); vha->marker_needed = 0; } @@ -6857,8 +6792,6 @@ qla2x00_restart_isp(scsi_qla_host_t *vha) { int status = 0; struct qla_hw_data *ha = vha->hw; - struct req_que *req = ha->req_q_map[0]; - struct rsp_que *rsp = ha->rsp_q_map[0]; /* If firmware needs to be loaded */ if (qla2x00_isp_firmware(vha)) { @@ -6878,7 +6811,7 @@ qla2x00_restart_isp(scsi_qla_host_t *vha) status = qla2x00_fw_ready(vha); if (!status) { /* Issue a marker after FW becomes ready. */ - qla2x00_marker(vha, req, rsp, 0, 0, MK_SYNC_ALL); + qla2x00_marker(vha, ha->base_qpair, 0, 0, MK_SYNC_ALL); set_bit(LOOP_RESYNC_NEEDED, &vha->dpc_flags); } @@ -7933,22 +7866,15 @@ qla24xx_configure_vhba(scsi_qla_host_t *vha) uint16_t mb[MAILBOX_REGISTER_COUNT]; struct qla_hw_data *ha = vha->hw; struct scsi_qla_host *base_vha = pci_get_drvdata(ha->pdev); - struct req_que *req; - struct rsp_que *rsp; if (!vha->vp_idx) return -EINVAL; rval = qla2x00_fw_ready(base_vha); - if (vha->qpair) - req = vha->qpair->req; - else - req = ha->req_q_map[0]; - rsp = req->rsp; if (rval == QLA_SUCCESS) { clear_bit(RESET_MARKER_NEEDED, &vha->dpc_flags); - qla2x00_marker(vha, req, rsp, 0, 0, MK_SYNC_ALL); + qla2x00_marker(vha, ha->base_qpair, 0, 0, MK_SYNC_ALL); } vha->flags.management_server_logged_in = 0; @@ -8340,8 +8266,6 @@ qla82xx_restart_isp(scsi_qla_host_t *vha) { int status, rval; struct qla_hw_data *ha = vha->hw; - struct req_que *req = ha->req_q_map[0]; - struct rsp_que *rsp = ha->rsp_q_map[0]; struct scsi_qla_host *vp; unsigned long flags; @@ -8353,7 +8277,7 @@ qla82xx_restart_isp(scsi_qla_host_t *vha) status = qla2x00_fw_ready(vha); if (!status) { /* Issue a marker after FW becomes ready. */ - qla2x00_marker(vha, req, rsp, 0, 0, MK_SYNC_ALL); + qla2x00_marker(vha, ha->base_qpair, 0, 0, MK_SYNC_ALL); vha->flags.online = 1; set_bit(LOOP_RESYNC_NEEDED, &vha->dpc_flags); } diff --git a/drivers/scsi/qla2xxx/qla_iocb.c b/drivers/scsi/qla2xxx/qla_iocb.c index 032635321ad6..63f8e3c19841 100644 --- a/drivers/scsi/qla2xxx/qla_iocb.c +++ b/drivers/scsi/qla2xxx/qla_iocb.c @@ -336,7 +336,7 @@ qla2x00_start_scsi(srb_t *sp) /* Send marker if required */ if (vha->marker_needed != 0) { - if (qla2x00_marker(vha, req, rsp, 0, 0, MK_SYNC_ALL) != + if (qla2x00_marker(vha, ha->base_qpair, 0, 0, MK_SYNC_ALL) != QLA_SUCCESS) { return (QLA_FUNCTION_FAILED); } @@ -490,8 +490,7 @@ qla2x00_start_iocbs(struct scsi_qla_host *vha, struct req_que *req) /** * qla2x00_marker() - Send a marker IOCB to the firmware. * @vha: HA context - * @req: request queue - * @rsp: response queue + * @qpair: queue pair pointer * @loop_id: loop ID * @lun: LUN * @type: marker modifier @@ -501,18 +500,16 @@ qla2x00_start_iocbs(struct scsi_qla_host *vha, struct req_que *req) * Returns non-zero if a failure occurred, else zero. */ static int -__qla2x00_marker(struct scsi_qla_host *vha, struct req_que *req, - struct rsp_que *rsp, uint16_t loop_id, - uint64_t lun, uint8_t type) +__qla2x00_marker(struct scsi_qla_host *vha, struct qla_qpair *qpair, + uint16_t loop_id, uint64_t lun, uint8_t type) { mrk_entry_t *mrk; struct mrk_entry_24xx *mrk24 = NULL; - + struct req_que *req = qpair->req; struct qla_hw_data *ha = vha->hw; scsi_qla_host_t *base_vha = pci_get_drvdata(ha->pdev); - req = ha->req_q_map[0]; - mrk = (mrk_entry_t *)qla2x00_alloc_iocbs(vha, NULL); + mrk = (mrk_entry_t *)__qla2x00_alloc_iocbs(qpair, NULL); if (mrk == NULL) { ql_log(ql_log_warn, base_vha, 0x3026, "Failed to allocate Marker IOCB.\n"); @@ -543,16 +540,15 @@ __qla2x00_marker(struct scsi_qla_host *vha, struct req_que *req, } int -qla2x00_marker(struct scsi_qla_host *vha, struct req_que *req, - struct rsp_que *rsp, uint16_t loop_id, uint64_t lun, - uint8_t type) +qla2x00_marker(struct scsi_qla_host *vha, struct qla_qpair *qpair, + uint16_t loop_id, uint64_t lun, uint8_t type) { int ret; unsigned long flags = 0; - spin_lock_irqsave(&vha->hw->hardware_lock, flags); - ret = __qla2x00_marker(vha, req, rsp, loop_id, lun, type); - spin_unlock_irqrestore(&vha->hw->hardware_lock, flags); + spin_lock_irqsave(qpair->qp_lock_ptr, flags); + ret = __qla2x00_marker(vha, qpair, loop_id, lun, type); + spin_unlock_irqrestore(qpair->qp_lock_ptr, flags); return (ret); } @@ -567,11 +563,11 @@ qla2x00_marker(struct scsi_qla_host *vha, struct req_que *req, int qla2x00_issue_marker(scsi_qla_host_t *vha, int ha_locked) { if (ha_locked) { - if (__qla2x00_marker(vha, vha->req, vha->req->rsp, 0, 0, + if (__qla2x00_marker(vha, vha->hw->base_qpair, 0, 0, MK_SYNC_ALL) != QLA_SUCCESS) return QLA_FUNCTION_FAILED; } else { - if (qla2x00_marker(vha, vha->req, vha->req->rsp, 0, 0, + if (qla2x00_marker(vha, vha->hw->base_qpair, 0, 0, MK_SYNC_ALL) != QLA_SUCCESS) return QLA_FUNCTION_FAILED; } @@ -1098,88 +1094,300 @@ qla24xx_walk_and_build_sglist(struct qla_hw_data *ha, srb_t *sp, uint32_t *dsd, int qla24xx_walk_and_build_prot_sglist(struct qla_hw_data *ha, srb_t *sp, - uint32_t *dsd, uint16_t tot_dsds, struct qla_tc_param *tc) + uint32_t *cur_dsd, uint16_t tot_dsds, struct qla_tgt_cmd *tc) { - void *next_dsd; - uint8_t avail_dsds = 0; - uint32_t dsd_list_len; - struct dsd_dma *dsd_ptr; + struct dsd_dma *dsd_ptr = NULL, *dif_dsd, *nxt_dsd; struct scatterlist *sg, *sgl; - int i; - struct scsi_cmnd *cmd; - uint32_t *cur_dsd = dsd; - uint16_t used_dsds = tot_dsds; + struct crc_context *difctx = NULL; struct scsi_qla_host *vha; + uint dsd_list_len; + uint avail_dsds = 0; + uint used_dsds = tot_dsds; + bool dif_local_dma_alloc = false; + bool direction_to_device = false; + int i; if (sp) { - cmd = GET_CMD_SP(sp); + struct scsi_cmnd *cmd = GET_CMD_SP(sp); sgl = scsi_prot_sglist(cmd); vha = sp->vha; + difctx = sp->u.scmd.ctx; + direction_to_device = cmd->sc_data_direction == DMA_TO_DEVICE; + ql_dbg(ql_dbg_tgt + ql_dbg_verbose, vha, 0xe021, + "%s: scsi_cmnd: %p, crc_ctx: %p, sp: %p\n", + __func__, cmd, difctx, sp); } else if (tc) { vha = tc->vha; sgl = tc->prot_sg; + difctx = tc->ctx; + direction_to_device = tc->dma_data_direction == DMA_TO_DEVICE; } else { BUG(); return 1; } - ql_dbg(ql_dbg_tgt, vha, 0xe021, - "%s: enter\n", __func__); - - for_each_sg(sgl, sg, tot_dsds, i) { - dma_addr_t sle_dma; - - /* Allocate additional continuation packets? */ - if (avail_dsds == 0) { - avail_dsds = (used_dsds > QLA_DSDS_PER_IOCB) ? - QLA_DSDS_PER_IOCB : used_dsds; - dsd_list_len = (avail_dsds + 1) * 12; - used_dsds -= avail_dsds; - - /* allocate tracking DS */ - dsd_ptr = kzalloc(sizeof(struct dsd_dma), GFP_ATOMIC); - if (!dsd_ptr) - return 1; - - /* allocate new list */ - dsd_ptr->dsd_addr = next_dsd = - dma_pool_alloc(ha->dl_dma_pool, GFP_ATOMIC, - &dsd_ptr->dsd_list_dma); - - if (!next_dsd) { - /* - * Need to cleanup only this dsd_ptr, rest - * will be done by sp_free_dma() - */ - kfree(dsd_ptr); - return 1; + ql_dbg(ql_dbg_tgt + ql_dbg_verbose, vha, 0xe021, + "%s: enter (write=%u)\n", __func__, direction_to_device); + + /* if initiator doing write or target doing read */ + if (direction_to_device) { + for_each_sg(sgl, sg, tot_dsds, i) { + dma_addr_t sle_phys = sg_phys(sg); + + /* If SGE addr + len flips bits in upper 32-bits */ + if (MSD(sle_phys + sg->length) ^ MSD(sle_phys)) { + ql_dbg(ql_dbg_tgt + ql_dbg_verbose, vha, 0xe022, + "%s: page boundary crossing (phys=%llx len=%x)\n", + __func__, sle_phys, sg->length); + + if (difctx) { + ha->dif_bundle_crossed_pages++; + dif_local_dma_alloc = true; + } else { + ql_dbg(ql_dbg_tgt + ql_dbg_verbose, + vha, 0xe022, + "%s: difctx pointer is NULL\n", + __func__); + } + break; } + } + ha->dif_bundle_writes++; + } else { + ha->dif_bundle_reads++; + } + + if (ql2xdifbundlinginternalbuffers) + dif_local_dma_alloc = direction_to_device; + + if (dif_local_dma_alloc) { + u32 track_difbundl_buf = 0; + u32 ldma_sg_len = 0; + u8 ldma_needed = 1; + + difctx->no_dif_bundl = 0; + difctx->dif_bundl_len = 0; + + /* Track DSD buffers */ + INIT_LIST_HEAD(&difctx->ldif_dsd_list); + /* Track local DMA buffers */ + INIT_LIST_HEAD(&difctx->ldif_dma_hndl_list); + + for_each_sg(sgl, sg, tot_dsds, i) { + u32 sglen = sg_dma_len(sg); + + ql_dbg(ql_dbg_tgt + ql_dbg_verbose, vha, 0xe023, + "%s: sg[%x] (phys=%llx sglen=%x) ldma_sg_len: %x dif_bundl_len: %x ldma_needed: %x\n", + __func__, i, sg_phys(sg), sglen, ldma_sg_len, + difctx->dif_bundl_len, ldma_needed); + + while (sglen) { + u32 xfrlen = 0; + + if (ldma_needed) { + /* + * Allocate list item to store + * the DMA buffers + */ + dsd_ptr = kzalloc(sizeof(*dsd_ptr), + GFP_ATOMIC); + if (!dsd_ptr) { + ql_dbg(ql_dbg_tgt, vha, 0xe024, + "%s: failed alloc dsd_ptr\n", + __func__); + return 1; + } + ha->dif_bundle_kallocs++; + + /* allocate dma buffer */ + dsd_ptr->dsd_addr = dma_pool_alloc + (ha->dif_bundl_pool, GFP_ATOMIC, + &dsd_ptr->dsd_list_dma); + if (!dsd_ptr->dsd_addr) { + ql_dbg(ql_dbg_tgt, vha, 0xe024, + "%s: failed alloc ->dsd_ptr\n", + __func__); + /* + * need to cleanup only this + * dsd_ptr rest will be done + * by sp_free_dma() + */ + kfree(dsd_ptr); + ha->dif_bundle_kallocs--; + return 1; + } + ha->dif_bundle_dma_allocs++; + ldma_needed = 0; + difctx->no_dif_bundl++; + list_add_tail(&dsd_ptr->list, + &difctx->ldif_dma_hndl_list); + } + + /* xfrlen is min of dma pool size and sglen */ + xfrlen = (sglen > + (DIF_BUNDLING_DMA_POOL_SIZE - ldma_sg_len)) ? + DIF_BUNDLING_DMA_POOL_SIZE - ldma_sg_len : + sglen; + + /* replace with local allocated dma buffer */ + sg_pcopy_to_buffer(sgl, sg_nents(sgl), + dsd_ptr->dsd_addr + ldma_sg_len, xfrlen, + difctx->dif_bundl_len); + difctx->dif_bundl_len += xfrlen; + sglen -= xfrlen; + ldma_sg_len += xfrlen; + if (ldma_sg_len == DIF_BUNDLING_DMA_POOL_SIZE || + sg_is_last(sg)) { + ldma_needed = 1; + ldma_sg_len = 0; + } + } + } - if (sp) { - list_add_tail(&dsd_ptr->list, - &((struct crc_context *) - sp->u.scmd.ctx)->dsd_list); + track_difbundl_buf = used_dsds = difctx->no_dif_bundl; + ql_dbg(ql_dbg_tgt + ql_dbg_verbose, vha, 0xe025, + "dif_bundl_len=%x, no_dif_bundl=%x track_difbundl_buf: %x\n", + difctx->dif_bundl_len, difctx->no_dif_bundl, + track_difbundl_buf); - sp->flags |= SRB_CRC_CTX_DSD_VALID; - } else { - list_add_tail(&dsd_ptr->list, - &(tc->ctx->dsd_list)); - *tc->ctx_dsd_alloced = 1; + if (sp) + sp->flags |= SRB_DIF_BUNDL_DMA_VALID; + else + tc->prot_flags = DIF_BUNDL_DMA_VALID; + + list_for_each_entry_safe(dif_dsd, nxt_dsd, + &difctx->ldif_dma_hndl_list, list) { + u32 sglen = (difctx->dif_bundl_len > + DIF_BUNDLING_DMA_POOL_SIZE) ? + DIF_BUNDLING_DMA_POOL_SIZE : difctx->dif_bundl_len; + + BUG_ON(track_difbundl_buf == 0); + + /* Allocate additional continuation packets? */ + if (avail_dsds == 0) { + ql_dbg(ql_dbg_tgt + ql_dbg_verbose, vha, + 0xe024, + "%s: adding continuation iocb's\n", + __func__); + avail_dsds = (used_dsds > QLA_DSDS_PER_IOCB) ? + QLA_DSDS_PER_IOCB : used_dsds; + dsd_list_len = (avail_dsds + 1) * 12; + used_dsds -= avail_dsds; + + /* allocate tracking DS */ + dsd_ptr = kzalloc(sizeof(*dsd_ptr), GFP_ATOMIC); + if (!dsd_ptr) { + ql_dbg(ql_dbg_tgt, vha, 0xe026, + "%s: failed alloc dsd_ptr\n", + __func__); + return 1; + } + ha->dif_bundle_kallocs++; + + difctx->no_ldif_dsd++; + /* allocate new list */ + dsd_ptr->dsd_addr = + dma_pool_alloc(ha->dl_dma_pool, GFP_ATOMIC, + &dsd_ptr->dsd_list_dma); + if (!dsd_ptr->dsd_addr) { + ql_dbg(ql_dbg_tgt, vha, 0xe026, + "%s: failed alloc ->dsd_addr\n", + __func__); + /* + * need to cleanup only this dsd_ptr + * rest will be done by sp_free_dma() + */ + kfree(dsd_ptr); + ha->dif_bundle_kallocs--; + return 1; + } + ha->dif_bundle_dma_allocs++; + + if (sp) { + list_add_tail(&dsd_ptr->list, + &difctx->ldif_dsd_list); + sp->flags |= SRB_CRC_CTX_DSD_VALID; + } else { + list_add_tail(&dsd_ptr->list, + &difctx->ldif_dsd_list); + tc->ctx_dsd_alloced = 1; + } + + /* add new list to cmd iocb or last list */ + *cur_dsd++ = + cpu_to_le32(LSD(dsd_ptr->dsd_list_dma)); + *cur_dsd++ = + cpu_to_le32(MSD(dsd_ptr->dsd_list_dma)); + *cur_dsd++ = dsd_list_len; + cur_dsd = dsd_ptr->dsd_addr; } - - /* add new list to cmd iocb or last list */ - *cur_dsd++ = cpu_to_le32(LSD(dsd_ptr->dsd_list_dma)); - *cur_dsd++ = cpu_to_le32(MSD(dsd_ptr->dsd_list_dma)); - *cur_dsd++ = dsd_list_len; - cur_dsd = (uint32_t *)next_dsd; + *cur_dsd++ = cpu_to_le32(LSD(dif_dsd->dsd_list_dma)); + *cur_dsd++ = cpu_to_le32(MSD(dif_dsd->dsd_list_dma)); + *cur_dsd++ = cpu_to_le32(sglen); + avail_dsds--; + difctx->dif_bundl_len -= sglen; + track_difbundl_buf--; } - sle_dma = sg_dma_address(sg); - - *cur_dsd++ = cpu_to_le32(LSD(sle_dma)); - *cur_dsd++ = cpu_to_le32(MSD(sle_dma)); - *cur_dsd++ = cpu_to_le32(sg_dma_len(sg)); - avail_dsds--; + ql_dbg(ql_dbg_tgt + ql_dbg_verbose, vha, 0xe026, + "%s: no_ldif_dsd:%x, no_dif_bundl:%x\n", __func__, + difctx->no_ldif_dsd, difctx->no_dif_bundl); + } else { + for_each_sg(sgl, sg, tot_dsds, i) { + dma_addr_t sle_dma; + + /* Allocate additional continuation packets? */ + if (avail_dsds == 0) { + avail_dsds = (used_dsds > QLA_DSDS_PER_IOCB) ? + QLA_DSDS_PER_IOCB : used_dsds; + dsd_list_len = (avail_dsds + 1) * 12; + used_dsds -= avail_dsds; + + /* allocate tracking DS */ + dsd_ptr = kzalloc(sizeof(*dsd_ptr), GFP_ATOMIC); + if (!dsd_ptr) { + ql_dbg(ql_dbg_tgt + ql_dbg_verbose, + vha, 0xe027, + "%s: failed alloc dsd_dma...\n", + __func__); + return 1; + } + + /* allocate new list */ + dsd_ptr->dsd_addr = + dma_pool_alloc(ha->dl_dma_pool, GFP_ATOMIC, + &dsd_ptr->dsd_list_dma); + if (!dsd_ptr->dsd_addr) { + /* need to cleanup only this dsd_ptr */ + /* rest will be done by sp_free_dma() */ + kfree(dsd_ptr); + return 1; + } + + if (sp) { + list_add_tail(&dsd_ptr->list, + &difctx->dsd_list); + sp->flags |= SRB_CRC_CTX_DSD_VALID; + } else { + list_add_tail(&dsd_ptr->list, + &difctx->dsd_list); + tc->ctx_dsd_alloced = 1; + } + + /* add new list to cmd iocb or last list */ + *cur_dsd++ = + cpu_to_le32(LSD(dsd_ptr->dsd_list_dma)); + *cur_dsd++ = + cpu_to_le32(MSD(dsd_ptr->dsd_list_dma)); + *cur_dsd++ = dsd_list_len; + cur_dsd = dsd_ptr->dsd_addr; + } + sle_dma = sg_dma_address(sg); + *cur_dsd++ = cpu_to_le32(LSD(sle_dma)); + *cur_dsd++ = cpu_to_le32(MSD(sle_dma)); + *cur_dsd++ = cpu_to_le32(sg_dma_len(sg)); + avail_dsds--; + } } /* Null termination */ *cur_dsd++ = 0; @@ -1187,7 +1395,6 @@ qla24xx_walk_and_build_prot_sglist(struct qla_hw_data *ha, srb_t *sp, *cur_dsd++ = 0; return 0; } - /** * qla24xx_build_scsi_crc_2_iocbs() - Build IOCB command utilizing Command * Type 6 IOCB types. @@ -1416,21 +1623,19 @@ qla24xx_start_scsi(srb_t *sp) uint16_t req_cnt; uint16_t tot_dsds; struct req_que *req = NULL; - struct rsp_que *rsp = NULL; struct scsi_cmnd *cmd = GET_CMD_SP(sp); struct scsi_qla_host *vha = sp->vha; struct qla_hw_data *ha = vha->hw; /* Setup device pointers. */ req = vha->req; - rsp = req->rsp; /* So we know we haven't pci_map'ed anything yet */ tot_dsds = 0; /* Send marker if required */ if (vha->marker_needed != 0) { - if (qla2x00_marker(vha, req, rsp, 0, 0, MK_SYNC_ALL) != + if (qla2x00_marker(vha, ha->base_qpair, 0, 0, MK_SYNC_ALL) != QLA_SUCCESS) return QLA_FUNCTION_FAILED; vha->marker_needed = 0; @@ -1583,7 +1788,7 @@ qla24xx_dif_start_scsi(srb_t *sp) /* Send marker if required */ if (vha->marker_needed != 0) { - if (qla2x00_marker(vha, req, rsp, 0, 0, MK_SYNC_ALL) != + if (qla2x00_marker(vha, ha->base_qpair, 0, 0, MK_SYNC_ALL) != QLA_SUCCESS) return QLA_FUNCTION_FAILED; vha->marker_needed = 0; @@ -1754,7 +1959,6 @@ qla2xxx_start_scsi_mq(srb_t *sp) uint16_t req_cnt; uint16_t tot_dsds; struct req_que *req = NULL; - struct rsp_que *rsp = NULL; struct scsi_cmnd *cmd = GET_CMD_SP(sp); struct scsi_qla_host *vha = sp->fcport->vha; struct qla_hw_data *ha = vha->hw; @@ -1764,7 +1968,6 @@ qla2xxx_start_scsi_mq(srb_t *sp) spin_lock_irqsave(&qpair->qp_lock, flags); /* Setup qpair pointers */ - rsp = qpair->rsp; req = qpair->req; /* So we know we haven't pci_map'ed anything yet */ @@ -1772,7 +1975,7 @@ qla2xxx_start_scsi_mq(srb_t *sp) /* Send marker if required */ if (vha->marker_needed != 0) { - if (__qla2x00_marker(vha, req, rsp, 0, 0, MK_SYNC_ALL) != + if (__qla2x00_marker(vha, qpair, 0, 0, MK_SYNC_ALL) != QLA_SUCCESS) { spin_unlock_irqrestore(&qpair->qp_lock, flags); return QLA_FUNCTION_FAILED; @@ -1940,7 +2143,7 @@ qla2xxx_dif_start_scsi_mq(srb_t *sp) /* Send marker if required */ if (vha->marker_needed != 0) { - if (__qla2x00_marker(vha, req, rsp, 0, 0, MK_SYNC_ALL) != + if (__qla2x00_marker(vha, qpair, 0, 0, MK_SYNC_ALL) != QLA_SUCCESS) { spin_unlock_irqrestore(&qpair->qp_lock, flags); return QLA_FUNCTION_FAILED; @@ -2208,8 +2411,11 @@ qla24xx_prli_iocb(srb_t *sp, struct logio_entry_24xx *logio) logio->entry_type = LOGINOUT_PORT_IOCB_TYPE; logio->control_flags = cpu_to_le16(LCF_COMMAND_PRLI); - if (lio->u.logio.flags & SRB_LOGIN_NVME_PRLI) + if (lio->u.logio.flags & SRB_LOGIN_NVME_PRLI) { logio->control_flags |= LCF_NVME_PRLI; + if (sp->vha->flags.nvme_first_burst) + logio->io_parameter[0] = NVME_PRLI_SP_FIRST_BURST; + } logio->nport_handle = cpu_to_le16(sp->fcport->loop_id); logio->port_id[0] = sp->fcport->d_id.b.al_pa; @@ -2991,8 +3197,8 @@ qla82xx_start_scsi(srb_t *sp) /* Send marker if required */ if (vha->marker_needed != 0) { - if (qla2x00_marker(vha, req, - rsp, 0, 0, MK_SYNC_ALL) != QLA_SUCCESS) { + if (qla2x00_marker(vha, ha->base_qpair, + 0, 0, MK_SYNC_ALL) != QLA_SUCCESS) { ql_log(ql_log_warn, vha, 0x300c, "qla2x00_marker failed for cmd=%p.\n", cmd); return QLA_FUNCTION_FAILED; @@ -3434,23 +3640,22 @@ qla24xx_prlo_iocb(srb_t *sp, struct logio_entry_24xx *logio) int qla2x00_start_sp(srb_t *sp) { - int rval; + int rval = QLA_SUCCESS; scsi_qla_host_t *vha = sp->vha; struct qla_hw_data *ha = vha->hw; struct qla_qpair *qp = sp->qpair; void *pkt; unsigned long flags; - rval = QLA_FUNCTION_FAILED; spin_lock_irqsave(qp->qp_lock_ptr, flags); pkt = __qla2x00_alloc_iocbs(sp->qpair, sp); if (!pkt) { + rval = EAGAIN; ql_log(ql_log_warn, vha, 0x700c, "qla2x00_alloc_iocbs failed.\n"); goto done; } - rval = QLA_SUCCESS; switch (sp->type) { case SRB_LOGIN_CMD: IS_FWI2_CAPABLE(ha) ? @@ -3646,8 +3851,8 @@ qla2x00_start_bidir(srb_t *sp, struct scsi_qla_host *vha, uint32_t tot_dsds) /* Send marker if required */ if (vha->marker_needed != 0) { - if (qla2x00_marker(vha, req, - rsp, 0, 0, MK_SYNC_ALL) != QLA_SUCCESS) + if (qla2x00_marker(vha, ha->base_qpair, + 0, 0, MK_SYNC_ALL) != QLA_SUCCESS) return EXT_STATUS_MAILBOX; vha->marker_needed = 0; } diff --git a/drivers/scsi/qla2xxx/qla_isr.c b/drivers/scsi/qla2xxx/qla_isr.c index 8507c43b918c..69bbea9239cc 100644 --- a/drivers/scsi/qla2xxx/qla_isr.c +++ b/drivers/scsi/qla2xxx/qla_isr.c @@ -834,7 +834,8 @@ skip_rio: * Restore for Physical Port only */ if (!vha->vp_idx) { - if (ha->flags.fawwpn_enabled) { + if (ha->flags.fawwpn_enabled && + (ha->current_topology == ISP_CFG_F)) { void *wwpn = ha->init_cb->port_name; memcpy(vha->port_name, wwpn, WWN_SIZE); fc_host_port_name(vha->host) = @@ -1714,6 +1715,15 @@ qla24xx_logio_entry(scsi_qla_host_t *vha, struct req_que *req, vha->hw->exch_starvation = 0; data[0] = MBS_COMMAND_COMPLETE; + + if (sp->type == SRB_PRLI_CMD) { + lio->u.logio.iop[0] = + le32_to_cpu(logio->io_parameter[0]); + lio->u.logio.iop[1] = + le32_to_cpu(logio->io_parameter[1]); + goto logio_done; + } + if (sp->type != SRB_LOGIN_CMD) goto logio_done; @@ -2725,6 +2735,17 @@ check_scsi_status: cp->device->vendor); break; + case CS_DMA: + ql_log(ql_log_info, fcport->vha, 0x3022, + "CS_DMA error: 0x%x-0x%x (0x%x) nexus=%ld:%d:%llu portid=%06x oxid=0x%x cdb=%10phN len=0x%x rsp_info=0x%x resid=0x%x fw_resid=0x%x sp=%p cp=%p.\n", + comp_status, scsi_status, res, vha->host_no, + cp->device->id, cp->device->lun, fcport->d_id.b24, + ox_id, cp->cmnd, scsi_bufflen(cp), rsp_info_len, + resid_len, fw_resid_len, sp, cp); + ql_dump_buffer(ql_dbg_tgt + ql_dbg_verbose, vha, 0xe0ee, + pkt, sizeof(*sts24)); + res = DID_ERROR << 16; + break; default: res = DID_ERROR << 16; break; @@ -3410,7 +3431,7 @@ qla24xx_enable_msix(struct qla_hw_data *ha, struct rsp_que *rsp) min_vecs++; } - if (USER_CTRL_IRQ(ha)) { + if (USER_CTRL_IRQ(ha) || !ha->mqiobase) { /* user wants to control IRQ setting for target mode */ ret = pci_alloc_irq_vectors(ha->pdev, min_vecs, ha->msix_count, PCI_IRQ_MSIX); diff --git a/drivers/scsi/qla2xxx/qla_mbx.c b/drivers/scsi/qla2xxx/qla_mbx.c index 191b6b7c8747..5400696e1f6b 100644 --- a/drivers/scsi/qla2xxx/qla_mbx.c +++ b/drivers/scsi/qla2xxx/qla_mbx.c @@ -1109,7 +1109,12 @@ qla2x00_get_fw_version(scsi_qla_host_t *vha) * FW supports nvme and driver load parameter requested nvme. * BIT 26 of fw_attributes indicates NVMe support. */ - if ((ha->fw_attributes_h & 0x400) && ql2xnvmeenable) { + if ((ha->fw_attributes_h & + (FW_ATTR_H_NVME | FW_ATTR_H_NVME_UPDATED)) && + ql2xnvmeenable) { + if (ha->fw_attributes_h & FW_ATTR_H_NVME_FBURST) + vha->flags.nvme_first_burst = 1; + vha->flags.nvme_enabled = 1; ql_log(ql_log_info, vha, 0xd302, "%s: FC-NVMe is Enabled (0x%x)\n", @@ -1508,16 +1513,12 @@ qla2x00_abort_target(struct fc_port *fcport, uint64_t l, int tag) mbx_cmd_t mc; mbx_cmd_t *mcp = &mc; scsi_qla_host_t *vha; - struct req_que *req; - struct rsp_que *rsp; vha = fcport->vha; ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x103e, "Entered %s.\n", __func__); - req = vha->hw->req_q_map[0]; - rsp = req->rsp; mcp->mb[0] = MBC_ABORT_TARGET; mcp->out_mb = MBX_9|MBX_2|MBX_1|MBX_0; if (HAS_EXTENDED_IDS(vha->hw)) { @@ -1540,7 +1541,7 @@ qla2x00_abort_target(struct fc_port *fcport, uint64_t l, int tag) } /* Issue marker IOCB. */ - rval2 = qla2x00_marker(vha, req, rsp, fcport->loop_id, 0, + rval2 = qla2x00_marker(vha, vha->hw->base_qpair, fcport->loop_id, 0, MK_SYNC_ID); if (rval2 != QLA_SUCCESS) { ql_dbg(ql_dbg_mbx, vha, 0x1040, @@ -1560,16 +1561,12 @@ qla2x00_lun_reset(struct fc_port *fcport, uint64_t l, int tag) mbx_cmd_t mc; mbx_cmd_t *mcp = &mc; scsi_qla_host_t *vha; - struct req_que *req; - struct rsp_que *rsp; vha = fcport->vha; ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x1042, "Entered %s.\n", __func__); - req = vha->hw->req_q_map[0]; - rsp = req->rsp; mcp->mb[0] = MBC_LUN_RESET; mcp->out_mb = MBX_9|MBX_3|MBX_2|MBX_1|MBX_0; if (HAS_EXTENDED_IDS(vha->hw)) @@ -1589,7 +1586,7 @@ qla2x00_lun_reset(struct fc_port *fcport, uint64_t l, int tag) } /* Issue marker IOCB. */ - rval2 = qla2x00_marker(vha, req, rsp, fcport->loop_id, l, + rval2 = qla2x00_marker(vha, vha->hw->base_qpair, fcport->loop_id, l, MK_SYNC_ID_LUN); if (rval2 != QLA_SUCCESS) { ql_dbg(ql_dbg_mbx, vha, 0x1044, @@ -2244,10 +2241,7 @@ qla2x00_lip_reset(scsi_qla_host_t *vha) mcp->out_mb = MBX_2|MBX_1|MBX_0; } else if (IS_FWI2_CAPABLE(vha->hw)) { mcp->mb[0] = MBC_LIP_FULL_LOGIN; - if (N2N_TOPO(vha->hw)) - mcp->mb[1] = BIT_4; /* re-init */ - else - mcp->mb[1] = BIT_6; /* LIP */ + mcp->mb[1] = BIT_4; mcp->mb[2] = 0; mcp->mb[3] = vha->hw->loop_reset_delay; mcp->out_mb = MBX_3|MBX_2|MBX_1|MBX_0; @@ -2757,7 +2751,7 @@ qla2x00_full_login_lip(scsi_qla_host_t *vha) "Entered %s.\n", __func__); mcp->mb[0] = MBC_LIP_FULL_LOGIN; - mcp->mb[1] = IS_FWI2_CAPABLE(vha->hw) ? BIT_3 : 0; + mcp->mb[1] = IS_FWI2_CAPABLE(vha->hw) ? BIT_4 : 0; mcp->mb[2] = 0; mcp->mb[3] = 0; mcp->out_mb = MBX_3|MBX_2|MBX_1|MBX_0; @@ -3184,7 +3178,6 @@ __qla24xx_issue_tmf(char *name, uint32_t type, struct fc_port *fcport, scsi_qla_host_t *vha; struct qla_hw_data *ha; struct req_que *req; - struct rsp_que *rsp; struct qla_qpair *qpair; vha = fcport->vha; @@ -3197,10 +3190,7 @@ __qla24xx_issue_tmf(char *name, uint32_t type, struct fc_port *fcport, if (vha->vp_idx && vha->qpair) { /* NPIV port */ qpair = vha->qpair; - rsp = qpair->rsp; req = qpair->req; - } else { - rsp = req->rsp; } tsk = dma_pool_zalloc(ha->s_dma_pool, GFP_KERNEL, &tsk_dma); @@ -3257,7 +3247,7 @@ __qla24xx_issue_tmf(char *name, uint32_t type, struct fc_port *fcport, } /* Issue marker IOCB. */ - rval2 = qla2x00_marker(vha, req, rsp, fcport->loop_id, l, + rval2 = qla2x00_marker(vha, ha->base_qpair, fcport->loop_id, l, type == TCF_LUN_RESET ? MK_SYNC_ID_LUN: MK_SYNC_ID); if (rval2 != QLA_SUCCESS) { ql_dbg(ql_dbg_mbx, vha, 0x1099, @@ -5248,6 +5238,66 @@ qla81xx_write_mpi_register(scsi_qla_host_t *vha, uint16_t *mb) return rval; } +/* Set the specified data rate */ +int +qla2x00_set_data_rate(scsi_qla_host_t *vha, uint16_t mode) +{ + int rval; + mbx_cmd_t mc; + mbx_cmd_t *mcp = &mc; + struct qla_hw_data *ha = vha->hw; + uint16_t val; + + ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x1106, + "Entered %s speed:0x%x mode:0x%x.\n", __func__, ha->set_data_rate, + mode); + + if (!IS_FWI2_CAPABLE(ha)) + return QLA_FUNCTION_FAILED; + + memset(mcp, 0, sizeof(*mcp)); + switch (ha->set_data_rate) { + case PORT_SPEED_AUTO: + case PORT_SPEED_4GB: + case PORT_SPEED_8GB: + case PORT_SPEED_16GB: + case PORT_SPEED_32GB: + val = ha->set_data_rate; + break; + default: + ql_log(ql_log_warn, vha, 0x1199, + "Unrecognized speed setting:%d. Setting Autoneg\n", + ha->set_data_rate); + val = ha->set_data_rate = PORT_SPEED_AUTO; + break; + } + + mcp->mb[0] = MBC_DATA_RATE; + mcp->mb[1] = mode; + mcp->mb[2] = val; + + mcp->out_mb = MBX_2|MBX_1|MBX_0; + mcp->in_mb = MBX_2|MBX_1|MBX_0; + if (IS_QLA83XX(ha) || IS_QLA27XX(ha)) + mcp->in_mb |= MBX_4|MBX_3; + mcp->tov = MBX_TOV_SECONDS; + mcp->flags = 0; + rval = qla2x00_mailbox_command(vha, mcp); + if (rval != QLA_SUCCESS) { + ql_dbg(ql_dbg_mbx, vha, 0x1107, + "Failed=%x mb[0]=%x.\n", rval, mcp->mb[0]); + } else { + if (mcp->mb[1] != 0x7) + ql_dbg(ql_dbg_mbx, vha, 0x1179, + "Speed set:0x%x\n", mcp->mb[1]); + + ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x1108, + "Done %s.\n", __func__); + } + + return rval; +} + int qla2x00_get_data_rate(scsi_qla_host_t *vha) { @@ -5263,7 +5313,7 @@ qla2x00_get_data_rate(scsi_qla_host_t *vha) return QLA_FUNCTION_FAILED; mcp->mb[0] = MBC_DATA_RATE; - mcp->mb[1] = 0; + mcp->mb[1] = QLA_GET_DATA_RATE; mcp->out_mb = MBX_1|MBX_0; mcp->in_mb = MBX_2|MBX_1|MBX_0; if (IS_QLA83XX(ha) || IS_QLA27XX(ha)) @@ -6268,8 +6318,6 @@ int __qla24xx_parse_gpdb(struct scsi_qla_host *vha, fc_port_t *fcport, fcport->d_id.b.rsvd_1 = 0; if (fcport->fc4f_nvme) { - fcport->nvme_prli_service_param = - pd->prli_nvme_svc_param_word_3; fcport->port_type = FCT_NVME; } else { /* If not target must be initiator or unknown type. */ diff --git a/drivers/scsi/qla2xxx/qla_nvme.c b/drivers/scsi/qla2xxx/qla_nvme.c index 39d892bbd219..41c85da3ab32 100644 --- a/drivers/scsi/qla2xxx/qla_nvme.c +++ b/drivers/scsi/qla2xxx/qla_nvme.c @@ -185,6 +185,14 @@ static void qla_nvme_abort_work(struct work_struct *work) struct qla_hw_data *ha = fcport->vha->hw; int rval; + if (fcport) + ql_dbg(ql_dbg_io, fcport->vha, 0xffff, + "%s called for sp=%p, hndl=%x on fcport=%p deleted=%d\n", + __func__, sp, sp->handle, fcport, fcport->deleted); + + if (!ha->flags.fw_started && (fcport && fcport->deleted)) + return; + rval = ha->isp_ops->abort_command(sp); ql_dbg(ql_dbg_io, fcport->vha, 0x212b, @@ -358,17 +366,24 @@ static inline int qla2x00_start_nvme_mq(srb_t *sp) /* No data transfer how do we check buffer len == 0?? */ if (fd->io_dir == NVMEFC_FCP_READ) { - cmd_pkt->control_flags = - cpu_to_le16(CF_READ_DATA | CF_NVME_ENABLE); + cmd_pkt->control_flags = CF_READ_DATA; vha->qla_stats.input_bytes += fd->payload_length; vha->qla_stats.input_requests++; } else if (fd->io_dir == NVMEFC_FCP_WRITE) { - cmd_pkt->control_flags = - cpu_to_le16(CF_WRITE_DATA | CF_NVME_ENABLE); + cmd_pkt->control_flags = CF_WRITE_DATA; + if ((vha->flags.nvme_first_burst) && + (sp->fcport->nvme_prli_service_param & + NVME_PRLI_SP_FIRST_BURST)) { + if ((fd->payload_length <= + sp->fcport->nvme_first_burst_size) || + (sp->fcport->nvme_first_burst_size == 0)) + cmd_pkt->control_flags |= + CF_NVME_FIRST_BURST_ENABLE; + } vha->qla_stats.output_bytes += fd->payload_length; vha->qla_stats.output_requests++; } else if (fd->io_dir == 0) { - cmd_pkt->control_flags = cpu_to_le16(CF_NVME_ENABLE); + cmd_pkt->control_flags = 0; } /* Set NPORT-ID */ @@ -600,6 +615,7 @@ static void qla_nvme_unregister_remote_port(struct work_struct *work) struct fc_port *fcport = container_of(work, struct fc_port, nvme_del_work); struct qla_nvme_rport *qla_rport, *trport; + scsi_qla_host_t *base_vha; if (!IS_ENABLED(CONFIG_NVME_FC)) return; @@ -607,6 +623,15 @@ static void qla_nvme_unregister_remote_port(struct work_struct *work) ql_log(ql_log_warn, NULL, 0x2112, "%s: unregister remoteport on %p\n",__func__, fcport); + base_vha = pci_get_drvdata(fcport->vha->hw->pdev); + if (test_bit(PFLG_DRIVER_REMOVING, &base_vha->pci_flags)) { + ql_dbg(ql_dbg_disc, fcport->vha, 0x2114, + "%s: Notify FC-NVMe transport, set devloss=0\n", + __func__); + + nvme_fc_set_remoteport_devloss(fcport->nvme_remote_port, 0); + } + list_for_each_entry_safe(qla_rport, trport, &fcport->vha->nvme_rport_list, list) { if (qla_rport->fcport == fcport) { @@ -623,23 +648,11 @@ static void qla_nvme_unregister_remote_port(struct work_struct *work) void qla_nvme_delete(struct scsi_qla_host *vha) { - struct qla_nvme_rport *qla_rport, *trport; - fc_port_t *fcport; int nv_ret; if (!IS_ENABLED(CONFIG_NVME_FC)) return; - list_for_each_entry_safe(qla_rport, trport, - &vha->nvme_rport_list, list) { - fcport = qla_rport->fcport; - - ql_log(ql_log_info, fcport->vha, 0x2114, "%s: fcport=%p\n", - __func__, fcport); - - nvme_fc_set_remoteport_devloss(fcport->nvme_remote_port, 0); - } - if (vha->nvme_local_port) { init_completion(&vha->nvme_del_done); ql_log(ql_log_info, vha, 0x2116, diff --git a/drivers/scsi/qla2xxx/qla_nvme.h b/drivers/scsi/qla2xxx/qla_nvme.h index 4941d107fb1c..da8dad5ad693 100644 --- a/drivers/scsi/qla2xxx/qla_nvme.h +++ b/drivers/scsi/qla2xxx/qla_nvme.h @@ -57,7 +57,7 @@ struct cmd_nvme { uint64_t rsvd; uint16_t control_flags; /* Control Flags */ -#define CF_NVME_ENABLE BIT_9 +#define CF_NVME_FIRST_BURST_ENABLE BIT_11 #define CF_DIF_SEG_DESCR_ENABLE BIT_3 #define CF_DATA_SEG_DESCR_ENABLE BIT_2 #define CF_READ_DATA BIT_1 diff --git a/drivers/scsi/qla2xxx/qla_os.c b/drivers/scsi/qla2xxx/qla_os.c index c6ef83d0d99b..677f82fdf56f 100644 --- a/drivers/scsi/qla2xxx/qla_os.c +++ b/drivers/scsi/qla2xxx/qla_os.c @@ -285,6 +285,27 @@ MODULE_PARM_DESC(qla2xuseresexchforels, "Reserve 1/2 of emergency exchanges for ELS.\n" " 0 (default): disabled"); +int ql2xprotmask; +module_param(ql2xprotmask, int, 0644); +MODULE_PARM_DESC(ql2xprotmask, + "Override DIF/DIX protection capabilities mask\n" + "Default is 0 which sets protection mask based on " + "capabilities reported by HBA firmware.\n"); + +int ql2xprotguard; +module_param(ql2xprotguard, int, 0644); +MODULE_PARM_DESC(ql2xprotguard, "Override choice of DIX checksum\n" + " 0 -- Let HBA firmware decide\n" + " 1 -- Force T10 CRC\n" + " 2 -- Force IP checksum\n"); + +int ql2xdifbundlinginternalbuffers; +module_param(ql2xdifbundlinginternalbuffers, int, 0644); +MODULE_PARM_DESC(ql2xdifbundlinginternalbuffers, + "Force using internal buffers for DIF information\n" + "0 (Default). Based on check.\n" + "1 Force using internal buffers\n"); + /* * SCSI host template entry points */ @@ -804,7 +825,44 @@ qla2xxx_qpair_sp_free_dma(void *ptr) ha->gbl_dsd_inuse -= ctx1->dsd_use_cnt; ha->gbl_dsd_avail += ctx1->dsd_use_cnt; mempool_free(ctx1, ha->ctx_mempool); + sp->flags &= ~SRB_FCP_CMND_DMA_VALID; + } + if (sp->flags & SRB_DIF_BUNDL_DMA_VALID) { + struct crc_context *difctx = sp->u.scmd.ctx; + struct dsd_dma *dif_dsd, *nxt_dsd; + + list_for_each_entry_safe(dif_dsd, nxt_dsd, + &difctx->ldif_dma_hndl_list, list) { + list_del(&dif_dsd->list); + dma_pool_free(ha->dif_bundl_pool, dif_dsd->dsd_addr, + dif_dsd->dsd_list_dma); + kfree(dif_dsd); + difctx->no_dif_bundl--; + } + + list_for_each_entry_safe(dif_dsd, nxt_dsd, + &difctx->ldif_dsd_list, list) { + list_del(&dif_dsd->list); + dma_pool_free(ha->dl_dma_pool, dif_dsd->dsd_addr, + dif_dsd->dsd_list_dma); + kfree(dif_dsd); + difctx->no_ldif_dsd--; + } + + if (difctx->no_ldif_dsd) { + ql_dbg(ql_dbg_tgt+ql_dbg_verbose, sp->vha, 0xe022, + "%s: difctx->no_ldif_dsd=%x\n", + __func__, difctx->no_ldif_dsd); + } + + if (difctx->no_dif_bundl) { + ql_dbg(ql_dbg_tgt+ql_dbg_verbose, sp->vha, 0xe022, + "%s: difctx->no_dif_bundl=%x\n", + __func__, difctx->no_dif_bundl); + } + sp->flags &= ~SRB_DIF_BUNDL_DMA_VALID; } + end: CMD_SP(cmd) = NULL; qla2xxx_rel_qpair_sp(sp->qpair, sp); @@ -3342,13 +3400,16 @@ skip_dpc: "Registering for DIF/DIX type 1 and 3 protection.\n"); if (ql2xenabledif == 1) prot = SHOST_DIX_TYPE0_PROTECTION; - scsi_host_set_prot(host, - prot | SHOST_DIF_TYPE1_PROTECTION - | SHOST_DIF_TYPE2_PROTECTION - | SHOST_DIF_TYPE3_PROTECTION - | SHOST_DIX_TYPE1_PROTECTION - | SHOST_DIX_TYPE2_PROTECTION - | SHOST_DIX_TYPE3_PROTECTION); + if (ql2xprotmask) + scsi_host_set_prot(host, ql2xprotmask); + else + scsi_host_set_prot(host, + prot | SHOST_DIF_TYPE1_PROTECTION + | SHOST_DIF_TYPE2_PROTECTION + | SHOST_DIF_TYPE3_PROTECTION + | SHOST_DIX_TYPE1_PROTECTION + | SHOST_DIX_TYPE2_PROTECTION + | SHOST_DIX_TYPE3_PROTECTION); guard = SHOST_DIX_GUARD_CRC; @@ -3356,7 +3417,10 @@ skip_dpc: (ql2xenabledif > 1 || IS_PI_DIFB_DIX0_CAPABLE(ha))) guard |= SHOST_DIX_GUARD_IP; - scsi_host_set_guard(host, guard); + if (ql2xprotguard) + scsi_host_set_guard(host, ql2xprotguard); + else + scsi_host_set_guard(host, guard); } else base_vha->flags.difdix_supported = 0; } @@ -3997,9 +4061,86 @@ qla2x00_mem_alloc(struct qla_hw_data *ha, uint16_t req_len, uint16_t rsp_len, "Failed to allocate memory for fcp_cmnd_dma_pool.\n"); goto fail_dl_dma_pool; } + + if (ql2xenabledif) { + u64 bufsize = DIF_BUNDLING_DMA_POOL_SIZE; + struct dsd_dma *dsd, *nxt; + uint i; + /* Creata a DMA pool of buffers for DIF bundling */ + ha->dif_bundl_pool = dma_pool_create(name, + &ha->pdev->dev, DIF_BUNDLING_DMA_POOL_SIZE, 8, 0); + if (!ha->dif_bundl_pool) { + ql_dbg_pci(ql_dbg_init, ha->pdev, 0x0024, + "%s: failed create dif_bundl_pool\n", + __func__); + goto fail_dif_bundl_dma_pool; + } + + INIT_LIST_HEAD(&ha->pool.good.head); + INIT_LIST_HEAD(&ha->pool.unusable.head); + ha->pool.good.count = 0; + ha->pool.unusable.count = 0; + for (i = 0; i < 128; i++) { + dsd = kzalloc(sizeof(*dsd), GFP_ATOMIC); + if (!dsd) { + ql_dbg_pci(ql_dbg_init, ha->pdev, + 0xe0ee, "%s: failed alloc dsd\n", + __func__); + return 1; + } + ha->dif_bundle_kallocs++; + + dsd->dsd_addr = dma_pool_alloc( + ha->dif_bundl_pool, GFP_ATOMIC, + &dsd->dsd_list_dma); + if (!dsd->dsd_addr) { + ql_dbg_pci(ql_dbg_init, ha->pdev, + 0xe0ee, + "%s: failed alloc ->dsd_addr\n", + __func__); + kfree(dsd); + ha->dif_bundle_kallocs--; + continue; + } + ha->dif_bundle_dma_allocs++; + + /* + * if DMA buffer crosses 4G boundary, + * put it on bad list + */ + if (MSD(dsd->dsd_list_dma) ^ + MSD(dsd->dsd_list_dma + bufsize)) { + list_add_tail(&dsd->list, + &ha->pool.unusable.head); + ha->pool.unusable.count++; + } else { + list_add_tail(&dsd->list, + &ha->pool.good.head); + ha->pool.good.count++; + } + } + + /* return the good ones back to the pool */ + list_for_each_entry_safe(dsd, nxt, + &ha->pool.good.head, list) { + list_del(&dsd->list); + dma_pool_free(ha->dif_bundl_pool, + dsd->dsd_addr, dsd->dsd_list_dma); + ha->dif_bundle_dma_allocs--; + kfree(dsd); + ha->dif_bundle_kallocs--; + } + + ql_dbg_pci(ql_dbg_init, ha->pdev, 0x0024, + "%s: dif dma pool (good=%u unusable=%u)\n", + __func__, ha->pool.good.count, + ha->pool.unusable.count); + } + ql_dbg_pci(ql_dbg_init, ha->pdev, 0x0025, - "dl_dma_pool=%p fcp_cmnd_dma_pool=%p.\n", - ha->dl_dma_pool, ha->fcp_cmnd_dma_pool); + "dl_dma_pool=%p fcp_cmnd_dma_pool=%p dif_bundl_pool=%p.\n", + ha->dl_dma_pool, ha->fcp_cmnd_dma_pool, + ha->dif_bundl_pool); } /* Allocate memory for SNS commands */ @@ -4164,6 +4305,24 @@ fail_free_ms_iocb: dma_free_coherent(&ha->pdev->dev, sizeof(struct sns_cmd_pkt), ha->sns_cmd, ha->sns_cmd_dma); fail_dma_pool: + if (ql2xenabledif) { + struct dsd_dma *dsd, *nxt; + + list_for_each_entry_safe(dsd, nxt, &ha->pool.unusable.head, + list) { + list_del(&dsd->list); + dma_pool_free(ha->dif_bundl_pool, dsd->dsd_addr, + dsd->dsd_list_dma); + ha->dif_bundle_dma_allocs--; + kfree(dsd); + ha->dif_bundle_kallocs--; + ha->pool.unusable.count--; + } + dma_pool_destroy(ha->dif_bundl_pool); + ha->dif_bundl_pool = NULL; + } + +fail_dif_bundl_dma_pool: if (IS_QLA82XX(ha) || ql2xenabledif) { dma_pool_destroy(ha->fcp_cmnd_dma_pool); ha->fcp_cmnd_dma_pool = NULL; @@ -4544,6 +4703,32 @@ qla2x00_mem_free(struct qla_hw_data *ha) mempool_destroy(ha->ctx_mempool); + if (ql2xenabledif) { + struct dsd_dma *dsd, *nxt; + + list_for_each_entry_safe(dsd, nxt, &ha->pool.unusable.head, + list) { + list_del(&dsd->list); + dma_pool_free(ha->dif_bundl_pool, dsd->dsd_addr, + dsd->dsd_list_dma); + ha->dif_bundle_dma_allocs--; + kfree(dsd); + ha->dif_bundle_kallocs--; + ha->pool.unusable.count--; + } + list_for_each_entry_safe(dsd, nxt, &ha->pool.good.head, list) { + list_del(&dsd->list); + dma_pool_free(ha->dif_bundl_pool, dsd->dsd_addr, + dsd->dsd_list_dma); + ha->dif_bundle_dma_allocs--; + kfree(dsd); + ha->dif_bundle_kallocs--; + } + } + + if (ha->dif_bundl_pool) + dma_pool_destroy(ha->dif_bundl_pool); + qlt_mem_free(ha); if (ha->init_cb) @@ -5019,14 +5204,14 @@ qla2x00_do_work(struct scsi_qla_host *vha) struct qla_work_evt *e, *tmp; unsigned long flags; LIST_HEAD(work); + int rc; spin_lock_irqsave(&vha->work_lock, flags); list_splice_init(&vha->work_list, &work); spin_unlock_irqrestore(&vha->work_lock, flags); list_for_each_entry_safe(e, tmp, &work, list) { - list_del_init(&e->list); - + rc = QLA_SUCCESS; switch (e->type) { case QLA_EVT_AEN: fc_host_post_event(vha->host, fc_get_event_number(), @@ -5040,7 +5225,7 @@ qla2x00_do_work(struct scsi_qla_host *vha) e->u.logio.data); break; case QLA_EVT_ASYNC_LOGOUT: - qla2x00_async_logout(vha, e->u.logio.fcport); + rc = qla2x00_async_logout(vha, e->u.logio.fcport); break; case QLA_EVT_ASYNC_LOGOUT_DONE: qla2x00_async_logout_done(vha, e->u.logio.fcport, @@ -5085,7 +5270,7 @@ qla2x00_do_work(struct scsi_qla_host *vha) qla24xx_do_nack_work(vha, e); break; case QLA_EVT_ASYNC_PRLO: - qla2x00_async_prlo(vha, e->u.logio.fcport); + rc = qla2x00_async_prlo(vha, e->u.logio.fcport); break; case QLA_EVT_ASYNC_PRLO_DONE: qla2x00_async_prlo_done(vha, e->u.logio.fcport, @@ -5118,6 +5303,15 @@ qla2x00_do_work(struct scsi_qla_host *vha) e->u.fcport.fcport, false); break; } + + if (rc == EAGAIN) { + /* put 'work' at head of 'vha->work_list' */ + spin_lock_irqsave(&vha->work_lock, flags); + list_splice(&work, &vha->work_list); + spin_unlock_irqrestore(&vha->work_lock, flags); + break; + } + list_del_init(&e->list); if (e->flags & QLA_EVT_FLAG_FREE) kfree(e); @@ -6930,13 +7124,64 @@ qla2xxx_pci_resume(struct pci_dev *pdev) ha->flags.eeh_busy = 0; } +static void +qla_pci_reset_prepare(struct pci_dev *pdev) +{ + scsi_qla_host_t *base_vha = pci_get_drvdata(pdev); + struct qla_hw_data *ha = base_vha->hw; + struct qla_qpair *qpair; + + ql_log(ql_log_warn, base_vha, 0xffff, + "%s.\n", __func__); + + /* + * PCI FLR/function reset is about to reset the + * slot. Stop the chip to stop all DMA access. + * It is assumed that pci_reset_done will be called + * after FLR to resume Chip operation. + */ + ha->flags.eeh_busy = 1; + mutex_lock(&ha->mq_lock); + list_for_each_entry(qpair, &base_vha->qp_list, qp_list_elem) + qpair->online = 0; + mutex_unlock(&ha->mq_lock); + + set_bit(ABORT_ISP_ACTIVE, &base_vha->dpc_flags); + qla2x00_abort_isp_cleanup(base_vha); + qla2x00_abort_all_cmds(base_vha, DID_RESET << 16); +} + +static void +qla_pci_reset_done(struct pci_dev *pdev) +{ + scsi_qla_host_t *base_vha = pci_get_drvdata(pdev); + struct qla_hw_data *ha = base_vha->hw; + struct qla_qpair *qpair; + + ql_log(ql_log_warn, base_vha, 0xffff, + "%s.\n", __func__); + + /* + * FLR just completed by PCI layer. Resume adapter + */ + ha->flags.eeh_busy = 0; + mutex_lock(&ha->mq_lock); + list_for_each_entry(qpair, &base_vha->qp_list, qp_list_elem) + qpair->online = 1; + mutex_unlock(&ha->mq_lock); + + base_vha->flags.online = 1; + ha->isp_ops->abort_isp(base_vha); + clear_bit(ABORT_ISP_ACTIVE, &base_vha->dpc_flags); +} + static int qla2xxx_map_queues(struct Scsi_Host *shost) { int rc; scsi_qla_host_t *vha = (scsi_qla_host_t *)shost->hostdata; struct blk_mq_queue_map *qmap = &shost->tag_set.map[0]; - if (USER_CTRL_IRQ(vha->hw)) + if (USER_CTRL_IRQ(vha->hw) || !vha->hw->mqiobase) rc = blk_mq_map_queues(qmap); else rc = blk_mq_pci_map_queues(qmap, vha->hw->pdev, vha->irq_offset); @@ -6948,6 +7193,8 @@ static const struct pci_error_handlers qla2xxx_err_handler = { .mmio_enabled = qla2xxx_pci_mmio_enabled, .slot_reset = qla2xxx_pci_slot_reset, .resume = qla2xxx_pci_resume, + .reset_prepare = qla_pci_reset_prepare, + .reset_done = qla_pci_reset_done, }; static struct pci_device_id qla2xxx_pci_tbl[] = { diff --git a/drivers/scsi/qla2xxx/qla_target.c b/drivers/scsi/qla2xxx/qla_target.c index 510337eac106..582d1663f971 100644 --- a/drivers/scsi/qla2xxx/qla_target.c +++ b/drivers/scsi/qla2xxx/qla_target.c @@ -660,14 +660,14 @@ int qla24xx_async_notify_ack(scsi_qla_host_t *vha, fc_port_t *fcport, sp->u.iocb_cmd.u.nack.ntfy = ntfy; sp->done = qla2x00_async_nack_sp_done; - rval = qla2x00_start_sp(sp); - if (rval != QLA_SUCCESS) - goto done_free_sp; - ql_dbg(ql_dbg_disc, vha, 0x20f4, "Async-%s %8phC hndl %x %s\n", sp->name, fcport->port_name, sp->handle, c); + rval = qla2x00_start_sp(sp); + if (rval != QLA_SUCCESS) + goto done_free_sp; + return rval; done_free_sp: @@ -684,6 +684,9 @@ void qla24xx_do_nack_work(struct scsi_qla_host *vha, struct qla_work_evt *e) switch (e->u.nack.type) { case SRB_NACK_PRLI: + t = e->u.nack.fcport; + flush_work(&t->del_work); + flush_work(&t->free_work); mutex_lock(&vha->vha_tgt.tgt_mutex); t = qlt_create_sess(vha, e->u.nack.fcport, 0); mutex_unlock(&vha->vha_tgt.tgt_mutex); @@ -3230,7 +3233,7 @@ qlt_build_ctio_crc2_pkt(struct qla_qpair *qpair, struct qla_tgt_prm *prm) cur_dsd = (uint32_t *) &crc_ctx_pkt->u.bundling.dif_address; if (qla24xx_walk_and_build_prot_sglist(ha, NULL, cur_dsd, - prm->prot_seg_cnt, &tc)) + prm->prot_seg_cnt, cmd)) goto crc_queuing_error; } return QLA_SUCCESS; @@ -3257,13 +3260,10 @@ int qlt_xmit_response(struct qla_tgt_cmd *cmd, int xmit_type, unsigned long flags = 0; int res; - if (cmd->sess && cmd->sess->deleted) { + if (!qpair->fw_started || (cmd->reset_count != qpair->chip_reset) || + (cmd->sess && cmd->sess->deleted)) { cmd->state = QLA_TGT_STATE_PROCESSED; - if (cmd->sess->logout_completed) - /* no need to terminate. FW already freed exchange. */ - qlt_abort_cmd_on_host_reset(cmd->vha, cmd); - else - qlt_send_term_exchange(qpair, cmd, &cmd->atio, 0, 0); + qlt_abort_cmd_on_host_reset(cmd->vha, cmd); return 0; } @@ -6343,7 +6343,7 @@ static void qlt_tmr_work(struct qla_tgt *tgt, struct atio_from_isp *a = &prm->tm_iocb2; struct scsi_qla_host *vha = tgt->vha; struct qla_hw_data *ha = vha->hw; - struct fc_port *sess = NULL; + struct fc_port *sess; unsigned long flags; uint8_t *s_id = NULL; /* to hide compiler warnings */ int rc; @@ -6369,7 +6369,6 @@ static void qlt_tmr_work(struct qla_tgt *tgt, goto out_term2; } else { if (sess->deleted) { - sess = NULL; goto out_term2; } @@ -6377,7 +6376,6 @@ static void qlt_tmr_work(struct qla_tgt *tgt, ql_dbg(ql_dbg_tgt_tmr, vha, 0xf020, "%s: kref_get fail %8phC\n", __func__, sess->port_name); - sess = NULL; goto out_term2; } } @@ -6396,8 +6394,6 @@ static void qlt_tmr_work(struct qla_tgt *tgt, return; out_term2: - if (sess) - ha->tgt.tgt_ops->put_sess(sess); spin_unlock_irqrestore(&ha->tgt.sess_lock, flags); out_term: qlt_send_term_exchange(ha->base_qpair, NULL, &prm->tm_iocb2, 1, 0); diff --git a/drivers/scsi/qla2xxx/qla_target.h b/drivers/scsi/qla2xxx/qla_target.h index 577e1786a3f1..f3de75000a08 100644 --- a/drivers/scsi/qla2xxx/qla_target.h +++ b/drivers/scsi/qla2xxx/qla_target.h @@ -928,6 +928,8 @@ struct qla_tgt_cmd { uint64_t lba; uint16_t a_guard, e_guard, a_app_tag, e_app_tag; uint32_t a_ref_tag, e_ref_tag; +#define DIF_BUNDL_DMA_VALID 1 + uint16_t prot_flags; uint64_t jiffies_at_alloc; uint64_t jiffies_at_free; diff --git a/drivers/scsi/qla2xxx/qla_tmpl.c b/drivers/scsi/qla2xxx/qla_tmpl.c index 0ccd06f11f12..9e52500caff0 100644 --- a/drivers/scsi/qla2xxx/qla_tmpl.c +++ b/drivers/scsi/qla2xxx/qla_tmpl.c @@ -221,7 +221,13 @@ qla27xx_skip_entry(struct qla27xx_fwdt_entry *ent, void *buf) ent->hdr.driver_flags |= DRIVER_FLAG_SKIP_ENTRY; } -static int +static inline struct qla27xx_fwdt_entry * +qla27xx_next_entry(struct qla27xx_fwdt_entry *ent) +{ + return (void *)ent + ent->hdr.size; +} + +static struct qla27xx_fwdt_entry * qla27xx_fwdt_entry_t0(struct scsi_qla_host *vha, struct qla27xx_fwdt_entry *ent, void *buf, ulong *len) { @@ -229,10 +235,10 @@ qla27xx_fwdt_entry_t0(struct scsi_qla_host *vha, "%s: nop [%lx]\n", __func__, *len); qla27xx_skip_entry(ent, buf); - return false; + return qla27xx_next_entry(ent); } -static int +static struct qla27xx_fwdt_entry * qla27xx_fwdt_entry_t255(struct scsi_qla_host *vha, struct qla27xx_fwdt_entry *ent, void *buf, ulong *len) { @@ -241,10 +247,10 @@ qla27xx_fwdt_entry_t255(struct scsi_qla_host *vha, qla27xx_skip_entry(ent, buf); /* terminate */ - return true; + return NULL; } -static int +static struct qla27xx_fwdt_entry * qla27xx_fwdt_entry_t256(struct scsi_qla_host *vha, struct qla27xx_fwdt_entry *ent, void *buf, ulong *len) { @@ -255,10 +261,10 @@ qla27xx_fwdt_entry_t256(struct scsi_qla_host *vha, qla27xx_read_window(reg, ent->t256.base_addr, ent->t256.pci_offset, ent->t256.reg_count, ent->t256.reg_width, buf, len); - return false; + return qla27xx_next_entry(ent); } -static int +static struct qla27xx_fwdt_entry * qla27xx_fwdt_entry_t257(struct scsi_qla_host *vha, struct qla27xx_fwdt_entry *ent, void *buf, ulong *len) { @@ -269,10 +275,10 @@ qla27xx_fwdt_entry_t257(struct scsi_qla_host *vha, qla27xx_write_reg(reg, IOBASE_ADDR, ent->t257.base_addr, buf); qla27xx_write_reg(reg, ent->t257.pci_offset, ent->t257.write_data, buf); - return false; + return qla27xx_next_entry(ent); } -static int +static struct qla27xx_fwdt_entry * qla27xx_fwdt_entry_t258(struct scsi_qla_host *vha, struct qla27xx_fwdt_entry *ent, void *buf, ulong *len) { @@ -284,10 +290,10 @@ qla27xx_fwdt_entry_t258(struct scsi_qla_host *vha, qla27xx_read_window(reg, ent->t258.base_addr, ent->t258.pci_offset, ent->t258.reg_count, ent->t258.reg_width, buf, len); - return false; + return qla27xx_next_entry(ent); } -static int +static struct qla27xx_fwdt_entry * qla27xx_fwdt_entry_t259(struct scsi_qla_host *vha, struct qla27xx_fwdt_entry *ent, void *buf, ulong *len) { @@ -299,10 +305,10 @@ qla27xx_fwdt_entry_t259(struct scsi_qla_host *vha, qla27xx_write_reg(reg, ent->t259.banksel_offset, ent->t259.bank, buf); qla27xx_write_reg(reg, ent->t259.pci_offset, ent->t259.write_data, buf); - return false; + return qla27xx_next_entry(ent); } -static int +static struct qla27xx_fwdt_entry * qla27xx_fwdt_entry_t260(struct scsi_qla_host *vha, struct qla27xx_fwdt_entry *ent, void *buf, ulong *len) { @@ -313,10 +319,10 @@ qla27xx_fwdt_entry_t260(struct scsi_qla_host *vha, qla27xx_insert32(ent->t260.pci_offset, buf, len); qla27xx_read_reg(reg, ent->t260.pci_offset, buf, len); - return false; + return qla27xx_next_entry(ent); } -static int +static struct qla27xx_fwdt_entry * qla27xx_fwdt_entry_t261(struct scsi_qla_host *vha, struct qla27xx_fwdt_entry *ent, void *buf, ulong *len) { @@ -326,10 +332,10 @@ qla27xx_fwdt_entry_t261(struct scsi_qla_host *vha, "%s: wrpci [%lx]\n", __func__, *len); qla27xx_write_reg(reg, ent->t261.pci_offset, ent->t261.write_data, buf); - return false; + return qla27xx_next_entry(ent); } -static int +static struct qla27xx_fwdt_entry * qla27xx_fwdt_entry_t262(struct scsi_qla_host *vha, struct qla27xx_fwdt_entry *ent, void *buf, ulong *len) { @@ -362,6 +368,11 @@ qla27xx_fwdt_entry_t262(struct scsi_qla_host *vha, ent->t262.start_addr = start; ent->t262.end_addr = end; } + } else if (ent->t262.ram_area == T262_RAM_AREA_MISC) { + if (buf) { + ent->t262.start_addr = start; + ent->t262.end_addr = end; + } } else { ql_dbg(ql_dbg_misc, vha, 0xd022, "%s: unknown area %x\n", __func__, ent->t262.ram_area); @@ -384,10 +395,10 @@ qla27xx_fwdt_entry_t262(struct scsi_qla_host *vha, } *len += dwords * sizeof(uint32_t); done: - return false; + return qla27xx_next_entry(ent); } -static int +static struct qla27xx_fwdt_entry * qla27xx_fwdt_entry_t263(struct scsi_qla_host *vha, struct qla27xx_fwdt_entry *ent, void *buf, ulong *len) { @@ -450,10 +461,10 @@ qla27xx_fwdt_entry_t263(struct scsi_qla_host *vha, qla27xx_skip_entry(ent, buf); } - return false; + return qla27xx_next_entry(ent); } -static int +static struct qla27xx_fwdt_entry * qla27xx_fwdt_entry_t264(struct scsi_qla_host *vha, struct qla27xx_fwdt_entry *ent, void *buf, ulong *len) { @@ -478,10 +489,10 @@ qla27xx_fwdt_entry_t264(struct scsi_qla_host *vha, qla27xx_skip_entry(ent, buf); } - return false; + return qla27xx_next_entry(ent); } -static int +static struct qla27xx_fwdt_entry * qla27xx_fwdt_entry_t265(struct scsi_qla_host *vha, struct qla27xx_fwdt_entry *ent, void *buf, ulong *len) { @@ -492,10 +503,10 @@ qla27xx_fwdt_entry_t265(struct scsi_qla_host *vha, if (buf) qla24xx_pause_risc(reg, vha->hw); - return false; + return qla27xx_next_entry(ent); } -static int +static struct qla27xx_fwdt_entry * qla27xx_fwdt_entry_t266(struct scsi_qla_host *vha, struct qla27xx_fwdt_entry *ent, void *buf, ulong *len) { @@ -504,10 +515,10 @@ qla27xx_fwdt_entry_t266(struct scsi_qla_host *vha, if (buf) qla24xx_soft_reset(vha->hw); - return false; + return qla27xx_next_entry(ent); } -static int +static struct qla27xx_fwdt_entry * qla27xx_fwdt_entry_t267(struct scsi_qla_host *vha, struct qla27xx_fwdt_entry *ent, void *buf, ulong *len) { @@ -517,10 +528,10 @@ qla27xx_fwdt_entry_t267(struct scsi_qla_host *vha, "%s: dis intr [%lx]\n", __func__, *len); qla27xx_write_reg(reg, ent->t267.pci_offset, ent->t267.data, buf); - return false; + return qla27xx_next_entry(ent); } -static int +static struct qla27xx_fwdt_entry * qla27xx_fwdt_entry_t268(struct scsi_qla_host *vha, struct qla27xx_fwdt_entry *ent, void *buf, ulong *len) { @@ -587,10 +598,10 @@ qla27xx_fwdt_entry_t268(struct scsi_qla_host *vha, break; } - return false; + return qla27xx_next_entry(ent); } -static int +static struct qla27xx_fwdt_entry * qla27xx_fwdt_entry_t269(struct scsi_qla_host *vha, struct qla27xx_fwdt_entry *ent, void *buf, ulong *len) { @@ -604,10 +615,10 @@ qla27xx_fwdt_entry_t269(struct scsi_qla_host *vha, if (buf) ent->t269.scratch_size = 5 * sizeof(uint32_t); - return false; + return qla27xx_next_entry(ent); } -static int +static struct qla27xx_fwdt_entry * qla27xx_fwdt_entry_t270(struct scsi_qla_host *vha, struct qla27xx_fwdt_entry *ent, void *buf, ulong *len) { @@ -625,10 +636,10 @@ qla27xx_fwdt_entry_t270(struct scsi_qla_host *vha, addr += sizeof(uint32_t); } - return false; + return qla27xx_next_entry(ent); } -static int +static struct qla27xx_fwdt_entry * qla27xx_fwdt_entry_t271(struct scsi_qla_host *vha, struct qla27xx_fwdt_entry *ent, void *buf, ulong *len) { @@ -642,10 +653,10 @@ qla27xx_fwdt_entry_t271(struct scsi_qla_host *vha, qla27xx_write_reg(reg, 0xc4, data, buf); qla27xx_write_reg(reg, 0xc0, addr, buf); - return false; + return qla27xx_next_entry(ent); } -static int +static struct qla27xx_fwdt_entry * qla27xx_fwdt_entry_t272(struct scsi_qla_host *vha, struct qla27xx_fwdt_entry *ent, void *buf, ulong *len) { @@ -662,10 +673,10 @@ qla27xx_fwdt_entry_t272(struct scsi_qla_host *vha, } *len += dwords * sizeof(uint32_t); - return false; + return qla27xx_next_entry(ent); } -static int +static struct qla27xx_fwdt_entry * qla27xx_fwdt_entry_t273(struct scsi_qla_host *vha, struct qla27xx_fwdt_entry *ent, void *buf, ulong *len) { @@ -685,10 +696,10 @@ qla27xx_fwdt_entry_t273(struct scsi_qla_host *vha, addr += sizeof(uint32_t); } - return false; + return qla27xx_next_entry(ent); } -static int +static struct qla27xx_fwdt_entry * qla27xx_fwdt_entry_t274(struct scsi_qla_host *vha, struct qla27xx_fwdt_entry *ent, void *buf, ulong *len) { @@ -746,10 +757,10 @@ qla27xx_fwdt_entry_t274(struct scsi_qla_host *vha, qla27xx_skip_entry(ent, buf); } - return false; + return qla27xx_next_entry(ent); } -static int +static struct qla27xx_fwdt_entry * qla27xx_fwdt_entry_t275(struct scsi_qla_host *vha, struct qla27xx_fwdt_entry *ent, void *buf, ulong *len) { @@ -763,7 +774,7 @@ qla27xx_fwdt_entry_t275(struct scsi_qla_host *vha, qla27xx_skip_entry(ent, buf); goto done; } - if (offset + ent->t275.length > ent->hdr.entry_size) { + if (offset + ent->t275.length > ent->hdr.size) { ql_dbg(ql_dbg_misc, vha, 0xd030, "%s: buffer overflow\n", __func__); qla27xx_skip_entry(ent, buf); @@ -772,59 +783,103 @@ qla27xx_fwdt_entry_t275(struct scsi_qla_host *vha, qla27xx_insertbuf(ent->t275.buffer, ent->t275.length, buf, len); done: - return false; + return qla27xx_next_entry(ent); } -static int +static struct qla27xx_fwdt_entry * +qla27xx_fwdt_entry_t276(struct scsi_qla_host *vha, + struct qla27xx_fwdt_entry *ent, void *buf, ulong *len) +{ + uint type = vha->hw->pdev->device >> 4 & 0xf; + uint func = vha->hw->port_no & 0x3; + + ql_dbg(ql_dbg_misc + ql_dbg_verbose, vha, 0xd214, + "%s: cond [%lx]\n", __func__, *len); + + if (type != ent->t276.cond1 || func != ent->t276.cond2) { + ent = qla27xx_next_entry(ent); + qla27xx_skip_entry(ent, buf); + } + + return qla27xx_next_entry(ent); +} + +static struct qla27xx_fwdt_entry * +qla27xx_fwdt_entry_t277(struct scsi_qla_host *vha, + struct qla27xx_fwdt_entry *ent, void *buf, ulong *len) +{ + struct device_reg_24xx __iomem *reg = qla27xx_isp_reg(vha); + + ql_dbg(ql_dbg_misc + ql_dbg_verbose, vha, 0xd215, + "%s: rdpep [%lx]\n", __func__, *len); + qla27xx_insert32(ent->t277.wr_cmd_data, buf, len); + qla27xx_write_reg(reg, ent->t277.cmd_addr, ent->t277.wr_cmd_data, buf); + qla27xx_read_reg(reg, ent->t277.data_addr, buf, len); + + return qla27xx_next_entry(ent); +} + +static struct qla27xx_fwdt_entry * +qla27xx_fwdt_entry_t278(struct scsi_qla_host *vha, + struct qla27xx_fwdt_entry *ent, void *buf, ulong *len) +{ + struct device_reg_24xx __iomem *reg = qla27xx_isp_reg(vha); + + ql_dbg(ql_dbg_misc + ql_dbg_verbose, vha, 0xd216, + "%s: wrpep [%lx]\n", __func__, *len); + qla27xx_write_reg(reg, ent->t278.data_addr, ent->t278.wr_data, buf); + qla27xx_write_reg(reg, ent->t278.cmd_addr, ent->t278.wr_cmd_data, buf); + + return qla27xx_next_entry(ent); +} + +static struct qla27xx_fwdt_entry * qla27xx_fwdt_entry_other(struct scsi_qla_host *vha, struct qla27xx_fwdt_entry *ent, void *buf, ulong *len) { ql_dbg(ql_dbg_misc, vha, 0xd2ff, - "%s: type %x [%lx]\n", __func__, ent->hdr.entry_type, *len); + "%s: type %x [%lx]\n", __func__, ent->hdr.type, *len); qla27xx_skip_entry(ent, buf); - return false; + return qla27xx_next_entry(ent); } -struct qla27xx_fwdt_entry_call { +static struct { uint type; - int (*call)( - struct scsi_qla_host *, - struct qla27xx_fwdt_entry *, - void *, - ulong *); -}; - -static struct qla27xx_fwdt_entry_call ql27xx_fwdt_entry_call_list[] = { - { ENTRY_TYPE_NOP , qla27xx_fwdt_entry_t0 } , - { ENTRY_TYPE_TMP_END , qla27xx_fwdt_entry_t255 } , - { ENTRY_TYPE_RD_IOB_T1 , qla27xx_fwdt_entry_t256 } , - { ENTRY_TYPE_WR_IOB_T1 , qla27xx_fwdt_entry_t257 } , - { ENTRY_TYPE_RD_IOB_T2 , qla27xx_fwdt_entry_t258 } , - { ENTRY_TYPE_WR_IOB_T2 , qla27xx_fwdt_entry_t259 } , - { ENTRY_TYPE_RD_PCI , qla27xx_fwdt_entry_t260 } , - { ENTRY_TYPE_WR_PCI , qla27xx_fwdt_entry_t261 } , - { ENTRY_TYPE_RD_RAM , qla27xx_fwdt_entry_t262 } , - { ENTRY_TYPE_GET_QUEUE , qla27xx_fwdt_entry_t263 } , - { ENTRY_TYPE_GET_FCE , qla27xx_fwdt_entry_t264 } , - { ENTRY_TYPE_PSE_RISC , qla27xx_fwdt_entry_t265 } , - { ENTRY_TYPE_RST_RISC , qla27xx_fwdt_entry_t266 } , - { ENTRY_TYPE_DIS_INTR , qla27xx_fwdt_entry_t267 } , - { ENTRY_TYPE_GET_HBUF , qla27xx_fwdt_entry_t268 } , - { ENTRY_TYPE_SCRATCH , qla27xx_fwdt_entry_t269 } , - { ENTRY_TYPE_RDREMREG , qla27xx_fwdt_entry_t270 } , - { ENTRY_TYPE_WRREMREG , qla27xx_fwdt_entry_t271 } , - { ENTRY_TYPE_RDREMRAM , qla27xx_fwdt_entry_t272 } , - { ENTRY_TYPE_PCICFG , qla27xx_fwdt_entry_t273 } , - { ENTRY_TYPE_GET_SHADOW , qla27xx_fwdt_entry_t274 } , - { ENTRY_TYPE_WRITE_BUF , qla27xx_fwdt_entry_t275 } , - { -1 , qla27xx_fwdt_entry_other } + typeof(qla27xx_fwdt_entry_other)(*call); +} qla27xx_fwdt_entry_call[] = { + { ENTRY_TYPE_NOP, qla27xx_fwdt_entry_t0 }, + { ENTRY_TYPE_TMP_END, qla27xx_fwdt_entry_t255 }, + { ENTRY_TYPE_RD_IOB_T1, qla27xx_fwdt_entry_t256 }, + { ENTRY_TYPE_WR_IOB_T1, qla27xx_fwdt_entry_t257 }, + { ENTRY_TYPE_RD_IOB_T2, qla27xx_fwdt_entry_t258 }, + { ENTRY_TYPE_WR_IOB_T2, qla27xx_fwdt_entry_t259 }, + { ENTRY_TYPE_RD_PCI, qla27xx_fwdt_entry_t260 }, + { ENTRY_TYPE_WR_PCI, qla27xx_fwdt_entry_t261 }, + { ENTRY_TYPE_RD_RAM, qla27xx_fwdt_entry_t262 }, + { ENTRY_TYPE_GET_QUEUE, qla27xx_fwdt_entry_t263 }, + { ENTRY_TYPE_GET_FCE, qla27xx_fwdt_entry_t264 }, + { ENTRY_TYPE_PSE_RISC, qla27xx_fwdt_entry_t265 }, + { ENTRY_TYPE_RST_RISC, qla27xx_fwdt_entry_t266 }, + { ENTRY_TYPE_DIS_INTR, qla27xx_fwdt_entry_t267 }, + { ENTRY_TYPE_GET_HBUF, qla27xx_fwdt_entry_t268 }, + { ENTRY_TYPE_SCRATCH, qla27xx_fwdt_entry_t269 }, + { ENTRY_TYPE_RDREMREG, qla27xx_fwdt_entry_t270 }, + { ENTRY_TYPE_WRREMREG, qla27xx_fwdt_entry_t271 }, + { ENTRY_TYPE_RDREMRAM, qla27xx_fwdt_entry_t272 }, + { ENTRY_TYPE_PCICFG, qla27xx_fwdt_entry_t273 }, + { ENTRY_TYPE_GET_SHADOW, qla27xx_fwdt_entry_t274 }, + { ENTRY_TYPE_WRITE_BUF, qla27xx_fwdt_entry_t275 }, + { ENTRY_TYPE_CONDITIONAL, qla27xx_fwdt_entry_t276 }, + { ENTRY_TYPE_RDPEPREG, qla27xx_fwdt_entry_t277 }, + { ENTRY_TYPE_WRPEPREG, qla27xx_fwdt_entry_t278 }, + { -1, qla27xx_fwdt_entry_other } }; -static inline int (*qla27xx_find_entry(uint type)) - (struct scsi_qla_host *, struct qla27xx_fwdt_entry *, void *, ulong *) +static inline +typeof(qla27xx_fwdt_entry_call->call)(qla27xx_find_entry(uint type)) { - struct qla27xx_fwdt_entry_call *list = ql27xx_fwdt_entry_call_list; + typeof(*qla27xx_fwdt_entry_call) *list = qla27xx_fwdt_entry_call; while (list->type < type) list++; @@ -834,14 +889,6 @@ static inline int (*qla27xx_find_entry(uint type)) return qla27xx_fwdt_entry_other; } -static inline void * -qla27xx_next_entry(void *p) -{ - struct qla27xx_fwdt_entry *ent = p; - - return p + ent->hdr.entry_size; -} - static void qla27xx_walk_template(struct scsi_qla_host *vha, struct qla27xx_fwdt_template *tmp, void *buf, ulong *len) @@ -852,18 +899,16 @@ qla27xx_walk_template(struct scsi_qla_host *vha, ql_dbg(ql_dbg_misc, vha, 0xd01a, "%s: entry count %lx\n", __func__, count); while (count--) { - if (buf && *len >= vha->hw->fw_dump_len) + ent = qla27xx_find_entry(ent->hdr.type)(vha, ent, buf, len); + if (!ent) break; - if (qla27xx_find_entry(ent->hdr.entry_type)(vha, ent, buf, len)) - break; - ent = qla27xx_next_entry(ent); } if (count) ql_dbg(ql_dbg_misc, vha, 0xd018, "%s: entry residual count (%lx)\n", __func__, count); - if (ent->hdr.entry_type != ENTRY_TYPE_TMP_END) + if (ent) ql_dbg(ql_dbg_misc, vha, 0xd019, "%s: missing end entry (%lx)\n", __func__, count); diff --git a/drivers/scsi/qla2xxx/qla_tmpl.h b/drivers/scsi/qla2xxx/qla_tmpl.h index 141c1c5e73f4..5c2c2a8a19c4 100644 --- a/drivers/scsi/qla2xxx/qla_tmpl.h +++ b/drivers/scsi/qla2xxx/qla_tmpl.h @@ -54,6 +54,9 @@ struct __packed qla27xx_fwdt_template { #define ENTRY_TYPE_PCICFG 273 #define ENTRY_TYPE_GET_SHADOW 274 #define ENTRY_TYPE_WRITE_BUF 275 +#define ENTRY_TYPE_CONDITIONAL 276 +#define ENTRY_TYPE_RDPEPREG 277 +#define ENTRY_TYPE_WRPEPREG 278 #define CAPTURE_FLAG_PHYS_ONLY BIT_0 #define CAPTURE_FLAG_PHYS_VIRT BIT_1 @@ -62,8 +65,8 @@ struct __packed qla27xx_fwdt_template { struct __packed qla27xx_fwdt_entry { struct __packed { - uint32_t entry_type; - uint32_t entry_size; + uint32_t type; + uint32_t size; uint32_t reserved_1; uint8_t capture_flags; @@ -199,6 +202,24 @@ struct __packed qla27xx_fwdt_entry { uint32_t length; uint8_t buffer[]; } t275; + + struct __packed { + uint32_t cond1; + uint32_t cond2; + } t276; + + struct __packed { + uint32_t cmd_addr; + uint32_t wr_cmd_data; + uint32_t data_addr; + } t277; + + struct __packed { + uint32_t cmd_addr; + uint32_t wr_cmd_data; + uint32_t data_addr; + uint32_t wr_data; + } t278; }; }; @@ -206,6 +227,7 @@ struct __packed qla27xx_fwdt_entry { #define T262_RAM_AREA_EXTERNAL_RAM 2 #define T262_RAM_AREA_SHARED_RAM 3 #define T262_RAM_AREA_DDR_RAM 4 +#define T262_RAM_AREA_MISC 5 #define T263_QUEUE_TYPE_REQ 1 #define T263_QUEUE_TYPE_RSP 2 diff --git a/drivers/scsi/qla2xxx/qla_version.h b/drivers/scsi/qla2xxx/qla_version.h index ca7945cb959b..0690dac24081 100644 --- a/drivers/scsi/qla2xxx/qla_version.h +++ b/drivers/scsi/qla2xxx/qla_version.h @@ -7,7 +7,7 @@ /* * Driver version */ -#define QLA2XXX_VERSION "10.00.00.12-k" +#define QLA2XXX_VERSION "10.00.00.14-k" #define QLA_DRIVER_MAJOR_VER 10 #define QLA_DRIVER_MINOR_VER 0 diff --git a/drivers/scsi/qla2xxx/tcm_qla2xxx.c b/drivers/scsi/qla2xxx/tcm_qla2xxx.c index 283e6b80abb5..8a3075d17c63 100644 --- a/drivers/scsi/qla2xxx/tcm_qla2xxx.c +++ b/drivers/scsi/qla2xxx/tcm_qla2xxx.c @@ -420,26 +420,6 @@ static int tcm_qla2xxx_write_pending(struct se_cmd *se_cmd) return qlt_rdy_to_xfer(cmd); } -static int tcm_qla2xxx_write_pending_status(struct se_cmd *se_cmd) -{ - unsigned long flags; - /* - * Check for WRITE_PENDING status to determine if we need to wait for - * CTIO aborts to be posted via hardware in tcm_qla2xxx_handle_data(). - */ - spin_lock_irqsave(&se_cmd->t_state_lock, flags); - if (se_cmd->t_state == TRANSPORT_WRITE_PENDING || - se_cmd->t_state == TRANSPORT_COMPLETE_QF_WP) { - spin_unlock_irqrestore(&se_cmd->t_state_lock, flags); - wait_for_completion_timeout(&se_cmd->t_transport_stop_comp, - 50); - return 0; - } - spin_unlock_irqrestore(&se_cmd->t_state_lock, flags); - - return 0; -} - static void tcm_qla2xxx_set_default_node_attrs(struct se_node_acl *nacl) { return; @@ -537,15 +517,6 @@ static void tcm_qla2xxx_handle_data_work(struct work_struct *work) cmd->qpair->tgt_counters.qla_core_ret_ctio++; if (!cmd->write_data_transferred) { - /* - * Check if se_cmd has already been aborted via LUN_RESET, and - * waiting upon completion in tcm_qla2xxx_write_pending_status() - */ - if (cmd->se_cmd.transport_state & CMD_T_ABORTED) { - complete(&cmd->se_cmd.t_transport_stop_comp); - return; - } - switch (cmd->dif_err_code) { case DIF_ERR_GRD: cmd->se_cmd.pi_err = @@ -1902,7 +1873,6 @@ static const struct target_core_fabric_ops tcm_qla2xxx_ops = { .sess_get_index = tcm_qla2xxx_sess_get_index, .sess_get_initiator_sid = NULL, .write_pending = tcm_qla2xxx_write_pending, - .write_pending_status = tcm_qla2xxx_write_pending_status, .set_default_node_attributes = tcm_qla2xxx_set_default_node_attrs, .get_cmd_state = tcm_qla2xxx_get_cmd_state, .queue_data_in = tcm_qla2xxx_queue_data_in, @@ -1943,7 +1913,6 @@ static const struct target_core_fabric_ops tcm_qla2xxx_npiv_ops = { .sess_get_index = tcm_qla2xxx_sess_get_index, .sess_get_initiator_sid = NULL, .write_pending = tcm_qla2xxx_write_pending, - .write_pending_status = tcm_qla2xxx_write_pending_status, .set_default_node_attributes = tcm_qla2xxx_set_default_node_attrs, .get_cmd_state = tcm_qla2xxx_get_cmd_state, .queue_data_in = tcm_qla2xxx_queue_data_in, diff --git a/drivers/scsi/qla4xxx/ql4_os.c b/drivers/scsi/qla4xxx/ql4_os.c index a77bfb224248..16a18d5d856f 100644 --- a/drivers/scsi/qla4xxx/ql4_os.c +++ b/drivers/scsi/qla4xxx/ql4_os.c @@ -2875,7 +2875,7 @@ static int qla4xxx_session_get_param(struct iscsi_cls_session *cls_sess, chap_tbl.secret_len); } } - /* allow fall-through */ + /* fall through */ default: return iscsi_session_get_param(cls_sess, param, buf); } diff --git a/drivers/scsi/qlogicpti.c b/drivers/scsi/qlogicpti.c index e35ce762d454..0e22512bd3e4 100644 --- a/drivers/scsi/qlogicpti.c +++ b/drivers/scsi/qlogicpti.c @@ -1314,8 +1314,7 @@ static int qpti_sbus_probe(struct platform_device *op) qpti->qhost = host; qpti->op = op; qpti->qpti_id = nqptis; - strcpy(qpti->prom_name, op->dev.of_node->name); - qpti->is_pti = strcmp(qpti->prom_name, "QLGC,isp"); + qpti->is_pti = !of_node_name_eq(op->dev.of_node, "QLGC,isp"); if (qpti_map_regs(qpti) < 0) goto fail_unlink; diff --git a/drivers/scsi/qlogicpti.h b/drivers/scsi/qlogicpti.h index 884ad72ade57..2b6374e08a7d 100644 --- a/drivers/scsi/qlogicpti.h +++ b/drivers/scsi/qlogicpti.h @@ -364,7 +364,6 @@ struct qlogicpti { int qpti_id; int scsi_id; int prom_node; - char prom_name[64]; int irq; char differential, ultra, clock; unsigned char bursts; @@ -379,7 +378,7 @@ struct qlogicpti { #define SREG_IMASK 0x0c /* Interrupt level */ #define SREG_SPMASK 0x03 /* Mask for switch pack */ unsigned char swsreg; - unsigned int + unsigned int gotirq : 1, /* this instance got an irq */ is_pti : 1; /* Non-zero if this is a PTI board. */ }; diff --git a/drivers/scsi/scsi.c b/drivers/scsi/scsi.c index 7675ff0ca2ea..99a7b9f520ae 100644 --- a/drivers/scsi/scsi.c +++ b/drivers/scsi/scsi.c @@ -175,22 +175,6 @@ void scsi_log_completion(struct scsi_cmnd *cmd, int disposition) #endif /** - * scsi_cmd_get_serial - Assign a serial number to a command - * @host: the scsi host - * @cmd: command to assign serial number to - * - * Description: a serial number identifies a request for error recovery - * and debugging purposes. Protected by the Host_Lock of host. - */ -void scsi_cmd_get_serial(struct Scsi_Host *host, struct scsi_cmnd *cmd) -{ - cmd->serial_number = host->cmd_serial_number++; - if (cmd->serial_number == 0) - cmd->serial_number = host->cmd_serial_number++; -} -EXPORT_SYMBOL(scsi_cmd_get_serial); - -/** * scsi_finish_command - cleanup and pass command back to upper layer * @cmd: the command * diff --git a/drivers/scsi/scsi_debug.c b/drivers/scsi/scsi_debug.c index e27f4df24021..2740a90501a0 100644 --- a/drivers/scsi/scsi_debug.c +++ b/drivers/scsi/scsi_debug.c @@ -76,6 +76,7 @@ static const char *sdebug_version_date = "20190125"; #define LBA_OUT_OF_RANGE 0x21 #define INVALID_FIELD_IN_CDB 0x24 #define INVALID_FIELD_IN_PARAM_LIST 0x26 +#define WRITE_PROTECTED 0x27 #define UA_RESET_ASC 0x29 #define UA_CHANGED_ASC 0x2a #define TARGET_CHANGED_ASC 0x3f @@ -351,12 +352,11 @@ enum sdeb_opcode_index { SDEB_I_ATA_PT = 22, /* 12, 16 */ SDEB_I_SEND_DIAG = 23, SDEB_I_UNMAP = 24, - SDEB_I_XDWRITEREAD = 25, /* 10 only */ - SDEB_I_WRITE_BUFFER = 26, - SDEB_I_WRITE_SAME = 27, /* 10, 16 */ - SDEB_I_SYNC_CACHE = 28, /* 10, 16 */ - SDEB_I_COMP_WRITE = 29, - SDEB_I_LAST_ELEMENT = 30, /* keep this last (previous + 1) */ + SDEB_I_WRITE_BUFFER = 25, + SDEB_I_WRITE_SAME = 26, /* 10, 16 */ + SDEB_I_SYNC_CACHE = 27, /* 10, 16 */ + SDEB_I_COMP_WRITE = 28, + SDEB_I_LAST_ELEMENT = 29, /* keep this last (previous + 1) */ }; @@ -377,7 +377,7 @@ static const unsigned char opcode_ind_arr[256] = { /* 0x40; 0x40->0x5f: 10 byte cdbs */ 0, SDEB_I_WRITE_SAME, SDEB_I_UNMAP, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, SDEB_I_LOG_SENSE, 0, 0, - 0, 0, 0, SDEB_I_XDWRITEREAD, 0, SDEB_I_MODE_SELECT, SDEB_I_RESERVE, + 0, 0, 0, 0, 0, SDEB_I_MODE_SELECT, SDEB_I_RESERVE, SDEB_I_RELEASE, 0, 0, SDEB_I_MODE_SENSE, 0, 0, 0, 0, 0, /* 0x60; 0x60->0x7d are reserved, 0x7e is "extended cdb" */ @@ -430,7 +430,6 @@ static int resp_rsup_opcodes(struct scsi_cmnd *, struct sdebug_dev_info *); static int resp_rsup_tmfs(struct scsi_cmnd *, struct sdebug_dev_info *); static int resp_write_same_10(struct scsi_cmnd *, struct sdebug_dev_info *); static int resp_write_same_16(struct scsi_cmnd *, struct sdebug_dev_info *); -static int resp_xdwriteread_10(struct scsi_cmnd *, struct sdebug_dev_info *); static int resp_comp_write(struct scsi_cmnd *, struct sdebug_dev_info *); static int resp_write_buffer(struct scsi_cmnd *, struct sdebug_dev_info *); static int resp_sync_cache(struct scsi_cmnd *, struct sdebug_dev_info *); @@ -600,9 +599,6 @@ static const struct opcode_info_t opcode_info_arr[SDEB_I_LAST_ELEMENT + 1] = { {0, 0x42, 0, F_D_OUT | FF_MEDIA_IO, resp_unmap, NULL, /* UNMAP */ {10, 0x1, 0, 0, 0, 0, 0x3f, 0xff, 0xff, 0xc7, 0, 0, 0, 0, 0, 0} }, /* 25 */ - {0, 0x53, 0, F_D_IN | F_D_OUT | FF_MEDIA_IO, resp_xdwriteread_10, - NULL, {10, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3f, 0xff, 0xff, 0xc7, - 0, 0, 0, 0, 0, 0} }, /* XDWRITEREAD(10) */ {0, 0x3b, 0, F_D_OUT_MAYBE, resp_write_buffer, NULL, {10, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xc7, 0, 0, 0, 0, 0, 0} }, /* WRITE_BUFFER */ @@ -618,7 +614,7 @@ static const struct opcode_info_t opcode_info_arr[SDEB_I_LAST_ELEMENT + 1] = { {16, 0xf8, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0, 0, 0, 0xff, 0x3f, 0xc7} }, /* COMPARE AND WRITE */ -/* 30 */ +/* 29 */ {0xff, 0, 0, 0, NULL, NULL, /* terminating element */ {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} }, }; @@ -673,6 +669,7 @@ static bool sdebug_verbose; static bool have_dif_prot; static bool write_since_sync; static bool sdebug_statistics = DEF_STATISTICS; +static bool sdebug_wp; static unsigned int sdebug_store_sectors; static sector_t sdebug_capacity; /* in sectors */ @@ -836,7 +833,8 @@ static void mk_sense_invalid_opcode(struct scsi_cmnd *scp) mk_sense_buffer(scp, ILLEGAL_REQUEST, INVALID_OPCODE, 0); } -static int scsi_debug_ioctl(struct scsi_device *dev, int cmd, void __user *arg) +static int scsi_debug_ioctl(struct scsi_device *dev, unsigned int cmd, + void __user *arg) { if (sdebug_verbose) { if (0x1261 == cmd) @@ -1010,16 +1008,16 @@ static int fill_from_dev_buffer(struct scsi_cmnd *scp, unsigned char *arr, int arr_len) { int act_len; - struct scsi_data_buffer *sdb = scsi_in(scp); + struct scsi_data_buffer *sdb = &scp->sdb; if (!sdb->length) return 0; - if (!(scsi_bidi_cmnd(scp) || scp->sc_data_direction == DMA_FROM_DEVICE)) + if (scp->sc_data_direction != DMA_FROM_DEVICE) return DID_ERROR << 16; act_len = sg_copy_from_buffer(sdb->table.sgl, sdb->table.nents, arr, arr_len); - sdb->resid = scsi_bufflen(scp) - act_len; + scsi_set_resid(scp, scsi_bufflen(scp) - act_len); return 0; } @@ -1033,20 +1031,21 @@ static int p_fill_from_dev_buffer(struct scsi_cmnd *scp, const void *arr, int arr_len, unsigned int off_dst) { int act_len, n; - struct scsi_data_buffer *sdb = scsi_in(scp); + struct scsi_data_buffer *sdb = &scp->sdb; off_t skip = off_dst; if (sdb->length <= off_dst) return 0; - if (!(scsi_bidi_cmnd(scp) || scp->sc_data_direction == DMA_FROM_DEVICE)) + if (scp->sc_data_direction != DMA_FROM_DEVICE) return DID_ERROR << 16; act_len = sg_pcopy_from_buffer(sdb->table.sgl, sdb->table.nents, arr, arr_len, skip); pr_debug("%s: off_dst=%u, scsi_bufflen=%u, act_len=%u, resid=%d\n", - __func__, off_dst, scsi_bufflen(scp), act_len, sdb->resid); + __func__, off_dst, scsi_bufflen(scp), act_len, + scsi_get_resid(scp)); n = (int)scsi_bufflen(scp) - ((int)off_dst + act_len); - sdb->resid = min(sdb->resid, n); + scsi_set_resid(scp, min(scsi_get_resid(scp), n)); return 0; } @@ -1058,7 +1057,7 @@ static int fetch_to_dev_buffer(struct scsi_cmnd *scp, unsigned char *arr, { if (!scsi_bufflen(scp)) return 0; - if (!(scsi_bidi_cmnd(scp) || scp->sc_data_direction == DMA_TO_DEVICE)) + if (scp->sc_data_direction != DMA_TO_DEVICE) return -1; return scsi_sg_copy_to_buffer(scp, arr, arr_len); @@ -2146,9 +2145,11 @@ static int resp_mode_sense(struct scsi_cmnd *scp, target_dev_id = ((devip->sdbg_host->shost->host_no + 1) * 2000) + (devip->target * 1000) - 3; /* for disks set DPOFUA bit and clear write protect (WP) bit */ - if (is_disk) + if (is_disk) { dev_spec = 0x10; /* =0x90 if WP=1 implies read-only */ - else + if (sdebug_wp) + dev_spec |= 0x80; + } else dev_spec = 0x0; if (msense_6) { arr[2] = dev_spec; @@ -2331,6 +2332,10 @@ static int resp_mode_select(struct scsi_cmnd *scp, if (ctrl_m_pg[1] == arr[off + 1]) { memcpy(ctrl_m_pg + 2, arr + off + 2, sizeof(ctrl_m_pg) - 2); + if (ctrl_m_pg[4] & 0x8) + sdebug_wp = true; + else + sdebug_wp = false; sdebug_dsense = !!(ctrl_m_pg[2] & 0x4); goto set_mode_changed_ua; } @@ -2455,8 +2460,8 @@ static int resp_log_sense(struct scsi_cmnd *scp, min(len, SDEBUG_MAX_INQ_ARR_SZ)); } -static int check_device_access_params(struct scsi_cmnd *scp, - unsigned long long lba, unsigned int num) +static inline int check_device_access_params(struct scsi_cmnd *scp, + unsigned long long lba, unsigned int num, bool write) { if (lba + num > sdebug_capacity) { mk_sense_buffer(scp, ILLEGAL_REQUEST, LBA_OUT_OF_RANGE, 0); @@ -2468,6 +2473,10 @@ static int check_device_access_params(struct scsi_cmnd *scp, mk_sense_buffer(scp, ILLEGAL_REQUEST, INVALID_FIELD_IN_CDB, 0); return check_condition_result; } + if (write && unlikely(sdebug_wp)) { + mk_sense_buffer(scp, DATA_PROTECT, WRITE_PROTECTED, 0x2); + return check_condition_result; + } return 0; } @@ -2477,21 +2486,19 @@ static int do_device_access(struct scsi_cmnd *scmd, u32 sg_skip, u64 lba, { int ret; u64 block, rest = 0; - struct scsi_data_buffer *sdb; + struct scsi_data_buffer *sdb = &scmd->sdb; enum dma_data_direction dir; if (do_write) { - sdb = scsi_out(scmd); dir = DMA_TO_DEVICE; write_since_sync = true; } else { - sdb = scsi_in(scmd); dir = DMA_FROM_DEVICE; } if (!sdb->length) return 0; - if (!(scsi_bidi_cmnd(scmd) || scmd->sc_data_direction == dir)) + if (scmd->sc_data_direction != dir) return -1; block = do_div(lba, sdebug_store_sectors); @@ -2728,18 +2735,9 @@ static int resp_read_dt0(struct scsi_cmnd *scp, struct sdebug_dev_info *devip) } else sqcp = NULL; - /* inline check_device_access_params() */ - if (unlikely(lba + num > sdebug_capacity)) { - mk_sense_buffer(scp, ILLEGAL_REQUEST, LBA_OUT_OF_RANGE, 0); - return check_condition_result; - } - /* transfer length excessive (tie in to block limits VPD page) */ - if (unlikely(num > sdebug_store_sectors)) { - /* needs work to find which cdb byte 'num' comes from */ - mk_sense_buffer(scp, ILLEGAL_REQUEST, INVALID_FIELD_IN_CDB, 0); - return check_condition_result; - } - + ret = check_device_access_params(scp, lba, num, false); + if (ret) + return ret; if (unlikely((SDEBUG_OPT_MEDIUM_ERR & sdebug_opts) && (lba <= (sdebug_medium_error_start + sdebug_medium_error_count - 1)) && ((lba + num) > sdebug_medium_error_start))) { @@ -2774,7 +2772,7 @@ static int resp_read_dt0(struct scsi_cmnd *scp, struct sdebug_dev_info *devip) if (unlikely(ret == -1)) return DID_ERROR << 16; - scsi_in(scp)->resid = scsi_bufflen(scp) - ret; + scsi_set_resid(scp, scsi_bufflen(scp) - ret); if (unlikely(sqcp)) { if (sqcp->inj_recovered) { @@ -3031,19 +3029,9 @@ static int resp_write_dt0(struct scsi_cmnd *scp, struct sdebug_dev_info *devip) sdev_printk(KERN_ERR, scp->device, "Unprotected WR " "to DIF device\n"); } - - /* inline check_device_access_params() */ - if (unlikely(lba + num > sdebug_capacity)) { - mk_sense_buffer(scp, ILLEGAL_REQUEST, LBA_OUT_OF_RANGE, 0); - return check_condition_result; - } - /* transfer length excessive (tie in to block limits VPD page) */ - if (unlikely(num > sdebug_store_sectors)) { - /* needs work to find which cdb byte 'num' comes from */ - mk_sense_buffer(scp, ILLEGAL_REQUEST, INVALID_FIELD_IN_CDB, 0); - return check_condition_result; - } - + ret = check_device_access_params(scp, lba, num, true); + if (ret) + return ret; write_lock_irqsave(&atomic_rw, iflags); /* DIX + T10 DIF */ @@ -3182,7 +3170,7 @@ static int resp_write_scat(struct scsi_cmnd *scp, my_name, __func__, k, lba, num, sg_off); if (num == 0) continue; - ret = check_device_access_params(scp, lba, num); + ret = check_device_access_params(scp, lba, num, true); if (ret) goto err_out_unlock; num_by = num * lb_size; @@ -3268,7 +3256,7 @@ static int resp_write_same(struct scsi_cmnd *scp, u64 lba, u32 num, u64 block, lbaa; u8 *fs1p; - ret = check_device_access_params(scp, lba, num); + ret = check_device_access_params(scp, lba, num, true); if (ret) return ret; @@ -3440,18 +3428,9 @@ static int resp_comp_write(struct scsi_cmnd *scp, (cmd[1] & 0xe0) == 0) sdev_printk(KERN_ERR, scp->device, "Unprotected WR " "to DIF device\n"); - - /* inline check_device_access_params() */ - if (lba + num > sdebug_capacity) { - mk_sense_buffer(scp, ILLEGAL_REQUEST, LBA_OUT_OF_RANGE, 0); - return check_condition_result; - } - /* transfer length excessive (tie in to block limits VPD page) */ - if (num > sdebug_store_sectors) { - /* needs work to find which cdb byte 'num' comes from */ - mk_sense_buffer(scp, ILLEGAL_REQUEST, INVALID_FIELD_IN_CDB, 0); - return check_condition_result; - } + ret = check_device_access_params(scp, lba, num, false); + if (ret) + return ret; dnum = 2 * num; arr = kcalloc(lb_size, dnum, GFP_ATOMIC); if (NULL == arr) { @@ -3534,7 +3513,7 @@ static int resp_unmap(struct scsi_cmnd *scp, struct sdebug_dev_info *devip) unsigned long long lba = get_unaligned_be64(&desc[i].lba); unsigned int num = get_unaligned_be32(&desc[i].blocks); - ret = check_device_access_params(scp, lba, num); + ret = check_device_access_params(scp, lba, num, true); if (ret) goto out; @@ -3567,7 +3546,7 @@ static int resp_get_lba_status(struct scsi_cmnd *scp, if (alloc_len < 24) return 0; - ret = check_device_access_params(scp, lba, 1); + ret = check_device_access_params(scp, lba, 1, false); if (ret) return ret; @@ -3719,68 +3698,6 @@ static int resp_report_luns(struct scsi_cmnd *scp, return res; } -static int resp_xdwriteread(struct scsi_cmnd *scp, unsigned long long lba, - unsigned int num, struct sdebug_dev_info *devip) -{ - int j; - unsigned char *kaddr, *buf; - unsigned int offset; - struct scsi_data_buffer *sdb = scsi_in(scp); - struct sg_mapping_iter miter; - - /* better not to use temporary buffer. */ - buf = kzalloc(scsi_bufflen(scp), GFP_ATOMIC); - if (!buf) { - mk_sense_buffer(scp, ILLEGAL_REQUEST, INSUFF_RES_ASC, - INSUFF_RES_ASCQ); - return check_condition_result; - } - - scsi_sg_copy_to_buffer(scp, buf, scsi_bufflen(scp)); - - offset = 0; - sg_miter_start(&miter, sdb->table.sgl, sdb->table.nents, - SG_MITER_ATOMIC | SG_MITER_TO_SG); - - while (sg_miter_next(&miter)) { - kaddr = miter.addr; - for (j = 0; j < miter.length; j++) - *(kaddr + j) ^= *(buf + offset + j); - - offset += miter.length; - } - sg_miter_stop(&miter); - kfree(buf); - - return 0; -} - -static int resp_xdwriteread_10(struct scsi_cmnd *scp, - struct sdebug_dev_info *devip) -{ - u8 *cmd = scp->cmnd; - u64 lba; - u32 num; - int errsts; - - if (!scsi_bidi_cmnd(scp)) { - mk_sense_buffer(scp, ILLEGAL_REQUEST, INSUFF_RES_ASC, - INSUFF_RES_ASCQ); - return check_condition_result; - } - errsts = resp_read_dt0(scp, devip); - if (errsts) - return errsts; - if (!(cmd[1] & 0x4)) { /* DISABLE_WRITE is not set */ - errsts = resp_write_dt0(scp, devip); - if (errsts) - return errsts; - } - lba = get_unaligned_be32(cmd + 2); - num = get_unaligned_be16(cmd + 7); - return resp_xdwriteread(scp, lba, num, devip); -} - static struct sdebug_queue *get_queue(struct scsi_cmnd *cmnd) { u32 tag = blk_mq_unique_tag(cmnd->request); @@ -3954,7 +3871,6 @@ static int scsi_debug_slave_alloc(struct scsi_device *sdp) if (sdebug_verbose) pr_info("slave_alloc <%u %u %u %llu>\n", sdp->host->host_no, sdp->channel, sdp->id, sdp->lun); - blk_queue_flag_set(QUEUE_FLAG_BIDI, sdp->request_queue); return 0; } @@ -4554,6 +4470,7 @@ module_param_named(virtual_gb, sdebug_virtual_gb, int, S_IRUGO | S_IWUSR); module_param_named(uuid_ctl, sdebug_uuid_ctl, int, S_IRUGO); module_param_named(vpd_use_hostno, sdebug_vpd_use_hostno, int, S_IRUGO | S_IWUSR); +module_param_named(wp, sdebug_wp, bool, S_IRUGO | S_IWUSR); module_param_named(write_same_length, sdebug_write_same_length, int, S_IRUGO | S_IWUSR); @@ -4613,6 +4530,7 @@ MODULE_PARM_DESC(uuid_ctl, "1->use uuid for lu name, 0->don't, 2->all use same (def=0)"); MODULE_PARM_DESC(virtual_gb, "virtual gigabyte (GiB) size (def=0 -> use dev_size_mb)"); MODULE_PARM_DESC(vpd_use_hostno, "0 -> dev ids ignore hostno (def=1 -> unique dev ids)"); +MODULE_PARM_DESC(wp, "Write Protect (def=0)"); MODULE_PARM_DESC(write_same_length, "Maximum blocks per WRITE SAME cmd (def=0xffff)"); #define SDEBUG_INFO_LEN 256 diff --git a/drivers/scsi/scsi_error.c b/drivers/scsi/scsi_error.c index 16eef068e9e9..1b8378f36139 100644 --- a/drivers/scsi/scsi_error.c +++ b/drivers/scsi/scsi_error.c @@ -965,7 +965,6 @@ void scsi_eh_prep_cmnd(struct scsi_cmnd *scmd, struct scsi_eh_save *ses, ses->cmnd = scmd->cmnd; ses->data_direction = scmd->sc_data_direction; ses->sdb = scmd->sdb; - ses->next_rq = scmd->request->next_rq; ses->result = scmd->result; ses->underflow = scmd->underflow; ses->prot_op = scmd->prot_op; @@ -976,7 +975,6 @@ void scsi_eh_prep_cmnd(struct scsi_cmnd *scmd, struct scsi_eh_save *ses, scmd->cmnd = ses->eh_cmnd; memset(scmd->cmnd, 0, BLK_MAX_CDB); memset(&scmd->sdb, 0, sizeof(scmd->sdb)); - scmd->request->next_rq = NULL; scmd->result = 0; if (sense_bytes) { @@ -1029,7 +1027,6 @@ void scsi_eh_restore_cmnd(struct scsi_cmnd* scmd, struct scsi_eh_save *ses) scmd->cmnd = ses->cmnd; scmd->sc_data_direction = ses->data_direction; scmd->sdb = ses->sdb; - scmd->request->next_rq = ses->next_rq; scmd->result = ses->result; scmd->underflow = ses->underflow; scmd->prot_op = ses->prot_op; diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c index ca5fd3ae81f8..20189675677a 100644 --- a/drivers/scsi/scsi_lib.c +++ b/drivers/scsi/scsi_lib.c @@ -316,7 +316,6 @@ EXPORT_SYMBOL(__scsi_execute); */ static void scsi_init_cmd_errh(struct scsi_cmnd *cmd) { - cmd->serial_number = 0; scsi_set_resid(cmd, 0); memset(cmd->sense_buffer, 0, SCSI_SENSE_BUFFERSIZE); if (cmd->cmd_len == 0) @@ -556,15 +555,8 @@ static void scsi_uninit_cmd(struct scsi_cmnd *cmd) static void scsi_mq_free_sgtables(struct scsi_cmnd *cmd) { - struct scsi_data_buffer *sdb; - if (cmd->sdb.table.nents) sg_free_table_chained(&cmd->sdb.table, true); - if (cmd->request->next_rq) { - sdb = cmd->request->next_rq->special; - if (sdb) - sg_free_table_chained(&sdb->table, true); - } if (scsi_prot_sg_count(cmd)) sg_free_table_chained(&cmd->prot_sdb->table, true); } @@ -578,7 +570,7 @@ static void scsi_mq_uninit_cmd(struct scsi_cmnd *cmd) /* Returns false when no more bytes to process, true if there are more */ static bool scsi_end_request(struct request *req, blk_status_t error, - unsigned int bytes, unsigned int bidi_bytes) + unsigned int bytes) { struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req); struct scsi_device *sdev = cmd->device; @@ -587,11 +579,6 @@ static bool scsi_end_request(struct request *req, blk_status_t error, if (blk_update_request(req, error, bytes)) return true; - /* Bidi request must be completed as a whole */ - if (unlikely(bidi_bytes) && - blk_update_request(req->next_rq, error, bidi_bytes)) - return true; - if (blk_queue_add_random(q)) add_disk_randomness(req->rq_disk); @@ -817,7 +804,7 @@ static void scsi_io_completion_action(struct scsi_cmnd *cmd, int result) scsi_print_command(cmd); } } - if (!scsi_end_request(req, blk_stat, blk_rq_err_bytes(req), 0)) + if (!scsi_end_request(req, blk_stat, blk_rq_err_bytes(req))) return; /*FALLTHRU*/ case ACTION_REPREP: @@ -951,30 +938,6 @@ void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes) * scsi_result_to_blk_status may have reset the host_byte */ scsi_req(req)->result = cmd->result; - scsi_req(req)->resid_len = scsi_get_resid(cmd); - - if (unlikely(scsi_bidi_cmnd(cmd))) { - /* - * Bidi commands Must be complete as a whole, - * both sides at once. - */ - scsi_req(req->next_rq)->resid_len = scsi_in(cmd)->resid; - if (scsi_end_request(req, BLK_STS_OK, blk_rq_bytes(req), - blk_rq_bytes(req->next_rq))) - WARN_ONCE(true, - "Bidi command with remaining bytes"); - return; - } - } - - /* no bidi support yet, other than in pass-through */ - if (unlikely(blk_bidi_rq(req))) { - WARN_ONCE(true, "Only support bidi command in passthrough"); - scmd_printk(KERN_ERR, cmd, "Killing bidi command\n"); - if (scsi_end_request(req, BLK_STS_IOERR, blk_rq_bytes(req), - blk_rq_bytes(req->next_rq))) - WARN_ONCE(true, "Bidi command with remaining bytes"); - return; } /* @@ -991,13 +954,13 @@ void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes) * to retry code. Fast path should return in this block. */ if (likely(blk_rq_bytes(req) > 0 || blk_stat == BLK_STS_OK)) { - if (likely(!scsi_end_request(req, blk_stat, good_bytes, 0))) + if (likely(!scsi_end_request(req, blk_stat, good_bytes))) return; /* no bytes remaining */ } /* Kill remainder if no retries. */ if (unlikely(blk_stat && scsi_noretry_cmd(cmd))) { - if (scsi_end_request(req, blk_stat, blk_rq_bytes(req), 0)) + if (scsi_end_request(req, blk_stat, blk_rq_bytes(req))) WARN_ONCE(true, "Bytes remaining after failed, no-retry command"); return; @@ -1059,12 +1022,6 @@ blk_status_t scsi_init_io(struct scsi_cmnd *cmd) if (ret) return ret; - if (blk_bidi_rq(rq)) { - ret = scsi_init_sgtable(rq->next_rq, rq->next_rq->special); - if (ret) - goto out_free_sgtables; - } - if (blk_integrity_rq(rq)) { struct scsi_data_buffer *prot_sdb = cmd->prot_sdb; int ivecs, count; @@ -1608,10 +1565,7 @@ static blk_status_t scsi_mq_prep_fn(struct request *req) scsi_init_command(sdev, cmd); - req->special = cmd; - cmd->request = req; - cmd->tag = req->tag; cmd->prot_op = SCSI_PROT_NORMAL; @@ -1625,17 +1579,6 @@ static blk_status_t scsi_mq_prep_fn(struct request *req) (struct scatterlist *)(cmd->prot_sdb + 1); } - if (blk_bidi_rq(req)) { - struct request *next_rq = req->next_rq; - struct scsi_data_buffer *bidi_sdb = blk_mq_rq_to_pdu(next_rq); - - memset(bidi_sdb, 0, sizeof(struct scsi_data_buffer)); - bidi_sdb->table.sgl = - (struct scatterlist *)(bidi_sdb + 1); - - next_rq->special = bidi_sdb; - } - blk_mq_start_request(req); return scsi_setup_cmnd(sdev, req); @@ -1713,13 +1656,13 @@ static blk_status_t scsi_queue_rq(struct blk_mq_hw_ctx *hctx, if (!scsi_host_queue_ready(q, shost, sdev)) goto out_dec_target_busy; - clear_bit(SCMD_STATE_COMPLETE, &cmd->state); if (!(req->rq_flags & RQF_DONTPREP)) { ret = scsi_mq_prep_fn(req); if (ret != BLK_STS_OK) goto out_dec_host_busy; req->rq_flags |= RQF_DONTPREP; } else { + clear_bit(SCMD_STATE_COMPLETE, &cmd->state); blk_mq_start_request(req); } diff --git a/drivers/scsi/scsi_scan.c b/drivers/scsi/scsi_scan.c index dd0d516f65e2..53380e07b40e 100644 --- a/drivers/scsi/scsi_scan.c +++ b/drivers/scsi/scsi_scan.c @@ -220,7 +220,7 @@ static struct scsi_device *scsi_alloc_sdev(struct scsi_target *starget, struct Scsi_Host *shost = dev_to_shost(starget->dev.parent); sdev = kzalloc(sizeof(*sdev) + shost->transportt->device_size, - GFP_ATOMIC); + GFP_KERNEL); if (!sdev) goto out; @@ -788,7 +788,7 @@ static int scsi_add_lun(struct scsi_device *sdev, unsigned char *inq_result, */ sdev->inquiry = kmemdup(inq_result, max_t(size_t, sdev->inquiry_len, 36), - GFP_ATOMIC); + GFP_KERNEL); if (sdev->inquiry == NULL) return SCSI_SCAN_NO_RESPONSE; @@ -1079,7 +1079,7 @@ static int scsi_probe_and_add_lun(struct scsi_target *starget, if (!sdev) goto out; - result = kmalloc(result_len, GFP_ATOMIC | + result = kmalloc(result_len, GFP_KERNEL | ((shost->unchecked_isa_dma) ? __GFP_DMA : 0)); if (!result) goto out_free_sdev; diff --git a/drivers/scsi/scsi_transport_sas.c b/drivers/scsi/scsi_transport_sas.c index 692b46937e52..60f1a81d2034 100644 --- a/drivers/scsi/scsi_transport_sas.c +++ b/drivers/scsi/scsi_transport_sas.c @@ -213,7 +213,6 @@ static int sas_bsg_initialize(struct Scsi_Host *shost, struct sas_rphy *rphy) to_sas_host_attrs(shost)->q = q; } - blk_queue_flag_set(QUEUE_FLAG_BIDI, q); return 0; } diff --git a/drivers/scsi/sd.c b/drivers/scsi/sd.c index 5464d467e23e..251db30d0882 100644 --- a/drivers/scsi/sd.c +++ b/drivers/scsi/sd.c @@ -665,6 +665,68 @@ static int sd_sec_submit(void *data, u16 spsp, u8 secp, void *buffer, } #endif /* CONFIG_BLK_SED_OPAL */ +/* + * Look up the DIX operation based on whether the command is read or + * write and whether dix and dif are enabled. + */ +static unsigned int sd_prot_op(bool write, bool dix, bool dif) +{ + /* Lookup table: bit 2 (write), bit 1 (dix), bit 0 (dif) */ + static const unsigned int ops[] = { /* wrt dix dif */ + SCSI_PROT_NORMAL, /* 0 0 0 */ + SCSI_PROT_READ_STRIP, /* 0 0 1 */ + SCSI_PROT_READ_INSERT, /* 0 1 0 */ + SCSI_PROT_READ_PASS, /* 0 1 1 */ + SCSI_PROT_NORMAL, /* 1 0 0 */ + SCSI_PROT_WRITE_INSERT, /* 1 0 1 */ + SCSI_PROT_WRITE_STRIP, /* 1 1 0 */ + SCSI_PROT_WRITE_PASS, /* 1 1 1 */ + }; + + return ops[write << 2 | dix << 1 | dif]; +} + +/* + * Returns a mask of the protection flags that are valid for a given DIX + * operation. + */ +static unsigned int sd_prot_flag_mask(unsigned int prot_op) +{ + static const unsigned int flag_mask[] = { + [SCSI_PROT_NORMAL] = 0, + + [SCSI_PROT_READ_STRIP] = SCSI_PROT_TRANSFER_PI | + SCSI_PROT_GUARD_CHECK | + SCSI_PROT_REF_CHECK | + SCSI_PROT_REF_INCREMENT, + + [SCSI_PROT_READ_INSERT] = SCSI_PROT_REF_INCREMENT | + SCSI_PROT_IP_CHECKSUM, + + [SCSI_PROT_READ_PASS] = SCSI_PROT_TRANSFER_PI | + SCSI_PROT_GUARD_CHECK | + SCSI_PROT_REF_CHECK | + SCSI_PROT_REF_INCREMENT | + SCSI_PROT_IP_CHECKSUM, + + [SCSI_PROT_WRITE_INSERT] = SCSI_PROT_TRANSFER_PI | + SCSI_PROT_REF_INCREMENT, + + [SCSI_PROT_WRITE_STRIP] = SCSI_PROT_GUARD_CHECK | + SCSI_PROT_REF_CHECK | + SCSI_PROT_REF_INCREMENT | + SCSI_PROT_IP_CHECKSUM, + + [SCSI_PROT_WRITE_PASS] = SCSI_PROT_TRANSFER_PI | + SCSI_PROT_GUARD_CHECK | + SCSI_PROT_REF_CHECK | + SCSI_PROT_REF_INCREMENT | + SCSI_PROT_IP_CHECKSUM, + }; + + return flag_mask[prot_op]; +} + static unsigned char sd_setup_protect_cmnd(struct scsi_cmnd *scmd, unsigned int dix, unsigned int dif) { @@ -761,8 +823,8 @@ static blk_status_t sd_setup_unmap_cmnd(struct scsi_cmnd *cmd) { struct scsi_device *sdp = cmd->device; struct request *rq = cmd->request; - u64 sector = blk_rq_pos(rq) >> (ilog2(sdp->sector_size) - 9); - u32 nr_sectors = blk_rq_sectors(rq) >> (ilog2(sdp->sector_size) - 9); + u64 lba = sectors_to_logical(sdp, blk_rq_pos(rq)); + u32 nr_blocks = sectors_to_logical(sdp, blk_rq_sectors(rq)); unsigned int data_len = 24; char *buf; @@ -781,13 +843,12 @@ static blk_status_t sd_setup_unmap_cmnd(struct scsi_cmnd *cmd) buf = page_address(rq->special_vec.bv_page); put_unaligned_be16(6 + 16, &buf[0]); put_unaligned_be16(16, &buf[2]); - put_unaligned_be64(sector, &buf[8]); - put_unaligned_be32(nr_sectors, &buf[16]); + put_unaligned_be64(lba, &buf[8]); + put_unaligned_be32(nr_blocks, &buf[16]); cmd->allowed = SD_MAX_RETRIES; cmd->transfersize = data_len; rq->timeout = SD_TIMEOUT; - scsi_req(rq)->resid_len = data_len; return scsi_init_io(cmd); } @@ -797,8 +858,8 @@ static blk_status_t sd_setup_write_same16_cmnd(struct scsi_cmnd *cmd, { struct scsi_device *sdp = cmd->device; struct request *rq = cmd->request; - u64 sector = blk_rq_pos(rq) >> (ilog2(sdp->sector_size) - 9); - u32 nr_sectors = blk_rq_sectors(rq) >> (ilog2(sdp->sector_size) - 9); + u64 lba = sectors_to_logical(sdp, blk_rq_pos(rq)); + u32 nr_blocks = sectors_to_logical(sdp, blk_rq_sectors(rq)); u32 data_len = sdp->sector_size; rq->special_vec.bv_page = mempool_alloc(sd_page_pool, GFP_ATOMIC); @@ -813,13 +874,12 @@ static blk_status_t sd_setup_write_same16_cmnd(struct scsi_cmnd *cmd, cmd->cmnd[0] = WRITE_SAME_16; if (unmap) cmd->cmnd[1] = 0x8; /* UNMAP */ - put_unaligned_be64(sector, &cmd->cmnd[2]); - put_unaligned_be32(nr_sectors, &cmd->cmnd[10]); + put_unaligned_be64(lba, &cmd->cmnd[2]); + put_unaligned_be32(nr_blocks, &cmd->cmnd[10]); cmd->allowed = SD_MAX_RETRIES; cmd->transfersize = data_len; rq->timeout = unmap ? SD_TIMEOUT : SD_WRITE_SAME_TIMEOUT; - scsi_req(rq)->resid_len = data_len; return scsi_init_io(cmd); } @@ -829,8 +889,8 @@ static blk_status_t sd_setup_write_same10_cmnd(struct scsi_cmnd *cmd, { struct scsi_device *sdp = cmd->device; struct request *rq = cmd->request; - u64 sector = blk_rq_pos(rq) >> (ilog2(sdp->sector_size) - 9); - u32 nr_sectors = blk_rq_sectors(rq) >> (ilog2(sdp->sector_size) - 9); + u64 lba = sectors_to_logical(sdp, blk_rq_pos(rq)); + u32 nr_blocks = sectors_to_logical(sdp, blk_rq_sectors(rq)); u32 data_len = sdp->sector_size; rq->special_vec.bv_page = mempool_alloc(sd_page_pool, GFP_ATOMIC); @@ -845,13 +905,12 @@ static blk_status_t sd_setup_write_same10_cmnd(struct scsi_cmnd *cmd, cmd->cmnd[0] = WRITE_SAME; if (unmap) cmd->cmnd[1] = 0x8; /* UNMAP */ - put_unaligned_be32(sector, &cmd->cmnd[2]); - put_unaligned_be16(nr_sectors, &cmd->cmnd[7]); + put_unaligned_be32(lba, &cmd->cmnd[2]); + put_unaligned_be16(nr_blocks, &cmd->cmnd[7]); cmd->allowed = SD_MAX_RETRIES; cmd->transfersize = data_len; rq->timeout = unmap ? SD_TIMEOUT : SD_WRITE_SAME_TIMEOUT; - scsi_req(rq)->resid_len = data_len; return scsi_init_io(cmd); } @@ -861,8 +920,8 @@ static blk_status_t sd_setup_write_zeroes_cmnd(struct scsi_cmnd *cmd) struct request *rq = cmd->request; struct scsi_device *sdp = cmd->device; struct scsi_disk *sdkp = scsi_disk(rq->rq_disk); - u64 sector = blk_rq_pos(rq) >> (ilog2(sdp->sector_size) - 9); - u32 nr_sectors = blk_rq_sectors(rq) >> (ilog2(sdp->sector_size) - 9); + u64 lba = sectors_to_logical(sdp, blk_rq_pos(rq)); + u32 nr_blocks = sectors_to_logical(sdp, blk_rq_sectors(rq)); if (!(rq->cmd_flags & REQ_NOUNMAP)) { switch (sdkp->zeroing_mode) { @@ -876,7 +935,7 @@ static blk_status_t sd_setup_write_zeroes_cmnd(struct scsi_cmnd *cmd) if (sdp->no_write_same) return BLK_STS_TARGET; - if (sdkp->ws16 || sector > 0xffffffff || nr_sectors > 0xffff) + if (sdkp->ws16 || lba > 0xffffffff || nr_blocks > 0xffff) return sd_setup_write_same16_cmnd(cmd, false); return sd_setup_write_same10_cmnd(cmd, false); @@ -957,9 +1016,8 @@ static blk_status_t sd_setup_write_same_cmnd(struct scsi_cmnd *cmd) struct scsi_device *sdp = cmd->device; struct scsi_disk *sdkp = scsi_disk(rq->rq_disk); struct bio *bio = rq->bio; - sector_t sector = blk_rq_pos(rq); - unsigned int nr_sectors = blk_rq_sectors(rq); - unsigned int nr_bytes = blk_rq_bytes(rq); + u64 lba = sectors_to_logical(sdp, blk_rq_pos(rq)); + u32 nr_blocks = sectors_to_logical(sdp, blk_rq_sectors(rq)); blk_status_t ret; if (sdkp->device->no_write_same) @@ -967,21 +1025,18 @@ static blk_status_t sd_setup_write_same_cmnd(struct scsi_cmnd *cmd) BUG_ON(bio_offset(bio) || bio_iovec(bio).bv_len != sdp->sector_size); - sector >>= ilog2(sdp->sector_size) - 9; - nr_sectors >>= ilog2(sdp->sector_size) - 9; - rq->timeout = SD_WRITE_SAME_TIMEOUT; - if (sdkp->ws16 || sector > 0xffffffff || nr_sectors > 0xffff) { + if (sdkp->ws16 || lba > 0xffffffff || nr_blocks > 0xffff) { cmd->cmd_len = 16; cmd->cmnd[0] = WRITE_SAME_16; - put_unaligned_be64(sector, &cmd->cmnd[2]); - put_unaligned_be32(nr_sectors, &cmd->cmnd[10]); + put_unaligned_be64(lba, &cmd->cmnd[2]); + put_unaligned_be32(nr_blocks, &cmd->cmnd[10]); } else { cmd->cmd_len = 10; cmd->cmnd[0] = WRITE_SAME; - put_unaligned_be32(sector, &cmd->cmnd[2]); - put_unaligned_be16(nr_sectors, &cmd->cmnd[7]); + put_unaligned_be32(lba, &cmd->cmnd[2]); + put_unaligned_be16(nr_blocks, &cmd->cmnd[7]); } cmd->transfersize = sdp->sector_size; @@ -999,7 +1054,7 @@ static blk_status_t sd_setup_write_same_cmnd(struct scsi_cmnd *cmd) */ rq->__data_len = sdp->sector_size; ret = scsi_init_io(cmd); - rq->__data_len = nr_bytes; + rq->__data_len = blk_rq_bytes(rq); return ret; } @@ -1020,224 +1075,186 @@ static blk_status_t sd_setup_flush_cmnd(struct scsi_cmnd *cmd) return BLK_STS_OK; } -static blk_status_t sd_setup_read_write_cmnd(struct scsi_cmnd *SCpnt) +static blk_status_t sd_setup_rw32_cmnd(struct scsi_cmnd *cmd, bool write, + sector_t lba, unsigned int nr_blocks, + unsigned char flags) { - struct request *rq = SCpnt->request; - struct scsi_device *sdp = SCpnt->device; - struct gendisk *disk = rq->rq_disk; - struct scsi_disk *sdkp = scsi_disk(disk); - sector_t block = blk_rq_pos(rq); + cmd->cmnd = mempool_alloc(sd_cdb_pool, GFP_ATOMIC); + if (unlikely(cmd->cmnd == NULL)) + return BLK_STS_RESOURCE; + + cmd->cmd_len = SD_EXT_CDB_SIZE; + memset(cmd->cmnd, 0, cmd->cmd_len); + + cmd->cmnd[0] = VARIABLE_LENGTH_CMD; + cmd->cmnd[7] = 0x18; /* Additional CDB len */ + cmd->cmnd[9] = write ? WRITE_32 : READ_32; + cmd->cmnd[10] = flags; + put_unaligned_be64(lba, &cmd->cmnd[12]); + put_unaligned_be32(lba, &cmd->cmnd[20]); /* Expected Indirect LBA */ + put_unaligned_be32(nr_blocks, &cmd->cmnd[28]); + + return BLK_STS_OK; +} + +static blk_status_t sd_setup_rw16_cmnd(struct scsi_cmnd *cmd, bool write, + sector_t lba, unsigned int nr_blocks, + unsigned char flags) +{ + cmd->cmd_len = 16; + cmd->cmnd[0] = write ? WRITE_16 : READ_16; + cmd->cmnd[1] = flags; + cmd->cmnd[14] = 0; + cmd->cmnd[15] = 0; + put_unaligned_be64(lba, &cmd->cmnd[2]); + put_unaligned_be32(nr_blocks, &cmd->cmnd[10]); + + return BLK_STS_OK; +} + +static blk_status_t sd_setup_rw10_cmnd(struct scsi_cmnd *cmd, bool write, + sector_t lba, unsigned int nr_blocks, + unsigned char flags) +{ + cmd->cmd_len = 10; + cmd->cmnd[0] = write ? WRITE_10 : READ_10; + cmd->cmnd[1] = flags; + cmd->cmnd[6] = 0; + cmd->cmnd[9] = 0; + put_unaligned_be32(lba, &cmd->cmnd[2]); + put_unaligned_be16(nr_blocks, &cmd->cmnd[7]); + + return BLK_STS_OK; +} + +static blk_status_t sd_setup_rw6_cmnd(struct scsi_cmnd *cmd, bool write, + sector_t lba, unsigned int nr_blocks, + unsigned char flags) +{ + /* Avoid that 0 blocks gets translated into 256 blocks. */ + if (WARN_ON_ONCE(nr_blocks == 0)) + return BLK_STS_IOERR; + + if (unlikely(flags & 0x8)) { + /* + * This happens only if this drive failed 10byte rw + * command with ILLEGAL_REQUEST during operation and + * thus turned off use_10_for_rw. + */ + scmd_printk(KERN_ERR, cmd, "FUA write on READ/WRITE(6) drive\n"); + return BLK_STS_IOERR; + } + + cmd->cmd_len = 6; + cmd->cmnd[0] = write ? WRITE_6 : READ_6; + cmd->cmnd[1] = (lba >> 16) & 0x1f; + cmd->cmnd[2] = (lba >> 8) & 0xff; + cmd->cmnd[3] = lba & 0xff; + cmd->cmnd[4] = nr_blocks; + cmd->cmnd[5] = 0; + + return BLK_STS_OK; +} + +static blk_status_t sd_setup_read_write_cmnd(struct scsi_cmnd *cmd) +{ + struct request *rq = cmd->request; + struct scsi_device *sdp = cmd->device; + struct scsi_disk *sdkp = scsi_disk(rq->rq_disk); + sector_t lba = sectors_to_logical(sdp, blk_rq_pos(rq)); sector_t threshold; - unsigned int this_count = blk_rq_sectors(rq); - unsigned int dif, dix; - unsigned char protect; + unsigned int nr_blocks = sectors_to_logical(sdp, blk_rq_sectors(rq)); + bool dif, dix; + unsigned int mask = logical_to_sectors(sdp, 1) - 1; + bool write = rq_data_dir(rq) == WRITE; + unsigned char protect, fua; blk_status_t ret; - ret = scsi_init_io(SCpnt); + ret = scsi_init_io(cmd); if (ret != BLK_STS_OK) return ret; - WARN_ON_ONCE(SCpnt != rq->special); - SCSI_LOG_HLQUEUE(1, - scmd_printk(KERN_INFO, SCpnt, - "%s: block=%llu, count=%d\n", - __func__, (unsigned long long)block, this_count)); - - if (!sdp || !scsi_device_online(sdp) || - block + blk_rq_sectors(rq) > get_capacity(disk)) { - SCSI_LOG_HLQUEUE(2, scmd_printk(KERN_INFO, SCpnt, - "Finishing %u sectors\n", - blk_rq_sectors(rq))); - SCSI_LOG_HLQUEUE(2, scmd_printk(KERN_INFO, SCpnt, - "Retry with 0x%p\n", SCpnt)); + if (!scsi_device_online(sdp) || sdp->changed) { + scmd_printk(KERN_ERR, cmd, "device offline or changed\n"); return BLK_STS_IOERR; } - if (sdp->changed) { - /* - * quietly refuse to do anything to a changed disc until - * the changed bit has been reset - */ - /* printk("SCSI disk has been changed or is not present. Prohibiting further I/O.\n"); */ + if (blk_rq_pos(rq) + blk_rq_sectors(rq) > get_capacity(rq->rq_disk)) { + scmd_printk(KERN_ERR, cmd, "access beyond end of device\n"); + return BLK_STS_IOERR; + } + + if ((blk_rq_pos(rq) & mask) || (blk_rq_sectors(rq) & mask)) { + scmd_printk(KERN_ERR, cmd, "request not aligned to the logical block size\n"); return BLK_STS_IOERR; } /* - * Some SD card readers can't handle multi-sector accesses which touch - * the last one or two hardware sectors. Split accesses as needed. + * Some SD card readers can't handle accesses which touch the + * last one or two logical blocks. Split accesses as needed. */ - threshold = get_capacity(disk) - SD_LAST_BUGGY_SECTORS * - (sdp->sector_size / 512); + threshold = sdkp->capacity - SD_LAST_BUGGY_SECTORS; - if (unlikely(sdp->last_sector_bug && block + this_count > threshold)) { - if (block < threshold) { + if (unlikely(sdp->last_sector_bug && lba + nr_blocks > threshold)) { + if (lba < threshold) { /* Access up to the threshold but not beyond */ - this_count = threshold - block; + nr_blocks = threshold - lba; } else { - /* Access only a single hardware sector */ - this_count = sdp->sector_size / 512; + /* Access only a single logical block */ + nr_blocks = 1; } } - SCSI_LOG_HLQUEUE(2, scmd_printk(KERN_INFO, SCpnt, "block=%llu\n", - (unsigned long long)block)); - - /* - * If we have a 1K hardware sectorsize, prevent access to single - * 512 byte sectors. In theory we could handle this - in fact - * the scsi cdrom driver must be able to handle this because - * we typically use 1K blocksizes, and cdroms typically have - * 2K hardware sectorsizes. Of course, things are simpler - * with the cdrom, since it is read-only. For performance - * reasons, the filesystems should be able to handle this - * and not force the scsi disk driver to use bounce buffers - * for this. - */ - if (sdp->sector_size == 1024) { - if ((block & 1) || (blk_rq_sectors(rq) & 1)) { - scmd_printk(KERN_ERR, SCpnt, - "Bad block number requested\n"); - return BLK_STS_IOERR; - } - block = block >> 1; - this_count = this_count >> 1; - } - if (sdp->sector_size == 2048) { - if ((block & 3) || (blk_rq_sectors(rq) & 3)) { - scmd_printk(KERN_ERR, SCpnt, - "Bad block number requested\n"); - return BLK_STS_IOERR; - } - block = block >> 2; - this_count = this_count >> 2; - } - if (sdp->sector_size == 4096) { - if ((block & 7) || (blk_rq_sectors(rq) & 7)) { - scmd_printk(KERN_ERR, SCpnt, - "Bad block number requested\n"); - return BLK_STS_IOERR; - } - block = block >> 3; - this_count = this_count >> 3; - } - if (rq_data_dir(rq) == WRITE) { - SCpnt->cmnd[0] = WRITE_6; - - if (blk_integrity_rq(rq)) - t10_pi_prepare(SCpnt->request, sdkp->protection_type); - - } else if (rq_data_dir(rq) == READ) { - SCpnt->cmnd[0] = READ_6; - } else { - scmd_printk(KERN_ERR, SCpnt, "Unknown command %d\n", req_op(rq)); - return BLK_STS_IOERR; - } - - SCSI_LOG_HLQUEUE(2, scmd_printk(KERN_INFO, SCpnt, - "%s %d/%u 512 byte blocks.\n", - (rq_data_dir(rq) == WRITE) ? - "writing" : "reading", this_count, - blk_rq_sectors(rq))); + fua = rq->cmd_flags & REQ_FUA ? 0x8 : 0; + dix = scsi_prot_sg_count(cmd); + dif = scsi_host_dif_capable(cmd->device->host, sdkp->protection_type); - dix = scsi_prot_sg_count(SCpnt); - dif = scsi_host_dif_capable(SCpnt->device->host, sdkp->protection_type); + if (write && dix) + t10_pi_prepare(cmd->request, sdkp->protection_type); if (dif || dix) - protect = sd_setup_protect_cmnd(SCpnt, dix, dif); + protect = sd_setup_protect_cmnd(cmd, dix, dif); else protect = 0; if (protect && sdkp->protection_type == T10_PI_TYPE2_PROTECTION) { - SCpnt->cmnd = mempool_alloc(sd_cdb_pool, GFP_ATOMIC); - - if (unlikely(!SCpnt->cmnd)) - return BLK_STS_RESOURCE; - - SCpnt->cmd_len = SD_EXT_CDB_SIZE; - memset(SCpnt->cmnd, 0, SCpnt->cmd_len); - SCpnt->cmnd[0] = VARIABLE_LENGTH_CMD; - SCpnt->cmnd[7] = 0x18; - SCpnt->cmnd[9] = (rq_data_dir(rq) == READ) ? READ_32 : WRITE_32; - SCpnt->cmnd[10] = protect | ((rq->cmd_flags & REQ_FUA) ? 0x8 : 0); - - /* LBA */ - SCpnt->cmnd[12] = sizeof(block) > 4 ? (unsigned char) (block >> 56) & 0xff : 0; - SCpnt->cmnd[13] = sizeof(block) > 4 ? (unsigned char) (block >> 48) & 0xff : 0; - SCpnt->cmnd[14] = sizeof(block) > 4 ? (unsigned char) (block >> 40) & 0xff : 0; - SCpnt->cmnd[15] = sizeof(block) > 4 ? (unsigned char) (block >> 32) & 0xff : 0; - SCpnt->cmnd[16] = (unsigned char) (block >> 24) & 0xff; - SCpnt->cmnd[17] = (unsigned char) (block >> 16) & 0xff; - SCpnt->cmnd[18] = (unsigned char) (block >> 8) & 0xff; - SCpnt->cmnd[19] = (unsigned char) block & 0xff; - - /* Expected Indirect LBA */ - SCpnt->cmnd[20] = (unsigned char) (block >> 24) & 0xff; - SCpnt->cmnd[21] = (unsigned char) (block >> 16) & 0xff; - SCpnt->cmnd[22] = (unsigned char) (block >> 8) & 0xff; - SCpnt->cmnd[23] = (unsigned char) block & 0xff; - - /* Transfer length */ - SCpnt->cmnd[28] = (unsigned char) (this_count >> 24) & 0xff; - SCpnt->cmnd[29] = (unsigned char) (this_count >> 16) & 0xff; - SCpnt->cmnd[30] = (unsigned char) (this_count >> 8) & 0xff; - SCpnt->cmnd[31] = (unsigned char) this_count & 0xff; - } else if (sdp->use_16_for_rw || (this_count > 0xffff)) { - SCpnt->cmnd[0] += READ_16 - READ_6; - SCpnt->cmnd[1] = protect | ((rq->cmd_flags & REQ_FUA) ? 0x8 : 0); - SCpnt->cmnd[2] = sizeof(block) > 4 ? (unsigned char) (block >> 56) & 0xff : 0; - SCpnt->cmnd[3] = sizeof(block) > 4 ? (unsigned char) (block >> 48) & 0xff : 0; - SCpnt->cmnd[4] = sizeof(block) > 4 ? (unsigned char) (block >> 40) & 0xff : 0; - SCpnt->cmnd[5] = sizeof(block) > 4 ? (unsigned char) (block >> 32) & 0xff : 0; - SCpnt->cmnd[6] = (unsigned char) (block >> 24) & 0xff; - SCpnt->cmnd[7] = (unsigned char) (block >> 16) & 0xff; - SCpnt->cmnd[8] = (unsigned char) (block >> 8) & 0xff; - SCpnt->cmnd[9] = (unsigned char) block & 0xff; - SCpnt->cmnd[10] = (unsigned char) (this_count >> 24) & 0xff; - SCpnt->cmnd[11] = (unsigned char) (this_count >> 16) & 0xff; - SCpnt->cmnd[12] = (unsigned char) (this_count >> 8) & 0xff; - SCpnt->cmnd[13] = (unsigned char) this_count & 0xff; - SCpnt->cmnd[14] = SCpnt->cmnd[15] = 0; - } else if ((this_count > 0xff) || (block > 0x1fffff) || - scsi_device_protection(SCpnt->device) || - SCpnt->device->use_10_for_rw) { - SCpnt->cmnd[0] += READ_10 - READ_6; - SCpnt->cmnd[1] = protect | ((rq->cmd_flags & REQ_FUA) ? 0x8 : 0); - SCpnt->cmnd[2] = (unsigned char) (block >> 24) & 0xff; - SCpnt->cmnd[3] = (unsigned char) (block >> 16) & 0xff; - SCpnt->cmnd[4] = (unsigned char) (block >> 8) & 0xff; - SCpnt->cmnd[5] = (unsigned char) block & 0xff; - SCpnt->cmnd[6] = SCpnt->cmnd[9] = 0; - SCpnt->cmnd[7] = (unsigned char) (this_count >> 8) & 0xff; - SCpnt->cmnd[8] = (unsigned char) this_count & 0xff; + ret = sd_setup_rw32_cmnd(cmd, write, lba, nr_blocks, + protect | fua); + } else if (sdp->use_16_for_rw || (nr_blocks > 0xffff)) { + ret = sd_setup_rw16_cmnd(cmd, write, lba, nr_blocks, + protect | fua); + } else if ((nr_blocks > 0xff) || (lba > 0x1fffff) || + sdp->use_10_for_rw || protect) { + ret = sd_setup_rw10_cmnd(cmd, write, lba, nr_blocks, + protect | fua); } else { - if (unlikely(rq->cmd_flags & REQ_FUA)) { - /* - * This happens only if this drive failed - * 10byte rw command with ILLEGAL_REQUEST - * during operation and thus turned off - * use_10_for_rw. - */ - scmd_printk(KERN_ERR, SCpnt, - "FUA write on READ/WRITE(6) drive\n"); - return BLK_STS_IOERR; - } - - SCpnt->cmnd[1] |= (unsigned char) ((block >> 16) & 0x1f); - SCpnt->cmnd[2] = (unsigned char) ((block >> 8) & 0xff); - SCpnt->cmnd[3] = (unsigned char) block & 0xff; - SCpnt->cmnd[4] = (unsigned char) this_count; - SCpnt->cmnd[5] = 0; + ret = sd_setup_rw6_cmnd(cmd, write, lba, nr_blocks, + protect | fua); } - SCpnt->sdb.length = this_count * sdp->sector_size; + + if (unlikely(ret != BLK_STS_OK)) + return ret; /* * We shouldn't disconnect in the middle of a sector, so with a dumb * host adapter, it's safe to assume that we can at least transfer * this many bytes between each connect / disconnect. */ - SCpnt->transfersize = sdp->sector_size; - SCpnt->underflow = this_count << 9; - SCpnt->allowed = SD_MAX_RETRIES; + cmd->transfersize = sdp->sector_size; + cmd->underflow = nr_blocks << 9; + cmd->allowed = SD_MAX_RETRIES; + cmd->sdb.length = nr_blocks * sdp->sector_size; + + SCSI_LOG_HLQUEUE(1, + scmd_printk(KERN_INFO, cmd, + "%s: block=%llu, count=%d\n", __func__, + (unsigned long long)blk_rq_pos(rq), + blk_rq_sectors(rq))); + SCSI_LOG_HLQUEUE(2, + scmd_printk(KERN_INFO, cmd, + "%s %d/%u 512 byte blocks.\n", + write ? "writing" : "reading", nr_blocks, + blk_rq_sectors(rq))); /* * This indicates that the command is ready from our end to be @@ -2549,25 +2566,25 @@ sd_print_capacity(struct scsi_disk *sdkp, int sector_size = sdkp->device->sector_size; char cap_str_2[10], cap_str_10[10]; + if (!sdkp->first_scan && old_capacity == sdkp->capacity) + return; + string_get_size(sdkp->capacity, sector_size, STRING_UNITS_2, cap_str_2, sizeof(cap_str_2)); string_get_size(sdkp->capacity, sector_size, - STRING_UNITS_10, cap_str_10, - sizeof(cap_str_10)); + STRING_UNITS_10, cap_str_10, sizeof(cap_str_10)); - if (sdkp->first_scan || old_capacity != sdkp->capacity) { - sd_printk(KERN_NOTICE, sdkp, - "%llu %d-byte logical blocks: (%s/%s)\n", - (unsigned long long)sdkp->capacity, - sector_size, cap_str_10, cap_str_2); + sd_printk(KERN_NOTICE, sdkp, + "%llu %d-byte logical blocks: (%s/%s)\n", + (unsigned long long)sdkp->capacity, + sector_size, cap_str_10, cap_str_2); - if (sdkp->physical_block_size != sector_size) - sd_printk(KERN_NOTICE, sdkp, - "%u-byte physical blocks\n", - sdkp->physical_block_size); + if (sdkp->physical_block_size != sector_size) + sd_printk(KERN_NOTICE, sdkp, + "%u-byte physical blocks\n", + sdkp->physical_block_size); - sd_zbc_print_zones(sdkp); - } + sd_zbc_print_zones(sdkp); } /* called with buffer of length 512 */ @@ -3047,6 +3064,55 @@ static void sd_read_security(struct scsi_disk *sdkp, unsigned char *buffer) sdkp->security = 1; } +/* + * Determine the device's preferred I/O size for reads and writes + * unless the reported value is unreasonably small, large, not a + * multiple of the physical block size, or simply garbage. + */ +static bool sd_validate_opt_xfer_size(struct scsi_disk *sdkp, + unsigned int dev_max) +{ + struct scsi_device *sdp = sdkp->device; + unsigned int opt_xfer_bytes = + logical_to_bytes(sdp, sdkp->opt_xfer_blocks); + + if (sdkp->opt_xfer_blocks > dev_max) { + sd_first_printk(KERN_WARNING, sdkp, + "Optimal transfer size %u logical blocks " \ + "> dev_max (%u logical blocks)\n", + sdkp->opt_xfer_blocks, dev_max); + return false; + } + + if (sdkp->opt_xfer_blocks > SD_DEF_XFER_BLOCKS) { + sd_first_printk(KERN_WARNING, sdkp, + "Optimal transfer size %u logical blocks " \ + "> sd driver limit (%u logical blocks)\n", + sdkp->opt_xfer_blocks, SD_DEF_XFER_BLOCKS); + return false; + } + + if (opt_xfer_bytes < PAGE_SIZE) { + sd_first_printk(KERN_WARNING, sdkp, + "Optimal transfer size %u bytes < " \ + "PAGE_SIZE (%u bytes)\n", + opt_xfer_bytes, (unsigned int)PAGE_SIZE); + return false; + } + + if (opt_xfer_bytes & (sdkp->physical_block_size - 1)) { + sd_first_printk(KERN_WARNING, sdkp, + "Optimal transfer size %u bytes not a " \ + "multiple of physical block size (%u bytes)\n", + opt_xfer_bytes, sdkp->physical_block_size); + return false; + } + + sd_first_printk(KERN_INFO, sdkp, "Optimal transfer size %u bytes\n", + opt_xfer_bytes); + return true; +} + /** * sd_revalidate_disk - called the first time a new disk is seen, * performs disk spin up, read_capacity, etc. @@ -3125,15 +3191,7 @@ static int sd_revalidate_disk(struct gendisk *disk) dev_max = min_not_zero(dev_max, sdkp->max_xfer_blocks); q->limits.max_dev_sectors = logical_to_sectors(sdp, dev_max); - /* - * Determine the device's preferred I/O size for reads and writes - * unless the reported value is unreasonably small, large, or - * garbage. - */ - if (sdkp->opt_xfer_blocks && - sdkp->opt_xfer_blocks <= dev_max && - sdkp->opt_xfer_blocks <= SD_DEF_XFER_BLOCKS && - logical_to_bytes(sdp, sdkp->opt_xfer_blocks) >= PAGE_SIZE) { + if (sd_validate_opt_xfer_size(sdkp, dev_max)) { q->limits.io_opt = logical_to_bytes(sdp, sdkp->opt_xfer_blocks); rw_max = logical_to_sectors(sdp, sdkp->opt_xfer_blocks); } else diff --git a/drivers/scsi/sd.h b/drivers/scsi/sd.h index 7f43e6839bce..5796ace76225 100644 --- a/drivers/scsi/sd.h +++ b/drivers/scsi/sd.h @@ -132,7 +132,7 @@ static inline struct scsi_disk *scsi_disk(struct gendisk *disk) #define sd_first_printk(prefix, sdsk, fmt, a...) \ do { \ - if ((sdkp)->first_scan) \ + if ((sdsk)->first_scan) \ sd_printk(prefix, sdsk, fmt, ##a); \ } while (0) @@ -188,68 +188,6 @@ static inline sector_t sectors_to_logical(struct scsi_device *sdev, sector_t sec return sector >> (ilog2(sdev->sector_size) - 9); } -/* - * Look up the DIX operation based on whether the command is read or - * write and whether dix and dif are enabled. - */ -static inline unsigned int sd_prot_op(bool write, bool dix, bool dif) -{ - /* Lookup table: bit 2 (write), bit 1 (dix), bit 0 (dif) */ - const unsigned int ops[] = { /* wrt dix dif */ - SCSI_PROT_NORMAL, /* 0 0 0 */ - SCSI_PROT_READ_STRIP, /* 0 0 1 */ - SCSI_PROT_READ_INSERT, /* 0 1 0 */ - SCSI_PROT_READ_PASS, /* 0 1 1 */ - SCSI_PROT_NORMAL, /* 1 0 0 */ - SCSI_PROT_WRITE_INSERT, /* 1 0 1 */ - SCSI_PROT_WRITE_STRIP, /* 1 1 0 */ - SCSI_PROT_WRITE_PASS, /* 1 1 1 */ - }; - - return ops[write << 2 | dix << 1 | dif]; -} - -/* - * Returns a mask of the protection flags that are valid for a given DIX - * operation. - */ -static inline unsigned int sd_prot_flag_mask(unsigned int prot_op) -{ - const unsigned int flag_mask[] = { - [SCSI_PROT_NORMAL] = 0, - - [SCSI_PROT_READ_STRIP] = SCSI_PROT_TRANSFER_PI | - SCSI_PROT_GUARD_CHECK | - SCSI_PROT_REF_CHECK | - SCSI_PROT_REF_INCREMENT, - - [SCSI_PROT_READ_INSERT] = SCSI_PROT_REF_INCREMENT | - SCSI_PROT_IP_CHECKSUM, - - [SCSI_PROT_READ_PASS] = SCSI_PROT_TRANSFER_PI | - SCSI_PROT_GUARD_CHECK | - SCSI_PROT_REF_CHECK | - SCSI_PROT_REF_INCREMENT | - SCSI_PROT_IP_CHECKSUM, - - [SCSI_PROT_WRITE_INSERT] = SCSI_PROT_TRANSFER_PI | - SCSI_PROT_REF_INCREMENT, - - [SCSI_PROT_WRITE_STRIP] = SCSI_PROT_GUARD_CHECK | - SCSI_PROT_REF_CHECK | - SCSI_PROT_REF_INCREMENT | - SCSI_PROT_IP_CHECKSUM, - - [SCSI_PROT_WRITE_PASS] = SCSI_PROT_TRANSFER_PI | - SCSI_PROT_GUARD_CHECK | - SCSI_PROT_REF_CHECK | - SCSI_PROT_REF_INCREMENT | - SCSI_PROT_IP_CHECKSUM, - }; - - return flag_mask[prot_op]; -} - #ifdef CONFIG_BLK_DEV_INTEGRITY extern void sd_dif_config_host(struct scsi_disk *); diff --git a/drivers/scsi/smartpqi/Makefile b/drivers/scsi/smartpqi/Makefile index e6b779930230..a03a6edb0060 100644 --- a/drivers/scsi/smartpqi/Makefile +++ b/drivers/scsi/smartpqi/Makefile @@ -1,3 +1,2 @@ -ccflags-y += -I. obj-$(CONFIG_SCSI_SMARTPQI) += smartpqi.o smartpqi-objs := smartpqi_init.o smartpqi_sis.o smartpqi_sas_transport.o diff --git a/drivers/scsi/smartpqi/smartpqi_init.c b/drivers/scsi/smartpqi/smartpqi_init.c index f564af8949e8..5d9ccbab7581 100644 --- a/drivers/scsi/smartpqi/smartpqi_init.c +++ b/drivers/scsi/smartpqi/smartpqi_init.c @@ -6043,7 +6043,8 @@ out: return rc; } -static int pqi_ioctl(struct scsi_device *sdev, int cmd, void __user *arg) +static int pqi_ioctl(struct scsi_device *sdev, unsigned int cmd, + void __user *arg) { int rc; struct pqi_ctrl_info *ctrl_info; diff --git a/drivers/scsi/snic/snic_debugfs.c b/drivers/scsi/snic/snic_debugfs.c index 0abe17c1a73b..2b349365592f 100644 --- a/drivers/scsi/snic/snic_debugfs.c +++ b/drivers/scsi/snic/snic_debugfs.c @@ -30,33 +30,13 @@ * fnic directory and statistics directory for trace buffer and * stats logging */ - -int -snic_debugfs_init(void) +void snic_debugfs_init(void) { - int rc = -1; - struct dentry *de = NULL; - - de = debugfs_create_dir("snic", NULL); - if (!de) { - SNIC_DBG("Cannot create debugfs root\n"); - - return rc; - } - snic_glob->trc_root = de; - - de = debugfs_create_dir("statistics", snic_glob->trc_root); - if (!de) { - SNIC_DBG("Cannot create Statistics directory\n"); + snic_glob->trc_root = debugfs_create_dir("snic", NULL); - return rc; - } - snic_glob->stats_root = de; - - rc = 0; - - return rc; -} /* end of snic_debugfs_init */ + snic_glob->stats_root = debugfs_create_dir("statistics", + snic_glob->trc_root); +} /* * snic_debugfs_term - Tear down debugfs intrastructure @@ -391,56 +371,23 @@ static const struct file_operations snic_reset_stats_fops = { * It will create file stats and reset_stats under statistics/host# directory * to log per snic stats */ -int -snic_stats_debugfs_init(struct snic *snic) +void snic_stats_debugfs_init(struct snic *snic) { - int rc = -1; char name[16]; - struct dentry *de = NULL; snprintf(name, sizeof(name), "host%d", snic->shost->host_no); - if (!snic_glob->stats_root) { - SNIC_DBG("snic_stats root doesn't exist\n"); - - return rc; - } - - de = debugfs_create_dir(name, snic_glob->stats_root); - if (!de) { - SNIC_DBG("Cannot create host directory\n"); - - return rc; - } - snic->stats_host = de; - - de = debugfs_create_file("stats", - S_IFREG|S_IRUGO, - snic->stats_host, - snic, - &snic_stats_fops); - if (!de) { - SNIC_DBG("Cannot create host's stats file\n"); - - return rc; - } - snic->stats_file = de; - - de = debugfs_create_file("reset_stats", - S_IFREG|S_IRUGO|S_IWUSR, - snic->stats_host, - snic, - &snic_reset_stats_fops); - if (!de) { - SNIC_DBG("Cannot create host's reset_stats file\n"); + snic->stats_host = debugfs_create_dir(name, snic_glob->stats_root); - return rc; - } - snic->reset_stats_file = de; - rc = 0; + snic->stats_file = debugfs_create_file("stats", S_IFREG|S_IRUGO, + snic->stats_host, snic, + &snic_stats_fops); - return rc; -} /* end of snic_stats_debugfs_init */ + snic->reset_stats_file = debugfs_create_file("reset_stats", + S_IFREG|S_IRUGO|S_IWUSR, + snic->stats_host, snic, + &snic_reset_stats_fops); +} /* * snic_stats_debugfs_remove - Tear down debugfs infrastructure of stats @@ -517,46 +464,18 @@ static const struct file_operations snic_trc_fops = { * snic_trc_debugfs_init : creates trace/tracing_enable files for trace * under debugfs */ -int -snic_trc_debugfs_init(void) +void snic_trc_debugfs_init(void) { - struct dentry *de = NULL; - int ret = -1; - - if (!snic_glob->trc_root) { - SNIC_ERR("Debugfs root directory for snic doesn't exist.\n"); - - return ret; - } - - de = debugfs_create_bool("tracing_enable", - S_IFREG | S_IRUGO | S_IWUSR, - snic_glob->trc_root, - &snic_glob->trc.enable); - - if (!de) { - SNIC_ERR("Can't create trace_enable file.\n"); - - return ret; - } - snic_glob->trc.trc_enable = de; - - de = debugfs_create_file("trace", - S_IFREG | S_IRUGO | S_IWUSR, - snic_glob->trc_root, - NULL, - &snic_trc_fops); - - if (!de) { - SNIC_ERR("Cannot create trace file.\n"); - - return ret; - } - snic_glob->trc.trc_file = de; - ret = 0; - - return ret; -} /* end of snic_trc_debugfs_init */ + snic_glob->trc.trc_enable = debugfs_create_bool("tracing_enable", + S_IFREG | S_IRUGO | S_IWUSR, + snic_glob->trc_root, + &snic_glob->trc.enable); + + snic_glob->trc.trc_file = debugfs_create_file("trace", + S_IFREG | S_IRUGO | S_IWUSR, + snic_glob->trc_root, NULL, + &snic_trc_fops); +} /* * snic_trc_debugfs_term : cleans up the files created for trace under debugfs diff --git a/drivers/scsi/snic/snic_main.c b/drivers/scsi/snic/snic_main.c index 5e824fd6047a..14f4ce665e58 100644 --- a/drivers/scsi/snic/snic_main.c +++ b/drivers/scsi/snic/snic_main.c @@ -397,12 +397,7 @@ snic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn)); #ifdef CONFIG_SCSI_SNIC_DEBUG_FS /* Per snic debugfs init */ - ret = snic_stats_debugfs_init(snic); - if (ret) { - SNIC_HOST_ERR(snic->shost, - "Failed to initialize debugfs stats\n"); - snic_stats_debugfs_remove(snic); - } + snic_stats_debugfs_init(snic); #endif /* Setup PCI Resources */ @@ -850,12 +845,7 @@ snic_global_data_init(void) #ifdef CONFIG_SCSI_SNIC_DEBUG_FS /* Debugfs related Initialization */ /* Create debugfs entries for snic */ - ret = snic_debugfs_init(); - if (ret < 0) { - SNIC_ERR("Failed to create sysfs dir for tracing and stats.\n"); - snic_debugfs_term(); - /* continue even if it fails */ - } + snic_debugfs_init(); /* Trace related Initialization */ /* Allocate memory for trace buffer */ diff --git a/drivers/scsi/snic/snic_stats.h b/drivers/scsi/snic/snic_stats.h index fd1066b1cad5..faf0cb601954 100644 --- a/drivers/scsi/snic/snic_stats.h +++ b/drivers/scsi/snic/snic_stats.h @@ -99,7 +99,7 @@ struct snic_stats { atomic64_t io_cmpl_skip; }; -int snic_stats_debugfs_init(struct snic *); +void snic_stats_debugfs_init(struct snic *); void snic_stats_debugfs_remove(struct snic *); /* Auxillary function to update active IO counter */ diff --git a/drivers/scsi/snic/snic_trc.c b/drivers/scsi/snic/snic_trc.c index 458eaba24c78..f23fe2f88438 100644 --- a/drivers/scsi/snic/snic_trc.c +++ b/drivers/scsi/snic/snic_trc.c @@ -138,12 +138,7 @@ snic_trc_init(void) trc->buf = (struct snic_trc_data *) tbuf; spin_lock_init(&trc->lock); - ret = snic_trc_debugfs_init(); - if (ret) { - SNIC_ERR("Failed to create Debugfs Files.\n"); - - goto error; - } + snic_trc_debugfs_init(); trc->max_idx = (tbuf_sz / SNIC_TRC_ENTRY_SZ); trc->rd_idx = trc->wr_idx = 0; @@ -153,11 +148,6 @@ snic_trc_init(void) ret = 0; return ret; - -error: - snic_trc_free(); - - return ret; } /* end of snic_trc_init */ /* diff --git a/drivers/scsi/snic/snic_trc.h b/drivers/scsi/snic/snic_trc.h index b37f8867bfde..87dcc7457d15 100644 --- a/drivers/scsi/snic/snic_trc.h +++ b/drivers/scsi/snic/snic_trc.h @@ -53,12 +53,12 @@ struct snic_trc { int snic_trc_init(void); void snic_trc_free(void); -int snic_trc_debugfs_init(void); +void snic_trc_debugfs_init(void); void snic_trc_debugfs_term(void); struct snic_trc_data *snic_get_trc_buf(void); int snic_get_trc_data(char *buf, int buf_sz); -int snic_debugfs_init(void); +void snic_debugfs_init(void); void snic_debugfs_term(void); static inline void diff --git a/drivers/scsi/sr.c b/drivers/scsi/sr.c index 38ddbbfe5f3c..039c27c2d7b3 100644 --- a/drivers/scsi/sr.c +++ b/drivers/scsi/sr.c @@ -394,7 +394,6 @@ static blk_status_t sr_init_command(struct scsi_cmnd *SCpnt) ret = scsi_init_io(SCpnt); if (ret != BLK_STS_OK) goto out; - WARN_ON_ONCE(SCpnt != rq->special); cd = scsi_cd(rq->rq_disk); /* from here on until we're complete, any goto out diff --git a/drivers/scsi/st.c b/drivers/scsi/st.c index 7ff22d3f03e3..19c022e66d63 100644 --- a/drivers/scsi/st.c +++ b/drivers/scsi/st.c @@ -169,7 +169,7 @@ static int debugging = DEBUG; /* Remove mode bits and auto-rewind bit (7) */ #define TAPE_NR(x) ( ((iminor(x) & ~255) >> (ST_NBR_MODE_BITS + 1)) | \ - (iminor(x) & ~(-1 << ST_MODE_SHIFT)) ) + (iminor(x) & ((1 << ST_MODE_SHIFT)-1))) #define TAPE_MODE(x) ((iminor(x) & ST_MODE_MASK) >> ST_MODE_SHIFT) /* Construct the minor number from the device (d), mode (m), and non-rewind (n) data */ @@ -337,12 +337,14 @@ static void st_analyze_sense(struct st_request *SRpnt, struct st_cmdstatus *s) switch (sense[0] & 0x7f) { case 0x71: s->deferred = 1; + /* fall through */ case 0x70: s->fixed_format = 1; s->flags = sense[2] & 0xe0; break; case 0x73: s->deferred = 1; + /* fall through */ case 0x72: s->fixed_format = 0; ucp = scsi_sense_desc_find(sense, SCSI_SENSE_BUFFERSIZE, 4); @@ -2721,6 +2723,7 @@ static int st_int_ioctl(struct scsi_tape *STp, unsigned int cmd_in, unsigned lon switch (cmd_in) { case MTFSFM: chg_eof = 0; /* Changed from the FSF after this */ + /* fall through */ case MTFSF: cmd[0] = SPACE; cmd[1] = 0x01; /* Space FileMarks */ @@ -2735,6 +2738,7 @@ static int st_int_ioctl(struct scsi_tape *STp, unsigned int cmd_in, unsigned lon break; case MTBSFM: chg_eof = 0; /* Changed from the FSF after this */ + /* fall through */ case MTBSF: cmd[0] = SPACE; cmd[1] = 0x01; /* Space FileMarks */ diff --git a/drivers/scsi/ufs/Kconfig b/drivers/scsi/ufs/Kconfig index 2ddbb26d9c26..6db37cf306b0 100644 --- a/drivers/scsi/ufs/Kconfig +++ b/drivers/scsi/ufs/Kconfig @@ -99,7 +99,6 @@ config SCSI_UFS_DWC_TC_PLATFORM config SCSI_UFS_QCOM tristate "QCOM specific hooks to UFS controller platform driver" depends on SCSI_UFSHCD_PLATFORM && ARCH_QCOM - select PHY_QCOM_UFS help This selects the QCOM specific additions to UFSHCD platform driver. UFS host on QCOM needs some vendor specific configuration before diff --git a/drivers/scsi/ufs/ufs-hisi.c b/drivers/scsi/ufs/ufs-hisi.c index 452e19f8fb47..f2d3df357a97 100644 --- a/drivers/scsi/ufs/ufs-hisi.c +++ b/drivers/scsi/ufs/ufs-hisi.c @@ -66,7 +66,7 @@ static int ufs_hisi_check_hibern8(struct ufs_hba *hba) return err; } -static void ufs_hi3660_clk_init(struct ufs_hba *hba) +static void ufs_hisi_clk_init(struct ufs_hba *hba) { struct ufs_hisi_host *host = ufshcd_get_variant(hba); @@ -80,7 +80,7 @@ static void ufs_hi3660_clk_init(struct ufs_hba *hba) ufs_sys_ctrl_set_bits(host, BIT_SYSCTRL_REF_CLOCK_EN, PHY_CLK_CTRL); } -static void ufs_hi3660_soc_init(struct ufs_hba *hba) +static void ufs_hisi_soc_init(struct ufs_hba *hba) { struct ufs_hisi_host *host = ufshcd_get_variant(hba); u32 reg; @@ -139,6 +139,7 @@ static void ufs_hi3660_soc_init(struct ufs_hba *hba) static int ufs_hisi_link_startup_pre_change(struct ufs_hba *hba) { + struct ufs_hisi_host *host = ufshcd_get_variant(hba); int err; uint32_t value; uint32_t reg; @@ -153,6 +154,14 @@ static int ufs_hisi_link_startup_pre_change(struct ufs_hba *hba) ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x8121, 0x0), 0x2D); /* MPHY CBOVRCTRL3 */ ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x8122, 0x0), 0x1); + + if (host->caps & UFS_HISI_CAP_PHY10nm) { + /* MPHY CBOVRCTRL4 */ + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x8127, 0x0), 0x98); + /* MPHY CBOVRCTRL5 */ + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x8128, 0x0), 0x1); + } + /* Unipro VS_MphyCfgUpdt */ ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0xD085, 0x0), 0x1); /* MPHY RXOVRCTRL4 rx0 */ @@ -173,10 +182,21 @@ static int ufs_hisi_link_startup_pre_change(struct ufs_hba *hba) ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x8113, 0x0), 0x1); ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0xD085, 0x0), 0x1); - /* Tactive RX */ - ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x008F, 0x4), 0x7); - /* Tactive RX */ - ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x008F, 0x5), 0x7); + if (host->caps & UFS_HISI_CAP_PHY10nm) { + /* RX_Hibern8Time_Capability*/ + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x0092, 0x4), 0xA); + /* RX_Hibern8Time_Capability*/ + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x0092, 0x5), 0xA); + /* RX_Min_ActivateTime */ + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x008f, 0x4), 0xA); + /* RX_Min_ActivateTime*/ + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x008f, 0x5), 0xA); + } else { + /* Tactive RX */ + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x008F, 0x4), 0x7); + /* Tactive RX */ + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x008F, 0x5), 0x7); + } /* Gear3 Synclength */ ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x0095, 0x4), 0x4F); @@ -208,7 +228,8 @@ static int ufs_hisi_link_startup_pre_change(struct ufs_hba *hba) if (err) dev_err(hba->dev, "ufs_hisi_check_hibern8 error\n"); - ufshcd_writel(hba, UFS_HCLKDIV_NORMAL_VALUE, UFS_REG_HCLKDIV); + if (!(host->caps & UFS_HISI_CAP_PHY10nm)) + ufshcd_writel(hba, UFS_HCLKDIV_NORMAL_VALUE, UFS_REG_HCLKDIV); /* disable auto H8 */ reg = ufshcd_readl(hba, REG_AUTO_HIBERNATE_IDLE_TIMER); @@ -253,7 +274,7 @@ static int ufs_hisi_link_startup_post_change(struct ufs_hba *hba) return 0; } -static int ufs_hi3660_link_startup_notify(struct ufs_hba *hba, +static int ufs_hisi_link_startup_notify(struct ufs_hba *hba, enum ufs_notify_change_status status) { int err = 0; @@ -391,6 +412,28 @@ static void ufs_hisi_set_dev_cap(struct ufs_hisi_dev_params *hisi_param) static void ufs_hisi_pwr_change_pre_change(struct ufs_hba *hba) { + struct ufs_hisi_host *host = ufshcd_get_variant(hba); + + if (host->caps & UFS_HISI_CAP_PHY10nm) { + /* + * Boston platform need to set SaveConfigTime to 0x13, + * and change sync length to maximum value + */ + /* VS_DebugSaveConfigTime */ + ufshcd_dme_set(hba, UIC_ARG_MIB((u32)0xD0A0), 0x13); + /* g1 sync length */ + ufshcd_dme_set(hba, UIC_ARG_MIB((u32)0x1552), 0x4f); + /* g2 sync length */ + ufshcd_dme_set(hba, UIC_ARG_MIB((u32)0x1554), 0x4f); + /* g3 sync length */ + ufshcd_dme_set(hba, UIC_ARG_MIB((u32)0x1556), 0x4f); + /* PA_Hibern8Time */ + ufshcd_dme_set(hba, UIC_ARG_MIB((u32)0x15a7), 0xA); + /* PA_Tactivate */ + ufshcd_dme_set(hba, UIC_ARG_MIB((u32)0x15a8), 0xA); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0xd085, 0x0), 0x01); + } + if (hba->dev_quirks & UFS_DEVICE_QUIRK_HOST_VS_DEBUGSAVECONFIGTIME) { pr_info("ufs flash device must set VS_DebugSaveConfigTime 0x10\n"); /* VS_DebugSaveConfigTime */ @@ -429,7 +472,7 @@ static void ufs_hisi_pwr_change_pre_change(struct ufs_hba *hba) ufshcd_dme_set(hba, UIC_ARG_MIB(0xd046), 32767); } -static int ufs_hi3660_pwr_change_notify(struct ufs_hba *hba, +static int ufs_hisi_pwr_change_notify(struct ufs_hba *hba, enum ufs_notify_change_status status, struct ufs_pa_layer_attr *dev_max_params, struct ufs_pa_layer_attr *dev_req_params) @@ -567,25 +610,72 @@ static int ufs_hi3660_init(struct ufs_hba *hba) return ret; } - ufs_hi3660_clk_init(hba); + ufs_hisi_clk_init(hba); + + ufs_hisi_soc_init(hba); + + return 0; +} + +static int ufs_hi3670_init(struct ufs_hba *hba) +{ + int ret = 0; + struct device *dev = hba->dev; + struct ufs_hisi_host *host; + + ret = ufs_hisi_init_common(hba); + if (ret) { + dev_err(dev, "%s: ufs common init fail\n", __func__); + return ret; + } + + ufs_hisi_clk_init(hba); + + ufs_hisi_soc_init(hba); - ufs_hi3660_soc_init(hba); + /* Add cap for 10nm PHY variant on HI3670 SoC */ + host = ufshcd_get_variant(hba); + host->caps |= UFS_HISI_CAP_PHY10nm; return 0; } -static struct ufs_hba_variant_ops ufs_hba_hisi_vops = { +static struct ufs_hba_variant_ops ufs_hba_hi3660_vops = { .name = "hi3660", .init = ufs_hi3660_init, - .link_startup_notify = ufs_hi3660_link_startup_notify, - .pwr_change_notify = ufs_hi3660_pwr_change_notify, + .link_startup_notify = ufs_hisi_link_startup_notify, + .pwr_change_notify = ufs_hisi_pwr_change_notify, .suspend = ufs_hisi_suspend, .resume = ufs_hisi_resume, }; +static struct ufs_hba_variant_ops ufs_hba_hi3670_vops = { + .name = "hi3670", + .init = ufs_hi3670_init, + .link_startup_notify = ufs_hisi_link_startup_notify, + .pwr_change_notify = ufs_hisi_pwr_change_notify, + .suspend = ufs_hisi_suspend, + .resume = ufs_hisi_resume, +}; + +static const struct of_device_id ufs_hisi_of_match[] = { + { .compatible = "hisilicon,hi3660-ufs", .data = &ufs_hba_hi3660_vops }, + { .compatible = "hisilicon,hi3670-ufs", .data = &ufs_hba_hi3670_vops }, + {}, +}; + +MODULE_DEVICE_TABLE(of, ufs_hisi_of_match); + static int ufs_hisi_probe(struct platform_device *pdev) { - return ufshcd_pltfrm_init(pdev, &ufs_hba_hisi_vops); + const struct of_device_id *of_id; + struct ufs_hba_variant_ops *vops; + struct device *dev = &pdev->dev; + + of_id = of_match_node(ufs_hisi_of_match, dev->of_node); + vops = (struct ufs_hba_variant_ops *)of_id->data; + + return ufshcd_pltfrm_init(pdev, vops); } static int ufs_hisi_remove(struct platform_device *pdev) @@ -596,13 +686,6 @@ static int ufs_hisi_remove(struct platform_device *pdev) return 0; } -static const struct of_device_id ufs_hisi_of_match[] = { - { .compatible = "hisilicon,hi3660-ufs" }, - {}, -}; - -MODULE_DEVICE_TABLE(of, ufs_hisi_of_match); - static const struct dev_pm_ops ufs_hisi_pm_ops = { .suspend = ufshcd_pltfrm_suspend, .resume = ufshcd_pltfrm_resume, diff --git a/drivers/scsi/ufs/ufs-hisi.h b/drivers/scsi/ufs/ufs-hisi.h index 3df9cd7acc29..667dfe39b57e 100644 --- a/drivers/scsi/ufs/ufs-hisi.h +++ b/drivers/scsi/ufs/ufs-hisi.h @@ -91,6 +91,9 @@ enum { #define UFS_HISI_LIMIT_HS_RATE PA_HS_MODE_B #define UFS_HISI_LIMIT_DESIRED_MODE FAST +#define UFS_HISI_CAP_RESERVED BIT(0) +#define UFS_HISI_CAP_PHY10nm BIT(1) + struct ufs_hisi_host { struct ufs_hba *hba; void __iomem *ufs_sys_ctrl; @@ -112,4 +115,5 @@ struct ufs_hisi_host { ufs_sys_ctrl_writel((host), \ ((~(mask)) & (ufs_sys_ctrl_readl((host), (reg)))), \ (reg)) + #endif /* UFS_HISI_H_ */ diff --git a/drivers/scsi/ufs/ufs.h b/drivers/scsi/ufs/ufs.h index 6d176815e6ce..21e4ccb5ba6e 100644 --- a/drivers/scsi/ufs/ufs.h +++ b/drivers/scsi/ufs/ufs.h @@ -514,7 +514,6 @@ struct ufs_vreg { struct regulator *reg; const char *name; bool enabled; - bool unused; int min_uV; int max_uV; int min_uA; diff --git a/drivers/scsi/ufs/ufs_bsg.c b/drivers/scsi/ufs/ufs_bsg.c index 775bb4e5e36e..869e71f861d6 100644 --- a/drivers/scsi/ufs/ufs_bsg.c +++ b/drivers/scsi/ufs/ufs_bsg.c @@ -27,15 +27,11 @@ static int ufs_bsg_get_query_desc_size(struct ufs_hba *hba, int *desc_len, static int ufs_bsg_verify_query_size(struct ufs_hba *hba, unsigned int request_len, - unsigned int reply_len, - int desc_len, enum query_opcode desc_op) + unsigned int reply_len) { int min_req_len = sizeof(struct ufs_bsg_request); int min_rsp_len = sizeof(struct ufs_bsg_reply); - if (desc_op == UPIU_QUERY_OPCODE_WRITE_DESC) - min_req_len += desc_len; - if (min_req_len > request_len || min_rsp_len > reply_len) { dev_err(hba->dev, "not enough space assigned\n"); return -EINVAL; @@ -44,21 +40,16 @@ static int ufs_bsg_verify_query_size(struct ufs_hba *hba, return 0; } -static int ufs_bsg_verify_query_params(struct ufs_hba *hba, - struct ufs_bsg_request *bsg_request, - unsigned int request_len, - unsigned int reply_len, - uint8_t *desc_buff, int *desc_len, - enum query_opcode desc_op) +static int ufs_bsg_alloc_desc_buffer(struct ufs_hba *hba, struct bsg_job *job, + uint8_t **desc_buff, int *desc_len, + enum query_opcode desc_op) { + struct ufs_bsg_request *bsg_request = job->request; struct utp_upiu_query *qr; + u8 *descp; - if (desc_op == UPIU_QUERY_OPCODE_READ_DESC) { - dev_err(hba->dev, "unsupported opcode %d\n", desc_op); - return -ENOTSUPP; - } - - if (desc_op != UPIU_QUERY_OPCODE_WRITE_DESC) + if (desc_op != UPIU_QUERY_OPCODE_WRITE_DESC && + desc_op != UPIU_QUERY_OPCODE_READ_DESC) goto out; qr = &bsg_request->upiu_req.qr; @@ -67,11 +58,21 @@ static int ufs_bsg_verify_query_params(struct ufs_hba *hba, return -EINVAL; } - if (ufs_bsg_verify_query_size(hba, request_len, reply_len, *desc_len, - desc_op)) + if (*desc_len > job->request_payload.payload_len) { + dev_err(hba->dev, "Illegal desc size\n"); return -EINVAL; + } - desc_buff = (uint8_t *)(bsg_request + 1); + descp = kzalloc(*desc_len, GFP_KERNEL); + if (!descp) + return -ENOMEM; + + if (desc_op == UPIU_QUERY_OPCODE_WRITE_DESC) + sg_copy_to_buffer(job->request_payload.sg_list, + job->request_payload.sg_cnt, descp, + *desc_len); + + *desc_buff = descp; out: return 0; @@ -91,7 +92,7 @@ static int ufs_bsg_request(struct bsg_job *job) enum query_opcode desc_op = UPIU_QUERY_OPCODE_NOP; int ret; - ret = ufs_bsg_verify_query_size(hba, req_len, reply_len, 0, desc_op); + ret = ufs_bsg_verify_query_size(hba, req_len, reply_len); if (ret) goto out; @@ -101,9 +102,8 @@ static int ufs_bsg_request(struct bsg_job *job) switch (msgcode) { case UPIU_TRANSACTION_QUERY_REQ: desc_op = bsg_request->upiu_req.qr.opcode; - ret = ufs_bsg_verify_query_params(hba, bsg_request, req_len, - reply_len, desc_buff, - &desc_len, desc_op); + ret = ufs_bsg_alloc_desc_buffer(hba, job, &desc_buff, + &desc_len, desc_op); if (ret) goto out; @@ -135,11 +135,20 @@ static int ufs_bsg_request(struct bsg_job *job) break; } + if (!desc_buff) + goto out; + + if (desc_op == UPIU_QUERY_OPCODE_READ_DESC && desc_len) + bsg_reply->reply_payload_rcv_len = + sg_copy_from_buffer(job->request_payload.sg_list, + job->request_payload.sg_cnt, + desc_buff, desc_len); + + kfree(desc_buff); + out: bsg_reply->result = ret; - job->reply_len = sizeof(struct ufs_bsg_reply) + - bsg_reply->reply_payload_rcv_len; - + job->reply_len = sizeof(struct ufs_bsg_reply); bsg_job_done(job, ret, bsg_reply->reply_payload_rcv_len); return ret; diff --git a/drivers/scsi/ufs/ufs_quirks.h b/drivers/scsi/ufs/ufs_quirks.h index 5d2dfdb41a6f..a9bbd34d6b16 100644 --- a/drivers/scsi/ufs/ufs_quirks.h +++ b/drivers/scsi/ufs/ufs_quirks.h @@ -45,21 +45,6 @@ struct ufs_dev_fix { } /* - * If UFS device is having issue in processing LCC (Line Control - * Command) coming from UFS host controller then enable this quirk. - * When this quirk is enabled, host controller driver should disable - * the LCC transmission on UFS host controller (by clearing - * TX_LCC_ENABLE attribute of host to 0). - */ -#define UFS_DEVICE_QUIRK_BROKEN_LCC (1 << 0) - -/* - * Some UFS devices don't need VCCQ rail for device operations. Enabling this - * quirk for such devices will make sure that VCCQ rail is not voted. - */ -#define UFS_DEVICE_NO_VCCQ (1 << 1) - -/* * Some vendor's UFS device sends back to back NACs for the DL data frames * causing the host controller to raise the DFES error status. Sometimes * such UFS devices send back to back NAC without waiting for new @@ -85,13 +70,6 @@ struct ufs_dev_fix { #define UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS (1 << 2) /* - * Some UFS devices may not work properly after resume if the link was kept - * in off state during suspend. Enabling this quirk will not allow the - * link to be kept in off state during suspend. - */ -#define UFS_DEVICE_QUIRK_NO_LINK_OFF (1 << 3) - -/* * Few Toshiba UFS device models advertise RX_MIN_ACTIVATETIME_CAPABILITY as * 600us which may not be enough for reliable hibern8 exit hardware sequence * from UFS device. @@ -101,13 +79,6 @@ struct ufs_dev_fix { #define UFS_DEVICE_QUIRK_PA_TACTIVATE (1 << 4) /* - * Some UFS memory devices may have really low read/write throughput in - * FAST AUTO mode, enable this quirk to make sure that FAST AUTO mode is - * never enabled for such devices. - */ -#define UFS_DEVICE_NO_FASTAUTO (1 << 5) - -/* * It seems some UFS devices may keep drawing more than sleep current * (atleast for 500us) from UFS rails (especially from VCCQ rail). * To avoid this situation, add 2ms delay before putting these UFS diff --git a/drivers/scsi/ufs/ufshcd-dwc.c b/drivers/scsi/ufs/ufshcd-dwc.c index 5fd16c72207f..977b21871a5d 100644 --- a/drivers/scsi/ufs/ufshcd-dwc.c +++ b/drivers/scsi/ufs/ufshcd-dwc.c @@ -50,7 +50,7 @@ static void ufshcd_dwc_program_clk_div(struct ufs_hba *hba, u32 divider_val) /** * ufshcd_dwc_link_is_up() * Check if link is up - * @hba: private structure poitner + * @hba: private structure pointer * * Returns 0 on success, non-zero value on failure */ @@ -110,7 +110,7 @@ static int ufshcd_dwc_connection_setup(struct ufs_hba *hba) /** * ufshcd_dwc_link_startup_notify() * UFS Host DWC specific link startup sequence - * @hba: private structure poitner + * @hba: private structure pointer * @status: Callback notify status * * Returns 0 on success, non-zero value on failure diff --git a/drivers/scsi/ufs/ufshcd.c b/drivers/scsi/ufs/ufshcd.c index 2ddf24466a62..e040f9dd9ff3 100644 --- a/drivers/scsi/ufs/ufshcd.c +++ b/drivers/scsi/ufs/ufshcd.c @@ -219,12 +219,9 @@ static struct ufs_dev_fix ufs_fixups[] = { /* UFS cards deviations table */ UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL, UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM), - UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL, UFS_DEVICE_NO_VCCQ), UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL, UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS), UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL, - UFS_DEVICE_NO_FASTAUTO), - UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL, UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE), UFS_FIX(UFS_VENDOR_TOSHIBA, UFS_ANY_MODEL, UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM), @@ -232,7 +229,6 @@ static struct ufs_dev_fix ufs_fixups[] = { UFS_DEVICE_QUIRK_PA_TACTIVATE), UFS_FIX(UFS_VENDOR_TOSHIBA, "THGLF2G9D8KBADG", UFS_DEVICE_QUIRK_PA_TACTIVATE), - UFS_FIX(UFS_VENDOR_SKHYNIX, UFS_ANY_MODEL, UFS_DEVICE_NO_VCCQ), UFS_FIX(UFS_VENDOR_SKHYNIX, UFS_ANY_MODEL, UFS_DEVICE_QUIRK_HOST_PA_SAVECONFIGTIME), UFS_FIX(UFS_VENDOR_SKHYNIX, "hB8aL1" /*H28U62301AMR*/, @@ -251,7 +247,6 @@ static int ufshcd_probe_hba(struct ufs_hba *hba); static int __ufshcd_setup_clocks(struct ufs_hba *hba, bool on, bool skip_ref_clk); static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on); -static int ufshcd_set_vccq_rail_unused(struct ufs_hba *hba, bool unused); static int ufshcd_uic_hibern8_exit(struct ufs_hba *hba); static int ufshcd_uic_hibern8_enter(struct ufs_hba *hba); static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba); @@ -399,15 +394,20 @@ static void ufshcd_print_uic_err_hist(struct ufs_hba *hba, struct ufs_uic_err_reg_hist *err_hist, char *err_name) { int i; + bool found = false; for (i = 0; i < UIC_ERR_REG_HIST_LENGTH; i++) { - int p = (i + err_hist->pos - 1) % UIC_ERR_REG_HIST_LENGTH; + int p = (i + err_hist->pos) % UIC_ERR_REG_HIST_LENGTH; if (err_hist->reg[p] == 0) continue; dev_err(hba->dev, "%s[%d] = 0x%x at %lld us\n", err_name, i, err_hist->reg[p], ktime_to_us(err_hist->tstamp[p])); + found = true; } + + if (!found) + dev_err(hba->dev, "No record of %s uic errors\n", err_name); } static void ufshcd_print_host_regs(struct ufs_hba *hba) @@ -5775,6 +5775,20 @@ static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba, /* just copy the upiu response as it is */ memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu)); + if (desc_buff && desc_op == UPIU_QUERY_OPCODE_READ_DESC) { + u8 *descp = (u8 *)lrbp->ucd_rsp_ptr + sizeof(*rsp_upiu); + u16 resp_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) & + MASK_QUERY_DATA_SEG_LEN; + + if (*buff_len >= resp_len) { + memcpy(desc_buff, descp, resp_len); + *buff_len = resp_len; + } else { + dev_warn(hba->dev, "rsp size is bigger than buffer"); + *buff_len = 0; + err = -EINVAL; + } + } ufshcd_put_dev_cmd_tag(hba, tag); wake_up(&hba->dev_cmd.tag_wq); @@ -5810,11 +5824,6 @@ int ufshcd_exec_raw_upiu_cmd(struct ufs_hba *hba, int ocs_value; u8 tm_f = be32_to_cpu(req_upiu->header.dword_1) >> 16 & MASK_TM_FUNC; - if (desc_buff && desc_op != UPIU_QUERY_OPCODE_WRITE_DESC) { - err = -ENOTSUPP; - goto out; - } - switch (msgcode) { case UPIU_TRANSACTION_NOP_OUT: cmd_type = DEV_CMD_TYPE_NOP; @@ -5855,7 +5864,6 @@ int ufshcd_exec_raw_upiu_cmd(struct ufs_hba *hba, break; } -out: return err; } @@ -6825,11 +6833,6 @@ static int ufshcd_probe_hba(struct ufs_hba *hba) ufs_fixup_device_setup(hba, &card); ufshcd_tune_unipro_params(hba); - ret = ufshcd_set_vccq_rail_unused(hba, - (hba->dev_quirks & UFS_DEVICE_NO_VCCQ) ? true : false); - if (ret) - goto out; - /* UFS device is also active now */ ufshcd_set_ufs_dev_active(hba); ufshcd_force_reset_auto_bkops(hba); @@ -7013,24 +7016,13 @@ static int ufshcd_config_vreg_load(struct device *dev, struct ufs_vreg *vreg, static inline int ufshcd_config_vreg_lpm(struct ufs_hba *hba, struct ufs_vreg *vreg) { - if (!vreg) - return 0; - else if (vreg->unused) - return 0; - else - return ufshcd_config_vreg_load(hba->dev, vreg, - UFS_VREG_LPM_LOAD_UA); + return ufshcd_config_vreg_load(hba->dev, vreg, UFS_VREG_LPM_LOAD_UA); } static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba, struct ufs_vreg *vreg) { - if (!vreg) - return 0; - else if (vreg->unused) - return 0; - else - return ufshcd_config_vreg_load(hba->dev, vreg, vreg->max_uA); + return ufshcd_config_vreg_load(hba->dev, vreg, vreg->max_uA); } static int ufshcd_config_vreg(struct device *dev, @@ -7068,9 +7060,7 @@ static int ufshcd_enable_vreg(struct device *dev, struct ufs_vreg *vreg) { int ret = 0; - if (!vreg) - goto out; - else if (vreg->enabled || vreg->unused) + if (!vreg || vreg->enabled) goto out; ret = ufshcd_config_vreg(dev, vreg, true); @@ -7090,9 +7080,7 @@ static int ufshcd_disable_vreg(struct device *dev, struct ufs_vreg *vreg) { int ret = 0; - if (!vreg) - goto out; - else if (!vreg->enabled || vreg->unused) + if (!vreg || !vreg->enabled) goto out; ret = regulator_disable(vreg->reg); @@ -7198,36 +7186,6 @@ static int ufshcd_init_hba_vreg(struct ufs_hba *hba) return 0; } -static int ufshcd_set_vccq_rail_unused(struct ufs_hba *hba, bool unused) -{ - int ret = 0; - struct ufs_vreg_info *info = &hba->vreg_info; - - if (!info) - goto out; - else if (!info->vccq) - goto out; - - if (unused) { - /* shut off the rail here */ - ret = ufshcd_toggle_vreg(hba->dev, info->vccq, false); - /* - * Mark this rail as no longer used, so it doesn't get enabled - * later by mistake - */ - if (!ret) - info->vccq->unused = true; - } else { - /* - * rail should have been already enabled hence just make sure - * that unused flag is cleared. - */ - info->vccq->unused = false; - } -out: - return ret; -} - static int __ufshcd_setup_clocks(struct ufs_hba *hba, bool on, bool skip_ref_clk) { diff --git a/drivers/scsi/virtio_scsi.c b/drivers/scsi/virtio_scsi.c index 772b976e4ee4..1a6f150cd2d8 100644 --- a/drivers/scsi/virtio_scsi.c +++ b/drivers/scsi/virtio_scsi.c @@ -100,16 +100,8 @@ static inline struct Scsi_Host *virtio_scsi_host(struct virtio_device *vdev) static void virtscsi_compute_resid(struct scsi_cmnd *sc, u32 resid) { - if (!resid) - return; - - if (!scsi_bidi_cmnd(sc)) { + if (resid) scsi_set_resid(sc, resid); - return; - } - - scsi_in(sc)->resid = min(resid, scsi_in(sc)->length); - scsi_out(sc)->resid = resid - scsi_in(sc)->resid; } /** @@ -403,9 +395,9 @@ static int virtscsi_add_cmd(struct virtqueue *vq, if (sc && sc->sc_data_direction != DMA_NONE) { if (sc->sc_data_direction != DMA_FROM_DEVICE) - out = &scsi_out(sc)->table; + out = &sc->sdb.table; if (sc->sc_data_direction != DMA_TO_DEVICE) - in = &scsi_in(sc)->table; + in = &sc->sdb.table; } /* Request header. */ |