drivers/scsi/leapraid/leapraid_func.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (C) 2026 LeapIO Tech Inc.
 *
 * LeapRAID storage and RAID controller driver.
 */
#include <linux/module.h>

#include "leapraid_func.h"

static int poll_queues;
module_param(poll_queues, int, 0444);
MODULE_PARM_DESC(poll_queues,
		 "specifies the number of queues for io_uring poll mode.");

static int max_msix_vectors = LEAPRAID_INVALID_MSIX_VECTORS;
module_param(max_msix_vectors, int, 0444);
MODULE_PARM_DESC(max_msix_vectors, "max msix vectors");

static void leapraid_remove_device(struct leapraid_adapter *adapter,
				   struct leapraid_sas_dev *sas_dev);
static void leapraid_set_led(struct leapraid_adapter *adapter,
			     struct leapraid_sas_dev *sas_dev, bool on);
static void leapraid_ublk_io_dev(struct leapraid_adapter *adapter,
				 u64 sas_address,
				 struct leapraid_card_port *port);
static void leapraid_clear_cached_boot_dev(struct leapraid_adapter *adapter,
					   void *dev, u32 chnl);
static int leapraid_make_adapter_available(struct leapraid_adapter *adapter);
static void leapraid_sync_irqs_for_cleanup(struct leapraid_adapter *adapter);
static int leapraid_fw_log_init(struct leapraid_adapter *adapter);
static bool leapraid_should_skip_poll_work(struct leapraid_adapter *adapter);
static int leapraid_make_adapter_ready(struct leapraid_adapter *adapter,
				       enum reset_type type);

static noinline bool leapraid_shost_in_recovery(struct Scsi_Host *shost)
{
	enum scsi_host_state state;

	state = scsi_get_host_state(shost);
	return state == SHOST_RECOVERY ||
	       state == SHOST_CANCEL_RECOVERY ||
	       state == SHOST_DEL_RECOVERY ||
	       shost->tmf_in_progress;
}

static inline bool leapraid_is_end_dev(u32 dev_type)
{
	return (dev_type & LEAPRAID_DEVTYP_END_DEV) &&
		((dev_type & LEAPRAID_DEVTYP_SSP_TGT) ||
		 (dev_type & LEAPRAID_DEVTYP_STP_TGT) ||
		 (dev_type & LEAPRAID_DEVTYP_SATA_DEV));
}

bool leapraid_pci_removed(struct leapraid_adapter *adapter)
{
	struct pci_dev *pdev = adapter->pdev;
	u32 vendor_id;

	if (pci_bus_read_config_dword(pdev->bus,
				      pdev->devfn,
				      PCI_VENDOR_ID,
				      &vendor_id))
		return true;

	return (vendor_id & LEAPRAID_PCI_VENDOR_ID_MASK) !=
		LEAPRAID_VENDOR_ID;
}

static bool leapraid_pci_active(struct leapraid_adapter *adapter)
{
	return !(adapter->access_ctrl.pcie_recovering ||
		leapraid_pci_removed(adapter));
}

void *leapraid_get_reply_vaddr(struct leapraid_adapter *adapter, u32 rep_paddr)
{
	if (!rep_paddr)
		return NULL;

	return adapter->mem_desc.rep_msg +
		(rep_paddr - (u32)adapter->mem_desc.rep_msg_dma);
}

void *leapraid_get_task_desc(struct leapraid_adapter *adapter, u16 taskid)
{
	return adapter->mem_desc.task_desc +
			taskid * LEAPRAID_REQUEST_SIZE;
}

void *leapraid_get_sense_buffer(struct leapraid_adapter *adapter, u16 taskid)
{
	return adapter->mem_desc.sense_data +
			(taskid - 1) * SCSI_SENSE_BUFFERSIZE;
}

__le32 leapraid_get_sense_buffer_dma(struct leapraid_adapter *adapter,
				     u16 taskid)
{
	return cpu_to_le32(adapter->mem_desc.sense_data_dma +
			   ((taskid - 1) * SCSI_SENSE_BUFFERSIZE));
}

void leapraid_mask_int(struct leapraid_adapter *adapter)
{
	u32 reg;

	adapter->mask_int = 1;
	reg = leapraid_readl(&adapter->iomem_base->host_int_mask);
	reg |= LEAPRAID_TO_SYS_DB_MASK + LEAPRAID_REPLY_INT_MASK +
		LEAPRAID_RESET_IRQ_MASK;
	writel(reg, &adapter->iomem_base->host_int_mask);
	leapraid_readl(&adapter->iomem_base->host_int_mask);
}

void leapraid_unmask_int(struct leapraid_adapter *adapter)
{
	u32 reg;

	reg = leapraid_readl(&adapter->iomem_base->host_int_mask);
	reg &= ~LEAPRAID_REPLY_INT_MASK;
	writel(reg, &adapter->iomem_base->host_int_mask);
	adapter->mask_int = 0;
}

static void leapraid_overheat_suspend(struct leapraid_adapter *adapter)
{
	struct workqueue_struct *wq;
	struct Scsi_Host *shost;
	struct pci_dev *pdev;

	if (!adapter)
		return;

	pdev = adapter->pdev;
	shost = pci_get_drvdata(pdev);
	if (!shost) {
		dev_warn(&pdev->dev,
			 "Overheat suspend failed, invalid host or adapter\n");
		return;
	}

	wq = adapter->overheat_desc.fault_overheat_wq;
	if (!wq)
		return;

	if (atomic_cmpxchg(&adapter->overheat_desc.thermal_alert, 0, 1))
		return;

	queue_work(wq, &adapter->overheat_desc.fault_overheat_work);
}

static void leapraid_stop_adapter_on_fault(struct leapraid_adapter *adapter,
					   u32 db)
{
	u32 adapter_state = db & LEAPRAID_DB_MASK;
	bool fault_1 = false;
	bool fault_2 = false;

	fault_1 = adapter_state == LEAPRAID_DB_MASK;
	fault_2 = adapter_state == LEAPRAID_DB_FAULT &&
		  (db & LEAPRAID_DB_DATA_MASK) == LEAPRAID_DB_OVER_TEMPERATURE;

	if (!fault_1 && !fault_2)
		return;

	if (fault_1)
		dev_err(&adapter->pdev->dev,
			"%s: Doorbell status 0xFFFF!\n", __func__);
	else
		dev_err(&adapter->pdev->dev,
			"%s: Adapter overheating detected!\n", __func__);

	leapraid_overheat_suspend(adapter);
}

u32 leapraid_get_adapter_state(struct leapraid_adapter *adapter)
{
	u32 db;
	u32 adapter_state;

	db = leapraid_readl(&adapter->iomem_base->db);
	adapter_state = db & LEAPRAID_DB_MASK;
	leapraid_stop_adapter_on_fault(adapter, db);
	return adapter_state;
}

static bool leapraid_wait_adapter_ready(struct leapraid_adapter *adapter)
{
	u32 cur_state;
	u32 cnt;

	for (cnt = LEAPRAID_ADAPTER_READY_MAX_RETRY; cnt > 0; cnt--) {
		cur_state = leapraid_get_adapter_state(adapter);

		if (cur_state == LEAPRAID_DB_READY)
			return true;
		if (cur_state == LEAPRAID_DB_FAULT)
			break;
		usleep_range(LEAPRAID_ADAPTER_READY_SLEEP_MIN_US,
			     LEAPRAID_ADAPTER_READY_SLEEP_MAX_US);
	}

	return false;
}

static int leapraid_db_wait_int_host(struct leapraid_adapter *adapter)
{
	u32 cnt;

	for (cnt = LEAPRAID_DB_WAIT_MAX_RETRY; cnt > 0; cnt--) {
		if (leapraid_readl(&adapter->iomem_base->host_int_status) &
		    LEAPRAID_ADAPTER2HOST_DB_STATUS)
			return 0;

		udelay(LEAPRAID_DB_WAIT_DELAY_US);
	}

	return -EFAULT;
}

static int leapraid_db_wait_ack_and_clear_int(struct leapraid_adapter *adapter)
{
	u32 adapter_state;
	u32 int_status;
	u32 cnt;

	for (cnt = LEAPRAID_ADAPTER_READY_MAX_RETRY; cnt > 0; cnt--) {
		int_status =
			leapraid_readl(&adapter->iomem_base->host_int_status);

		if (int_status == 0xFFFFFFFF)
			return -EFAULT;

		if (!(int_status & LEAPRAID_HOST2ADAPTER_DB_STATUS))
			return 0;

		if (int_status & LEAPRAID_ADAPTER2HOST_DB_STATUS) {
			adapter_state = leapraid_get_adapter_state(adapter);
			if (adapter_state == LEAPRAID_DB_FAULT)
				return -EFAULT;
		}

		usleep_range(LEAPRAID_ADAPTER_READY_SLEEP_MIN_US,
			     LEAPRAID_ADAPTER_READY_SLEEP_MAX_US);
	}

	return -EFAULT;
}

static int leapraid_handshake_func(struct leapraid_adapter *adapter,
				   int req_bytes, u32 *req,
				   int rep_bytes, u16 *rep)
{
	int failed, i;

	if (leapraid_readl(&adapter->iomem_base->db) & LEAPRAID_DB_USED) {
		dev_err(&adapter->pdev->dev, "doorbell used\n");
		return -EFAULT;
	}

	if (leapraid_readl(&adapter->iomem_base->host_int_status) &
	    LEAPRAID_ADAPTER2HOST_DB_STATUS)
		writel(0, &adapter->iomem_base->host_int_status);

	writel(((LEAPRAID_FUNC_HANDSHAKE << LEAPRAID_DB_FUNC_SHIFT) |
		((req_bytes / LEAPRAID_DWORDS_BYTE_SIZE) <<
		LEAPRAID_DB_ADD_DWORDS_SHIFT)),
	       &adapter->iomem_base->db);

	if (leapraid_db_wait_int_host(adapter)) {
		dev_err(&adapter->pdev->dev, "%d: Wait db interrupt timeout\n",
			__LINE__);
		return -EFAULT;
	}

	writel(0, &adapter->iomem_base->host_int_status);

	if (leapraid_db_wait_ack_and_clear_int(adapter)) {
		dev_err(&adapter->pdev->dev, "%d: Wait ack failure\n",
			__LINE__);
		return -EFAULT;
	}

	for (i = 0, failed = 0;
	     i < req_bytes / LEAPRAID_DWORDS_BYTE_SIZE && !failed;
	     i++) {
		writel((u32)req[i], &adapter->iomem_base->db);
		if (leapraid_db_wait_ack_and_clear_int(adapter))
			failed = 1;
	}
	if (failed) {
		dev_err(&adapter->pdev->dev, "%d: Wait ack failure\n",
			__LINE__);
		return -EFAULT;
	}

	for (i = 0; i < rep_bytes / LEAPRAID_WORD_BYTE_SIZE; i++) {
		if (leapraid_db_wait_int_host(adapter)) {
			dev_err(&adapter->pdev->dev,
				"%d: Wait db interrupt timeout\n", __LINE__);
			return -EFAULT;
		}
		rep[i] = (u16)(leapraid_readl(&adapter->iomem_base->db)
			       & LEAPRAID_DB_DATA_MASK);
		writel(0, &adapter->iomem_base->host_int_status);
	}

	if (leapraid_db_wait_int_host(adapter)) {
		dev_err(&adapter->pdev->dev, "%d: Wait db interrupt timeout\n",
			__LINE__);
		return -EFAULT;
	}

	writel(0, &adapter->iomem_base->host_int_status);

	return 0;
}

int leapraid_check_adapter_is_op(struct leapraid_adapter *adapter, int wait,
				 const char *caller)
{
	int wait_count;

	for (wait_count = wait; wait_count > 0; wait_count--) {
		if (READ_ONCE(adapter->access_ctrl.host_removing)) {
			dev_warn(&adapter->pdev->dev,
				 "%s: Host is removing\n", caller);
			return -EFAULT;
		}

		if (leapraid_pci_removed(adapter)) {
			dev_warn(&adapter->pdev->dev,
				 "%s: PCI device removed\n", __func__);
			return -EFAULT;
		}

		if (leapraid_get_adapter_state(adapter) ==
		    LEAPRAID_DB_OPERATIONAL)
			return 0;

		dev_dbg(&adapter->pdev->dev,
			"%s: Wait for adapter to become op status(cnt=%d)\n",
			caller, wait - wait_count);

		ssleep(1);
	}

	dev_err(&adapter->pdev->dev,
		"%s: Adapter failed to become op state, last state=%d\n",
		caller, leapraid_get_adapter_state(adapter));

	return -EFAULT;
}

struct leapraid_io_req_tracker *leapraid_get_io_tracker_from_taskid(
		struct leapraid_adapter *adapter, u16 taskid)
{
	struct scsi_cmnd *scmd;

	if (WARN_ON(!taskid))
		return NULL;

	if (WARN_ON(taskid > adapter->shost->can_queue))
		return NULL;

	scmd = leapraid_get_scmd_from_taskid(adapter, taskid);
	if (scmd)
		return scsi_cmd_priv(scmd);

	return NULL;
}

static u8 leapraid_get_cb_idx(struct leapraid_adapter *adapter, u16 taskid)
{
	struct leapraid_driver_cmd *sp_cmd;
	u8 cb_idx = 0xFF;

	if (WARN_ON(!taskid))
		return cb_idx;

	list_for_each_entry(sp_cmd, &adapter->driver_cmds.special_cmd_list,
			    list)
		if (taskid == sp_cmd->taskid ||
		    taskid == sp_cmd->hp_taskid ||
		    taskid == sp_cmd->inter_taskid)
			return sp_cmd->cb_idx;

	WARN_ON(cb_idx == 0xFF);
	return cb_idx;
}

struct scsi_cmnd *leapraid_get_scmd_from_taskid(
					struct leapraid_adapter *adapter,
					u16 taskid)
{
	struct leapraid_scsiio_req *leap_mpi_req;
	struct leapraid_io_req_tracker *st;
	struct scsi_cmnd *scmd;
	u32 uniq_tag;

	if (taskid <= 0 || taskid > adapter->shost->can_queue)
		return NULL;

	uniq_tag = adapter->mem_desc.taskid_to_uniq_tag[taskid - 1] <<
		 BLK_MQ_UNIQUE_TAG_BITS | (taskid - 1);
	leap_mpi_req = leapraid_get_task_desc(adapter, taskid);
	if (!leap_mpi_req->dev_hdl)
		return NULL;

	scmd = scsi_host_find_tag(adapter->shost, uniq_tag);
	if (scmd) {
		st = scsi_cmd_priv(scmd);
		if (st && st->taskid == taskid)
			return scmd;
	}

	return NULL;
}

u16 leapraid_alloc_scsiio_taskid(struct leapraid_adapter *adapter,
				 struct scsi_cmnd *scmd)
{
	struct leapraid_io_req_tracker *request;
	u16 taskid;
	u32 tag = scsi_cmd_to_rq(scmd)->tag;
	u32 unique_tag;

	unique_tag = blk_mq_unique_tag(scsi_cmd_to_rq(scmd));
	tag = blk_mq_unique_tag_to_tag(unique_tag);
	adapter->mem_desc.taskid_to_uniq_tag[tag] =
		blk_mq_unique_tag_to_hwq(unique_tag);

	request = scsi_cmd_priv(scmd);
	taskid = tag + 1;
	request->taskid = taskid;
	request->scmd = scmd;
	return taskid;
}

static void leapraid_check_pending_io(struct leapraid_adapter *adapter)
{
	if (adapter->access_ctrl.shost_recovering &&
	    adapter->reset_desc.pending_io_cnt) {
		if (adapter->reset_desc.pending_io_cnt == 1)
			wake_up(&adapter->reset_desc.reset_wait_queue);
		adapter->reset_desc.pending_io_cnt--;
	}
}

static void leapraid_clear_io_tracker(
		struct leapraid_adapter *adapter,
		struct leapraid_io_req_tracker *io_tracker)
{
	if (!io_tracker)
		return;

	if (WARN_ON(io_tracker->taskid == 0))
		return;

	io_tracker->scmd = NULL;
}

static bool leapraid_is_fixed_taskid(struct leapraid_adapter *adapter,
				     u16 taskid)
{
	struct leapraid_driver_cmds *driver_cmds = &adapter->driver_cmds;

	return (taskid == driver_cmds->ctl_cmd.taskid ||
		taskid == driver_cmds->tm_cmd.hp_taskid ||
		taskid == driver_cmds->ctl_cmd.hp_taskid ||
		taskid == driver_cmds->scan_dev_cmd.inter_taskid ||
		taskid == driver_cmds->transport_cmd.inter_taskid ||
		taskid == driver_cmds->cfg_op_cmd.inter_taskid ||
		taskid == driver_cmds->enc_cmd.inter_taskid ||
		taskid == driver_cmds->notify_event_cmd.inter_taskid);
}

void leapraid_free_taskid(struct leapraid_adapter *adapter, u16 taskid)
{
	struct leapraid_io_req_tracker *io_tracker;
	void *task_desc;

	if (leapraid_is_fixed_taskid(adapter, taskid))
		return;

	if (taskid <= adapter->shost->can_queue) {
		io_tracker = leapraid_get_io_tracker_from_taskid(adapter,
								 taskid);
		if (!io_tracker) {
			leapraid_check_pending_io(adapter);
			return;
		}

		task_desc = leapraid_get_task_desc(adapter, taskid);
		memset(task_desc, 0, LEAPRAID_REQUEST_SIZE);
		leapraid_clear_io_tracker(adapter, io_tracker);
		leapraid_check_pending_io(adapter);
		adapter->mem_desc.taskid_to_uniq_tag[taskid - 1] = 0xFFFF;
	}
}

static u8 leapraid_get_msix_idx(struct leapraid_adapter *adapter,
				struct scsi_cmnd *scmd)
{
	if (scmd && adapter->shost->nr_hw_queues > 1) {
		u32 tag = blk_mq_unique_tag(scsi_cmd_to_rq(scmd));

		return blk_mq_unique_tag_to_hwq(tag);
	}
	return adapter->notification_desc.msix_cpu_map[raw_smp_processor_id()];
}

static u8 leapraid_get_and_set_msix_idx_from_taskid(
		struct leapraid_adapter *adapter, u16 taskid)
{
	struct leapraid_io_req_tracker *io_tracker = NULL;

	if (taskid <= adapter->shost->can_queue)
		io_tracker = leapraid_get_io_tracker_from_taskid(adapter,
								 taskid);

	if (!io_tracker)
		return leapraid_get_msix_idx(adapter, NULL);

	io_tracker->msix_io = leapraid_get_msix_idx(adapter, io_tracker->scmd);

	return io_tracker->msix_io;
}

void leapraid_fire_scsi_io(struct leapraid_adapter *adapter, u16 taskid,
			   u16 handle)
{
	struct leapraid_atomic_req_desc desc;

	desc.flg = LEAPRAID_REQ_DESC_FLG_SCSI_IO;
	desc.msix_idx = leapraid_get_and_set_msix_idx_from_taskid(adapter,
								  taskid);
	desc.taskid = cpu_to_le16(taskid);
	writel((__force u32)cpu_to_le32(*((u32 *)&desc)),
	       &adapter->iomem_base->atomic_req_desc_post);
}

void leapraid_fire_hpr_task(struct leapraid_adapter *adapter, u16 taskid,
			    u16 msix_task)
{
	struct leapraid_atomic_req_desc desc;

	desc.flg = LEAPRAID_REQ_DESC_FLG_HPR;
	desc.msix_idx = msix_task;
	desc.taskid = cpu_to_le16(taskid);
	writel((__force u32)cpu_to_le32(*((u32 *)&desc)),
	       &adapter->iomem_base->atomic_req_desc_post);
}

void leapraid_fire_task(struct leapraid_adapter *adapter, u16 taskid)
{
	struct leapraid_atomic_req_desc desc;

	desc.flg = LEAPRAID_REQ_DESC_FLG_DFLT_TYPE;
	desc.msix_idx = leapraid_get_and_set_msix_idx_from_taskid(adapter,
								  taskid);
	desc.taskid = cpu_to_le16(taskid);
	writel((__force u32)cpu_to_le32(*((u32 *)&desc)),
	       &adapter->iomem_base->atomic_req_desc_post);
}

void leapraid_clean_active_scsi_cmds(struct leapraid_adapter *adapter)
{
	struct leapraid_io_req_tracker *io_tracker;
	struct scsi_cmnd *scmd;
	void *task_desc;
	u16 taskid;

	leapraid_sync_irqs_for_cleanup(adapter);

	for (taskid = 1; taskid <= adapter->shost->can_queue; taskid++) {
		scmd = leapraid_get_scmd_from_taskid(adapter, taskid);
		if (!scmd)
			continue;

		io_tracker = scsi_cmd_priv(scmd);
		if (io_tracker && io_tracker->taskid == 0)
			continue;

		scsi_dma_unmap(scmd);
		task_desc = leapraid_get_task_desc(adapter, taskid);
		memset(task_desc, 0, LEAPRAID_REQUEST_SIZE);
		leapraid_clear_io_tracker(adapter, io_tracker);
		if (!leapraid_pci_active(adapter) ||
		    adapter->reset_desc.adapter_reset_results != 0 ||
		    atomic_read(&adapter->overheat_desc.thermal_alert) ||
		    adapter->access_ctrl.host_removing)
			scmd->result = DID_NO_CONNECT << 16;
		else
			scmd->result = DID_RESET << LEAPRAID_SCSI_HOST_SHIFT;
		scsi_done(scmd);
	}
}

static void leapraid_clean_active_driver_cmd(struct leapraid_driver_cmd *cmd)
{
	if (cmd->status & LEAPRAID_CMD_PENDING) {
		cmd->status |= LEAPRAID_CMD_RESET;
		complete(&cmd->done);
	}
}

static void leapraid_clean_active_driver_cmds(struct leapraid_adapter *adapter)
{
	struct leapraid_driver_cmds *driver_cmds;

	driver_cmds = &adapter->driver_cmds;
	leapraid_clean_active_driver_cmd(&driver_cmds->tm_cmd);
	leapraid_clean_active_driver_cmd(&driver_cmds->transport_cmd);
	leapraid_clean_active_driver_cmd(&driver_cmds->enc_cmd);
	leapraid_clean_active_driver_cmd(&driver_cmds->notify_event_cmd);
	leapraid_clean_active_driver_cmd(&driver_cmds->cfg_op_cmd);
	leapraid_clean_active_driver_cmd(&driver_cmds->ctl_cmd);

	if (driver_cmds->scan_dev_cmd.status & LEAPRAID_CMD_PENDING) {
		adapter->scan_dev_desc.scan_dev_failed = 1;
		driver_cmds->scan_dev_cmd.status |= LEAPRAID_CMD_RESET;
		if (adapter->scan_dev_desc.driver_loading) {
			adapter->scan_dev_desc.scan_start_failed =
				LEAPRAID_ADAPTER_STATUS_INTERNAL_ERROR;
			adapter->scan_dev_desc.scan_start = 0;
		} else {
			complete(&driver_cmds->scan_dev_cmd.done);
		}
	}
}

static void leapraid_clean_active_cmds(struct leapraid_adapter *adapter)
{
	leapraid_clean_active_driver_cmds(adapter);
	leapraid_clean_active_fw_evt(adapter);
	leapraid_clean_active_scsi_cmds(adapter);
}

static void leapraid_tgt_not_responding(struct leapraid_adapter *adapter,
					u16 hdl)
{
	struct leapraid_starget_priv *starget_priv = NULL;
	struct leapraid_sas_dev *sas_dev;
	unsigned long flags = 0;
	u32 adapter_state;

	if (adapter->access_ctrl.pcie_recovering)
		return;

	adapter_state = leapraid_get_adapter_state(adapter);
	if (adapter_state != LEAPRAID_DB_OPERATIONAL)
		return;

	if (!hdl || hdl > adapter->adapter_attr.features.max_dev_handle ||
	    test_bit(hdl, adapter->dev_topo.pd_hdls))
		return;

	spin_lock_irqsave(&adapter->dev_topo.sas_dev_lock, flags);
	sas_dev = leapraid_hold_lock_get_sas_dev_by_hdl(adapter, hdl);
	if (sas_dev && sas_dev->starget && sas_dev->starget->hostdata) {
		starget_priv = sas_dev->starget->hostdata;
		starget_priv->deleted = 1;
	}
	spin_unlock_irqrestore(&adapter->dev_topo.sas_dev_lock, flags);

	if (starget_priv)
		starget_priv->hdl = LEAPRAID_INVALID_DEV_HANDLE;

	if (sas_dev)
		leapraid_sdev_put(sas_dev);
}

static void leapraid_tgt_rst_send(struct leapraid_adapter *adapter, u16 hdl)
{
	struct leapraid_starget_priv *starget_priv = NULL;
	struct leapraid_sas_dev *sas_dev;
	struct leapraid_card_port *port;
	u64 sas_address;
	unsigned long flags;
	u32 adapter_state;

	if (adapter->access_ctrl.pcie_recovering)
		return;

	adapter_state = leapraid_get_adapter_state(adapter);
	if (adapter_state != LEAPRAID_DB_OPERATIONAL)
		return;

	if (!hdl || hdl > adapter->adapter_attr.features.max_dev_handle ||
	    test_bit(hdl, adapter->dev_topo.pd_hdls))
		return;

	spin_lock_irqsave(&adapter->dev_topo.sas_dev_lock, flags);
	sas_dev = leapraid_hold_lock_get_sas_dev_by_hdl(adapter, hdl);
	if (sas_dev && sas_dev->starget && sas_dev->starget->hostdata) {
		starget_priv = sas_dev->starget->hostdata;
		starget_priv->deleted = 1;
		sas_address = sas_dev->sas_addr;
		port = sas_dev->card_port;
	}
	spin_unlock_irqrestore(&adapter->dev_topo.sas_dev_lock, flags);

	if (starget_priv) {
		leapraid_ublk_io_dev(adapter, sas_address, port);
		starget_priv->hdl = LEAPRAID_INVALID_DEV_HANDLE;
	}
	if (sas_dev)
		leapraid_sdev_put(sas_dev);
}

static inline void leapraid_single_mpi_sg_append(
				struct leapraid_adapter *adapter,
				void *sge,
				u32 flag_and_len,
				dma_addr_t dma_addr)
{
	if (adapter->adapter_attr.use_32_dma_mask) {
		struct leapraid_sge_simple32 *sge32 = sge;

		sge32->flg_and_len =
			cpu_to_le32(flag_and_len |
				    (LEAPRAID_SGE_FLG_32 |
				     LEAPRAID_SGE_FLG_SYSTEM_ADDR) <<
				    LEAPRAID_SGE_FLG_SHIFT);
		sge32->addr = cpu_to_le32(dma_addr);
	} else {
		struct leapraid_sge_simple64 *sge64 = sge;

		sge64->flg_and_len =
			cpu_to_le32(flag_and_len |
				    (LEAPRAID_SGE_FLG_64 |
				     LEAPRAID_SGE_FLG_SYSTEM_ADDR) <<
				    LEAPRAID_SGE_FLG_SHIFT);
		sge64->addr = cpu_to_le64(dma_addr);
	}
}

static inline void leapraid_single_ieee_sg_append(void *sge, u8 flag,
						  u8 next_chain_offset,
						  u32 len,
						  dma_addr_t dma_addr)
{
	struct leapraid_chain64_ieee_sg *ieee_sg = sge;

	ieee_sg->flg = flag;
	ieee_sg->next_chain_offset = next_chain_offset;
	ieee_sg->len = cpu_to_le32(len);
	ieee_sg->addr = cpu_to_le64(dma_addr);
}

static void leapraid_build_nodata_mpi_sg(struct leapraid_adapter *adapter,
					 void *sge)
{
	leapraid_single_mpi_sg_append(adapter,
				      sge,
				      (LEAPRAID_SGE_FLG_LAST_ONE |
					LEAPRAID_SGE_FLG_EOB |
					LEAPRAID_SGE_FLG_EOL |
					LEAPRAID_SGE_FLG_SIMPLE_ONE) <<
					LEAPRAID_SGE_FLG_SHIFT,
					LEAPRAID_SGE_NO_DATA_ADDR);
}

void leapraid_build_mpi_sg(struct leapraid_adapter *adapter, void *sge,
			   dma_addr_t h2c_dma_addr, size_t h2c_size,
			   dma_addr_t c2h_dma_addr, size_t c2h_size)
{
	if (h2c_size && !c2h_size) {
		leapraid_single_mpi_sg_append(adapter,
					      sge,
					      ((LEAPRAID_SGE_FLG_SIMPLE_ONE |
						LEAPRAID_SGE_FLG_LAST_ONE |
						LEAPRAID_SGE_FLG_EOB |
						LEAPRAID_SGE_FLG_EOL |
						LEAPRAID_SGE_FLG_H2C) <<
					       LEAPRAID_SGE_FLG_SHIFT) |
					       h2c_size,
					      h2c_dma_addr);
	} else if (!h2c_size && c2h_size) {
		leapraid_single_mpi_sg_append(adapter,
					      sge,
					      ((LEAPRAID_SGE_FLG_SIMPLE_ONE |
						LEAPRAID_SGE_FLG_LAST_ONE |
						LEAPRAID_SGE_FLG_EOB |
						LEAPRAID_SGE_FLG_EOL) <<
						LEAPRAID_SGE_FLG_SHIFT) |
						c2h_size,
					      c2h_dma_addr);
	} else if (h2c_size && c2h_size) {
		leapraid_single_mpi_sg_append(adapter,
					      sge,
					      ((LEAPRAID_SGE_FLG_SIMPLE_ONE |
						LEAPRAID_SGE_FLG_EOB |
						LEAPRAID_SGE_FLG_H2C) <<
					       LEAPRAID_SGE_FLG_SHIFT) |
					      h2c_size,
					      h2c_dma_addr);
		if (adapter->adapter_attr.use_32_dma_mask)
			sge += sizeof(struct leapraid_sge_simple32);
		else
			sge += sizeof(struct leapraid_sge_simple64);
		leapraid_single_mpi_sg_append(adapter,
					      sge,
					      ((LEAPRAID_SGE_FLG_SIMPLE_ONE |
						LEAPRAID_SGE_FLG_LAST_ONE |
						LEAPRAID_SGE_FLG_EOB |
						LEAPRAID_SGE_FLG_EOL) <<
						LEAPRAID_SGE_FLG_SHIFT) |
						c2h_size,
					      c2h_dma_addr);
	} else {
		leapraid_build_nodata_mpi_sg(adapter, sge);
	}
}

void leapraid_build_ieee_nodata_sg(struct leapraid_adapter *adapter, void *sge)
{
	leapraid_single_ieee_sg_append(sge,
				       (LEAPRAID_IEEE_SGE_FLG_SIMPLE_ONE |
					LEAPRAID_IEEE_SGE_FLG_SYSTEM_ADDR |
					LEAPRAID_IEEE_SGE_FLG_EOL),
					0,
					0,
					LEAPRAID_SGE_NO_DATA_ADDR);
}

int leapraid_build_scmd_ieee_sg(struct leapraid_adapter *adapter,
				struct scsi_cmnd *scmd, u16 taskid)
{
	struct leapraid_scsiio_req *scsiio_req;
	struct leapraid_io_req_tracker *io_tracker;
	struct scatterlist *scmd_sg_cur;
	int sg_entries_left;
	void *sg_entry_cur;
	void *host_chain;
	dma_addr_t host_chain_dma;
	u8 host_chain_cursor;
	u32 sg_entries_in_cur_seg;
	u32 chain_offset_in_cur_seg;
	u32 chain_len_in_cur_seg;

	io_tracker = scsi_cmd_priv(scmd);
	scsiio_req = leapraid_get_task_desc(adapter, taskid);
	scmd_sg_cur = scsi_sglist(scmd);
	sg_entries_left = scsi_dma_map(scmd);
	if (sg_entries_left < 0)
		return -ENOMEM;
	sg_entry_cur = &scsiio_req->sgl;
	if (sg_entries_left <= LEAPRAID_SGL_INLINE_THRESHOLD)
		goto fill_last_seg;

	scsiio_req->chain_offset = LEAPRAID_CHAIN_OFFSET_DWORDS;
	leapraid_single_ieee_sg_append(sg_entry_cur,
				       LEAPRAID_IEEE_SGE_FLG_SIMPLE_ONE |
				       LEAPRAID_IEEE_SGE_FLG_SYSTEM_ADDR,
				       0, sg_dma_len(scmd_sg_cur),
				       sg_dma_address(scmd_sg_cur));
	scmd_sg_cur = sg_next(scmd_sg_cur);
	sg_entry_cur += LEAPRAID_IEEE_SGE64_ENTRY_SIZE;
	sg_entries_left--;

	host_chain_cursor = 0;
	host_chain = io_tracker->chain +
		host_chain_cursor * LEAPRAID_CHAIN_SEG_SIZE;
	host_chain_dma = io_tracker->chain_dma +
		host_chain_cursor * LEAPRAID_CHAIN_SEG_SIZE;
	host_chain_cursor += 1;
	for (;;) {
		sg_entries_in_cur_seg =
			(sg_entries_left <= LEAPRAID_MAX_SGES_IN_CHAIN) ?
				sg_entries_left : LEAPRAID_MAX_SGES_IN_CHAIN;
		chain_offset_in_cur_seg =
			(sg_entries_left == (int)sg_entries_in_cur_seg) ?
				0 : sg_entries_in_cur_seg;
		chain_len_in_cur_seg = sg_entries_in_cur_seg *
			LEAPRAID_IEEE_SGE64_ENTRY_SIZE;
		if (chain_offset_in_cur_seg)
			chain_len_in_cur_seg += LEAPRAID_IEEE_SGE64_ENTRY_SIZE;

		leapraid_single_ieee_sg_append(
			sg_entry_cur,
			LEAPRAID_IEEE_SGE_FLG_CHAIN_ONE |
			LEAPRAID_IEEE_SGE_FLG_SYSTEM_ADDR,
			chain_offset_in_cur_seg, chain_len_in_cur_seg,
			host_chain_dma);
		sg_entry_cur = host_chain;
		if (!chain_offset_in_cur_seg)
			goto fill_last_seg;

		while (sg_entries_in_cur_seg) {
			leapraid_single_ieee_sg_append(
				sg_entry_cur,
				LEAPRAID_IEEE_SGE_FLG_SIMPLE_ONE |
				LEAPRAID_IEEE_SGE_FLG_SYSTEM_ADDR,
				0, sg_dma_len(scmd_sg_cur),
				sg_dma_address(scmd_sg_cur));
			scmd_sg_cur = sg_next(scmd_sg_cur);
			sg_entry_cur += LEAPRAID_IEEE_SGE64_ENTRY_SIZE;
			sg_entries_left--;
			sg_entries_in_cur_seg--;
		}
		host_chain = io_tracker->chain +
			host_chain_cursor * LEAPRAID_CHAIN_SEG_SIZE;
		host_chain_dma = io_tracker->chain_dma +
			host_chain_cursor * LEAPRAID_CHAIN_SEG_SIZE;
		host_chain_cursor += 1;
	}

fill_last_seg:
	while (sg_entries_left > 0) {
		u32 flags = LEAPRAID_IEEE_SGE_FLG_SIMPLE_ONE |
			LEAPRAID_IEEE_SGE_FLG_SYSTEM_ADDR;
		if (sg_entries_left == 1)
			flags |= LEAPRAID_IEEE_SGE_FLG_EOL;
		leapraid_single_ieee_sg_append(sg_entry_cur, flags,
					       0, sg_dma_len(scmd_sg_cur),
					       sg_dma_address(scmd_sg_cur));
		scmd_sg_cur = sg_next(scmd_sg_cur);
		sg_entry_cur += LEAPRAID_IEEE_SGE64_ENTRY_SIZE;
		sg_entries_left--;
	}
	return 0;
}

void leapraid_build_ieee_sg(struct leapraid_adapter *adapter, void *sge,
			    dma_addr_t h2c_dma_addr, size_t h2c_size,
			    dma_addr_t c2h_dma_addr, size_t c2h_size)
{
	u32 base_flag;
	u32 flag;

	base_flag = LEAPRAID_IEEE_SGE_FLG_SIMPLE_ONE |
			LEAPRAID_IEEE_SGE_FLG_SYSTEM_ADDR;

	if (h2c_size && !c2h_size) {
		flag = base_flag | LEAPRAID_IEEE_SGE_FLG_EOL;

		leapraid_single_ieee_sg_append(sge,
					       flag,
					       0,
					       h2c_size,
					       h2c_dma_addr);
	} else if (!h2c_size && c2h_size) {
		flag = base_flag | LEAPRAID_IEEE_SGE_FLG_EOL;

		leapraid_single_ieee_sg_append(sge,
					       flag,
					       0,
					       c2h_size,
					       c2h_dma_addr);
	} else if (h2c_size && c2h_size) {
		leapraid_single_ieee_sg_append(sge,
					       base_flag,
					       0,
					       h2c_size,
					       h2c_dma_addr);
		sge += LEAPRAID_IEEE_SGE64_ENTRY_SIZE;

		flag = base_flag | LEAPRAID_IEEE_SGE_FLG_EOL;
		leapraid_single_ieee_sg_append(sge,
					       flag,
					       0,
					       c2h_size,
					       c2h_dma_addr);
	} else {
		leapraid_build_ieee_nodata_sg(adapter, sge);
	}
}

struct leapraid_sas_dev *leapraid_hold_lock_get_sas_dev_from_tgt(
		struct leapraid_adapter *adapter,
		struct leapraid_starget_priv *tgt_priv)
{
	assert_spin_locked(&adapter->dev_topo.sas_dev_lock);
	if (tgt_priv->sas_dev)
		leapraid_sdev_get(tgt_priv->sas_dev);

	return tgt_priv->sas_dev;
}

struct leapraid_sas_dev *leapraid_get_sas_dev_from_tgt(
		struct leapraid_adapter *adapter,
		struct leapraid_starget_priv *tgt_priv)
{
	struct leapraid_sas_dev *sas_dev;
	unsigned long flags;

	spin_lock_irqsave(&adapter->dev_topo.sas_dev_lock, flags);
	sas_dev = leapraid_hold_lock_get_sas_dev_from_tgt(adapter, tgt_priv);
	spin_unlock_irqrestore(&adapter->dev_topo.sas_dev_lock, flags);
	return sas_dev;
}

static struct leapraid_card_port *leapraid_get_port_by_id(
		struct leapraid_adapter *adapter,
		u8 port_id, bool skip_dirty)
{
	struct leapraid_card_port *port;
	struct leapraid_card_port *dirty_port = NULL;

	if (!adapter->adapter_attr.enable_mp)
		port_id = LEAPRAID_DISABLE_MP_PORT_ID;

	list_for_each_entry(port, &adapter->dev_topo.card_port_list, list) {
		if (port->port_id != port_id)
			continue;

		if (!(port->flg & LEAPRAID_CARD_PORT_FLG_DIRTY))
			return port;

		if (skip_dirty && !dirty_port)
			dirty_port = port;
	}

	if (dirty_port)
		return dirty_port;

	if (unlikely(!adapter->adapter_attr.enable_mp)) {
		port = kzalloc_obj(*port, GFP_ATOMIC);
		if (!port) {
			dev_warn(&adapter->pdev->dev,
				 "%s: Failed to alloc port\n", __func__);
			return NULL;
		}

		port->port_id = LEAPRAID_DISABLE_MP_PORT_ID;
		list_add_tail(&port->list, &adapter->dev_topo.card_port_list);
		return port;
	}

	return NULL;
}

struct leapraid_vphy *leapraid_get_vphy_by_phy(struct leapraid_card_port *port,
					       u32 phy_seq_num)
{
	struct leapraid_vphy *vphy;

	if (!port || !port->vphys_mask)
		return NULL;

	list_for_each_entry(vphy, &port->vphys_list, list) {
		if (vphy->phy_mask & BIT(phy_seq_num))
			return vphy;
	}

	return NULL;
}

struct leapraid_sas_dev *leapraid_hold_lock_get_sas_dev_by_addr_and_rphy(
		struct leapraid_adapter *adapter,
		u64 sas_address,
		struct sas_rphy *rphy)
{
	struct leapraid_sas_dev *sas_dev;

	assert_spin_locked(&adapter->dev_topo.sas_dev_lock);
	list_for_each_entry(sas_dev, &adapter->dev_topo.sas_dev_list, list)
		if (sas_dev->sas_addr == sas_address &&
		    sas_dev->rphy == rphy) {
			leapraid_sdev_get(sas_dev);
			return sas_dev;
		}

	list_for_each_entry(sas_dev, &adapter->dev_topo.sas_dev_init_list,
			    list)
		if (sas_dev->sas_addr == sas_address &&
		    sas_dev->rphy == rphy) {
			leapraid_sdev_get(sas_dev);
			return sas_dev;
		}

	return NULL;
}

struct leapraid_sas_dev *leapraid_hold_lock_get_sas_dev_by_addr(
		struct leapraid_adapter *adapter,
		u64 sas_address, struct leapraid_card_port *port)
{
	struct leapraid_sas_dev *sas_dev;

	if (!port) {
		dev_warn(&adapter->pdev->dev, "%s: Invalid port\n", __func__);
		return NULL;
	}

	assert_spin_locked(&adapter->dev_topo.sas_dev_lock);
	list_for_each_entry(sas_dev, &adapter->dev_topo.sas_dev_list, list)
		if (sas_dev->sas_addr == sas_address &&
		    sas_dev->card_port == port) {
			leapraid_sdev_get(sas_dev);
			return sas_dev;
		}

	list_for_each_entry(sas_dev, &adapter->dev_topo.sas_dev_init_list,
			    list)
		if (sas_dev->sas_addr == sas_address &&
		    sas_dev->card_port == port) {
			leapraid_sdev_get(sas_dev);
			return sas_dev;
		}

	return NULL;
}

struct leapraid_sas_dev *leapraid_get_sas_dev_by_addr(
		struct leapraid_adapter *adapter,
		u64 sas_address, struct leapraid_card_port *port)
{
	struct leapraid_sas_dev *sas_dev;
	unsigned long flags;

	if (!port) {
		dev_warn(&adapter->pdev->dev, "%s: Invalid port\n", __func__);
		return NULL;
	}

	spin_lock_irqsave(&adapter->dev_topo.sas_dev_lock, flags);
	sas_dev = leapraid_hold_lock_get_sas_dev_by_addr(adapter, sas_address,
							 port);
	spin_unlock_irqrestore(&adapter->dev_topo.sas_dev_lock, flags);
	return sas_dev;
}

struct leapraid_sas_dev *leapraid_hold_lock_get_sas_dev_by_hdl(
		struct leapraid_adapter *adapter, u16 hdl)
{
	struct leapraid_sas_dev *sas_dev;

	assert_spin_locked(&adapter->dev_topo.sas_dev_lock);
	list_for_each_entry(sas_dev, &adapter->dev_topo.sas_dev_list, list)
		if (sas_dev->hdl == hdl) {
			leapraid_sdev_get(sas_dev);
			return sas_dev;
		}

	list_for_each_entry(sas_dev, &adapter->dev_topo.sas_dev_init_list,
			    list)
		if (sas_dev->hdl == hdl) {
			leapraid_sdev_get(sas_dev);
			return sas_dev;
		}

	return NULL;
}

struct leapraid_sas_dev *leapraid_get_sas_dev_by_hdl(
		struct leapraid_adapter *adapter, u16 hdl)
{
	struct leapraid_sas_dev *sas_dev;
	unsigned long flags;

	spin_lock_irqsave(&adapter->dev_topo.sas_dev_lock, flags);
	sas_dev = leapraid_hold_lock_get_sas_dev_by_hdl(adapter, hdl);
	spin_unlock_irqrestore(&adapter->dev_topo.sas_dev_lock, flags);
	return sas_dev;
}

void leapraid_sas_dev_remove(struct leapraid_adapter *adapter,
			     struct leapraid_sas_dev *sas_dev)
{
	unsigned long flags;
	bool del_from_list;

	if (!sas_dev) {
		dev_warn(&adapter->pdev->dev,
			 "%s: Invalid SAS device\n", __func__);
		return;
	}

	del_from_list = false;
	spin_lock_irqsave(&adapter->dev_topo.sas_dev_lock, flags);
	if (!list_empty(&sas_dev->list)) {
		list_del_init(&sas_dev->list);
		del_from_list = true;
	}
	spin_unlock_irqrestore(&adapter->dev_topo.sas_dev_lock, flags);

	if (del_from_list) {
		leapraid_clear_cached_boot_dev(adapter, sas_dev, 0);
		leapraid_sdev_put(sas_dev);
	}
}

static void leapraid_sas_dev_remove_by_hdl(struct leapraid_adapter *adapter,
					   u16 hdl)
{
	struct leapraid_sas_dev *sas_dev;
	unsigned long flags;
	bool del_from_list;

	if (adapter->access_ctrl.shost_recovering)
		return;

	del_from_list = false;
	spin_lock_irqsave(&adapter->dev_topo.sas_dev_lock, flags);
	sas_dev = leapraid_hold_lock_get_sas_dev_by_hdl(adapter, hdl);
	if (sas_dev && (!list_empty(&sas_dev->list))) {
		list_del_init(&sas_dev->list);
		del_from_list = true;
		leapraid_sdev_put(sas_dev);
	}
	spin_unlock_irqrestore(&adapter->dev_topo.sas_dev_lock, flags);

	if (del_from_list) {
		leapraid_remove_device(adapter, sas_dev);
		leapraid_sdev_put(sas_dev);
	}
}

void leapraid_sas_dev_remove_by_sas_address(struct leapraid_adapter *adapter,
					    u64 sas_address,
					    struct leapraid_card_port *port)
{
	struct leapraid_sas_dev *sas_dev;
	unsigned long flags;
	bool del_from_list;

	if (adapter->access_ctrl.shost_recovering)
		return;

	del_from_list = false;
	spin_lock_irqsave(&adapter->dev_topo.sas_dev_lock, flags);
	sas_dev = leapraid_hold_lock_get_sas_dev_by_addr(adapter, sas_address,
							 port);
	if (sas_dev && (!list_empty(&sas_dev->list))) {
		list_del_init(&sas_dev->list);
		del_from_list = true;
		leapraid_sdev_put(sas_dev);
	}
	spin_unlock_irqrestore(&adapter->dev_topo.sas_dev_lock, flags);

	if (del_from_list) {
		leapraid_remove_device(adapter, sas_dev);
		leapraid_sdev_put(sas_dev);
	}
}

struct leapraid_raid_volume *leapraid_raid_volume_find_by_id(
		struct leapraid_adapter *adapter, uint id, uint channel)
{
	struct leapraid_raid_volume *raid_volume;
	unsigned long flags;

	spin_lock_irqsave(&adapter->dev_topo.raid_volume_lock, flags);
	list_for_each_entry(raid_volume, &adapter->dev_topo.raid_volume_list,
			    list) {
		if (raid_volume->id == id && raid_volume->channel == channel) {
			leapraid_raid_volume_get(raid_volume);
			spin_unlock_irqrestore(
				&adapter->dev_topo.raid_volume_lock, flags);
			return raid_volume;
		}
	}
	spin_unlock_irqrestore(&adapter->dev_topo.raid_volume_lock, flags);

	return NULL;
}

struct leapraid_raid_volume *leapraid_raid_volume_find_by_hdl(
		struct leapraid_adapter *adapter, u16 hdl)
{
	struct leapraid_raid_volume *raid_volume;
	unsigned long flags;

	spin_lock_irqsave(&adapter->dev_topo.raid_volume_lock, flags);
	list_for_each_entry(raid_volume, &adapter->dev_topo.raid_volume_list,
			    list) {
		if (raid_volume->hdl == hdl) {
			leapraid_raid_volume_get(raid_volume);
			spin_unlock_irqrestore(
				&adapter->dev_topo.raid_volume_lock, flags);
			return raid_volume;
		}
	}
	spin_unlock_irqrestore(&adapter->dev_topo.raid_volume_lock, flags);

	return NULL;
}

static struct leapraid_raid_volume *leapraid_raid_volume_find_by_wwid(
		struct leapraid_adapter *adapter, u64 wwid)
{
	struct leapraid_raid_volume *raid_volume;
	unsigned long flags;

	spin_lock_irqsave(&adapter->dev_topo.raid_volume_lock, flags);
	list_for_each_entry(raid_volume, &adapter->dev_topo.raid_volume_list,
			    list) {
		if (raid_volume->wwid == wwid) {
			leapraid_raid_volume_get(raid_volume);
			spin_unlock_irqrestore(
				&adapter->dev_topo.raid_volume_lock, flags);
			return raid_volume;
		}
	}
	spin_unlock_irqrestore(&adapter->dev_topo.raid_volume_lock, flags);

	return NULL;
}

static void leapraid_raid_volume_add(struct leapraid_adapter *adapter,
				     struct leapraid_raid_volume *raid_volume)
{
	unsigned long flags;

	spin_lock_irqsave(&adapter->dev_topo.raid_volume_lock, flags);
	leapraid_raid_volume_get(raid_volume);
	list_add_tail(&raid_volume->list, &adapter->dev_topo.raid_volume_list);
	spin_unlock_irqrestore(&adapter->dev_topo.raid_volume_lock, flags);
}

void leapraid_raid_volume_remove(struct leapraid_adapter *adapter,
				 struct leapraid_raid_volume *raid_volume)
{
	unsigned long flags;
	bool del_from_list = false;

	if (!raid_volume) {
		dev_warn(&adapter->pdev->dev,
			 "%s: Invalid RAID volume\n", __func__);
		return;
	}

	spin_lock_irqsave(&adapter->dev_topo.raid_volume_lock, flags);
	if (!list_empty(&raid_volume->list)) {
		list_del_init(&raid_volume->list);
		del_from_list = true;
	}
	spin_unlock_irqrestore(&adapter->dev_topo.raid_volume_lock, flags);

	leapraid_clear_cached_boot_dev(adapter, raid_volume, RAID_CHANNEL);
	if (del_from_list)
		leapraid_raid_volume_put(raid_volume);
}

static struct leapraid_enc_node *leapraid_enc_find_by_hdl(
		struct leapraid_adapter *adapter, u16 hdl)
{
	struct leapraid_enc_node *enc_dev;

	list_for_each_entry(enc_dev, &adapter->dev_topo.enc_list, list)
		if (le16_to_cpu(enc_dev->pg0.enc_hdl) == hdl)
			return enc_dev;

	return NULL;
}

struct leapraid_topo_node *leapraid_exp_find_by_sas_address(
		struct leapraid_adapter *adapter,
		u64 sas_address, struct leapraid_card_port *port)
{
	struct leapraid_topo_node *sas_exp;

	if (!port) {
		dev_warn(&adapter->pdev->dev, "%s: Invalid port\n", __func__);
		return NULL;
	}

	list_for_each_entry(sas_exp, &adapter->dev_topo.exp_list, list)
		if (sas_exp->sas_address == sas_address &&
		    sas_exp->card_port == port)
			return sas_exp;

	dev_warn(&adapter->pdev->dev,
		 "%s: No expander found for SAS addr=0x%016llx port=%p\n",
		 __func__, (unsigned long long)sas_address, port);
	return NULL;
}

bool leapraid_scmd_find_by_tgt(struct leapraid_adapter *adapter, uint id,
			       uint channel)
{
	struct scsi_cmnd *scmd;
	int taskid;

	for (taskid = 1; taskid <= adapter->shost->can_queue; taskid++) {
		scmd = leapraid_get_scmd_from_taskid(adapter, taskid);
		if (!scmd)
			continue;

		if (scmd->device->id == id && scmd->device->channel == channel)
			return true;
	}

	return false;
}

bool leapraid_scmd_find_by_lun(struct leapraid_adapter *adapter, uint id,
			       unsigned int lun, uint channel)
{
	struct scsi_cmnd *scmd;
	int taskid;

	for (taskid = 1; taskid <= adapter->shost->can_queue; taskid++) {
		scmd = leapraid_get_scmd_from_taskid(adapter, taskid);
		if (!scmd)
			continue;

		if (scmd->device->id == id &&
		    scmd->device->channel == channel &&
		    scmd->device->lun == lun)
			return true;
	}

	return false;
}

static struct leapraid_topo_node *leapraid_exp_find_by_hdl(
		struct leapraid_adapter *adapter, u16 hdl)
{
	struct leapraid_topo_node *sas_exp;

	list_for_each_entry(sas_exp, &adapter->dev_topo.exp_list, list)
		if (sas_exp->hdl == hdl)
			return sas_exp;

	return NULL;
}

static enum leapraid_card_port_checking_flg leapraid_get_card_port_feature(
		struct leapraid_card_port *old_card_port,
		struct leapraid_card_port *card_port,
		struct leapraid_card_port_feature *feature)
{
	feature->dirty_flg =
		old_card_port->flg & LEAPRAID_CARD_PORT_FLG_DIRTY;
	feature->same_addr =
		old_card_port->sas_address == card_port->sas_address;
	feature->exact_phy =
		old_card_port->phy_mask == card_port->phy_mask;
	feature->phy_overlap =
		old_card_port->phy_mask & card_port->phy_mask;
	feature->same_port =
		old_card_port->port_id == card_port->port_id;
	feature->cur_chking_old_port = old_card_port;

	if (!feature->dirty_flg || !feature->same_addr)
		return CARD_PORT_SKIP_CHECKING;

	return CARD_PORT_FURTHER_CHECKING_NEEDED;
}

static bool leapraid_process_card_port_feature(
		struct leapraid_card_port_feature *feature)
{
	struct leapraid_card_port *old_card_port;

	old_card_port = feature->cur_chking_old_port;
	if (feature->exact_phy) {
		feature->checking_state = SAME_PORT_WITH_NOTHING_CHANGED;
		feature->expected_old_port = old_card_port;
		return true;
	}

	if (feature->phy_overlap) {
		if (feature->same_port) {
			feature->checking_state =
				SAME_PORT_WITH_PARTIALLY_CHANGED_PHYS;
			feature->expected_old_port = old_card_port;
		} else if (feature->checking_state !=
			   SAME_PORT_WITH_PARTIALLY_CHANGED_PHYS) {
			feature->checking_state =
				SAME_ADDR_WITH_PARTIALLY_CHANGED_PHYS;
			feature->expected_old_port = old_card_port;
		}
	} else if (feature->checking_state !=
		   SAME_PORT_WITH_PARTIALLY_CHANGED_PHYS &&
		   feature->checking_state !=
		   SAME_ADDR_WITH_PARTIALLY_CHANGED_PHYS) {
		feature->checking_state = SAME_ADDR_ONLY;
		feature->expected_old_port = old_card_port;
		feature->same_addr_port_count++;
	}

	return false;
}

static int leapraid_check_card_port(
		struct leapraid_adapter *adapter,
		struct leapraid_card_port *card_port,
		struct leapraid_card_port **expected_card_port,
		int *count)
{
	struct leapraid_card_port *old_card_port;
	struct leapraid_card_port_feature feature;

	*expected_card_port = NULL;
	memset(&feature, 0, sizeof(struct leapraid_card_port_feature));
	feature.expected_old_port = NULL;
	feature.same_addr_port_count = 0;
	feature.checking_state = NEW_CARD_PORT;

	list_for_each_entry(old_card_port, &adapter->dev_topo.card_port_list,
			    list) {
		if (leapraid_get_card_port_feature(old_card_port, card_port,
						   &feature))
			continue;

		if (leapraid_process_card_port_feature(&feature))
			break;
	}

	if (feature.checking_state == SAME_ADDR_ONLY)
		*count = feature.same_addr_port_count;

	*expected_card_port = feature.expected_old_port;
	return feature.checking_state;
}

static void leapraid_del_phy_part_of_anther_port(
		struct leapraid_adapter *adapter,
		struct leapraid_card_port *card_port_table, int index,
		u8 port_count, int offset)
{
	struct leapraid_topo_node *card_topo_node;
	bool found = false;
	int i;

	card_topo_node = &adapter->dev_topo.card;
	for (i = 0; i < port_count; i++) {
		if (i == index)
			continue;

		if (card_port_table[i].phy_mask & BIT(offset)) {
			leapraid_transport_detach_phy_to_port(
				adapter,
				card_topo_node,
				&card_topo_node->card_phy[offset]);
			found = true;
			break;
		}
	}

	if (!found)
		card_port_table[index].phy_mask |= BIT(offset);
}

static void leapraid_add_or_del_phys_from_existing_port(
		struct leapraid_adapter *adapter,
		struct leapraid_card_port *card_port,
		struct leapraid_card_port *card_port_table,
		int index, u8 port_count)
{
	struct leapraid_topo_node *card_topo_node;
	u32 phy_mask_diff;
	u32 offset;

	card_topo_node = &adapter->dev_topo.card;
	phy_mask_diff = card_port->phy_mask ^
		card_port_table[index].phy_mask;
	for (offset = 0; offset < adapter->dev_topo.card.phys_num; offset++) {
		if (!(phy_mask_diff & BIT(offset)))
			continue;

		if (!(card_port_table[index].phy_mask & BIT(offset))) {
			leapraid_del_phy_part_of_anther_port(adapter,
							     card_port_table,
							     index, port_count,
							     offset);
			continue;
		}

		if (card_topo_node->card_phy[offset].phy_is_assigned)
			leapraid_transport_detach_phy_to_port(
				adapter,
				card_topo_node,
				&card_topo_node->card_phy[offset]);

		leapraid_transport_attach_phy_to_port(
			adapter,
			card_topo_node,
			&card_topo_node->card_phy[offset],
			card_port->sas_address,
			card_port);
	}
}

struct leapraid_sas_dev *leapraid_get_next_sas_dev_from_init_list(
		struct leapraid_adapter *adapter)
{
	struct leapraid_sas_dev *sas_dev = NULL;
	unsigned long flags;

	spin_lock_irqsave(&adapter->dev_topo.sas_dev_lock, flags);
	if (!list_empty(&adapter->dev_topo.sas_dev_init_list)) {
		sas_dev = list_first_entry(&adapter->dev_topo.sas_dev_init_list,
					   struct leapraid_sas_dev,
					   list);
		leapraid_sdev_get(sas_dev);
	}
	spin_unlock_irqrestore(&adapter->dev_topo.sas_dev_lock, flags);
	return sas_dev;
}

static bool leapraid_check_boot_dev_internal(u64 sas_address, u64 dev_name,
					     u64 enc_lid, u16 slot,
					     struct leapraid_boot_dev *boot_dev,
					     u8 form)
{
	struct leapraid_boot_dev_format_sas_wwid *wwid;
	struct leapraid_boot_dev_format_enc_slot *es;
	struct leapraid_boot_dev_format_dev_name *dn;
	void *pg_dev;

	if (!boot_dev)
		return false;

	pg_dev = boot_dev->pg_dev;
	switch (form & LEAPRAID_BOOTDEV_FORM_MASK) {
	case LEAPRAID_BOOTDEV_FORM_SAS_WWID:
		wwid = pg_dev;

		if (!sas_address)
			return false;

		return sas_address == le64_to_cpu(wwid->sas_addr);
	case LEAPRAID_BOOTDEV_FORM_ENC_SLOT:
		es = pg_dev;

		if (!enc_lid)
			return false;

		return (enc_lid == le64_to_cpu(es->enc_lid) &&
			slot == le16_to_cpu(es->slot_num));
	case LEAPRAID_BOOTDEV_FORM_DEV_NAME:
		dn = pg_dev;

		if (!dev_name)
			return false;

		return dev_name == le64_to_cpu(dn->dev_name);
	case LEAPRAID_BOOTDEV_FORM_NONE:
	default:
		return false;
	}
}

void leapraid_boot_dev_get(void *dev, u32 chnl)
{
	if (!dev)
		return;

	if (chnl == RAID_CHANNEL)
		leapraid_raid_volume_get((struct leapraid_raid_volume *)dev);
	else
		leapraid_sdev_get((struct leapraid_sas_dev *)dev);
}

void leapraid_boot_dev_put(void *dev, u32 chnl)
{
	if (!dev)
		return;

	if (chnl == RAID_CHANNEL)
		leapraid_raid_volume_put((struct leapraid_raid_volume *)dev);
	else
		leapraid_sdev_put((struct leapraid_sas_dev *)dev);
}

static void leapraid_try_set_boot_dev(struct leapraid_boot_dev *boot_dev,
				      u64 sas_addr, u64 dev_name,
				      u64 enc_lid, u16 slot,
				      void *dev, u32 chnl)
{
	bool matched = false;

	if (boot_dev->dev)
		return;

	matched = leapraid_check_boot_dev_internal(sas_addr, dev_name, enc_lid,
						   slot, boot_dev,
						   boot_dev->form);
	if (matched) {
		leapraid_boot_dev_get(dev, chnl);
		boot_dev->dev = dev;
		boot_dev->chnl = chnl;
	}
}

static void leapraid_clear_boot_dev(struct leapraid_adapter *adapter,
				    struct leapraid_boot_dev *boot_dev,
				    void *dev, u32 chnl)
{
	void *cached_dev;
	u32 cached_chnl;
	unsigned long flags;

	spin_lock_irqsave(&adapter->boot_devs.lock, flags);
	if (boot_dev->dev != dev || boot_dev->chnl != chnl)
		goto out_unlock;

	cached_dev = boot_dev->dev;
	cached_chnl = boot_dev->chnl;
	boot_dev->dev = NULL;
	boot_dev->chnl = 0;
	spin_unlock_irqrestore(&adapter->boot_devs.lock, flags);
	leapraid_boot_dev_put(cached_dev, cached_chnl);
	return;

out_unlock:
	spin_unlock_irqrestore(&adapter->boot_devs.lock, flags);
}

static void leapraid_clear_cached_boot_dev(struct leapraid_adapter *adapter,
					   void *dev, u32 chnl)
{
	leapraid_clear_boot_dev(adapter,
				&adapter->boot_devs.requested_boot_dev,
				dev, chnl);
	leapraid_clear_boot_dev(adapter,
				&adapter->boot_devs.requested_alt_boot_dev,
				dev, chnl);
	leapraid_clear_boot_dev(adapter,
				&adapter->boot_devs.current_boot_dev,
				dev, chnl);
}

static void leapraid_check_boot_dev(struct leapraid_adapter *adapter,
				    void *dev, u32 chnl)
{
	struct leapraid_raid_volume *raid_volume;
	struct leapraid_sas_dev *sas_dev;
	u64 sas_addr;
	u64 dev_name = 0;
	u64 enc_lid = 0;
	u16 slot = 0;

	if (!adapter->scan_dev_desc.driver_loading)
		return;

	switch (chnl) {
	case RAID_CHANNEL:
		raid_volume = dev;
		sas_addr = raid_volume->wwid;
		break;
	default:
		sas_dev = dev;
		sas_addr = sas_dev->sas_addr;
		dev_name = sas_dev->dev_name;
		enc_lid = sas_dev->enc_lid;
		slot = sas_dev->slot;
		break;
	}

	leapraid_try_set_boot_dev(&adapter->boot_devs.requested_boot_dev,
				  sas_addr, dev_name, enc_lid,
				  slot, dev, chnl);
	leapraid_try_set_boot_dev(&adapter->boot_devs.requested_alt_boot_dev,
				  sas_addr, dev_name, enc_lid,
				  slot, dev, chnl);
	leapraid_try_set_boot_dev(&adapter->boot_devs.current_boot_dev,
				  sas_addr, dev_name, enc_lid,
				  slot, dev, chnl);
}

static const char *leapraid_func_name(u8 func)
{
	switch (func) {
	case LEAPRAID_FUNC_SCSIIO:
		return "SCSIIO";
	case LEAPRAID_FUNC_SCSI_TMF:
		return "SCSI_TMF";
	case LEAPRAID_FUNC_ADAPTER_INIT:
		return "ADAPTER_INIT";
	case LEAPRAID_FUNC_GET_ADAPTER_FEATURES:
		return "GET_ADAPTER_FEATURES";
	case LEAPRAID_FUNC_CONFIG_OP:
		return "CONFIG_OP";
	case LEAPRAID_FUNC_SCAN_DEV:
		return "SCAN_DEV";
	case LEAPRAID_FUNC_EVENT_NOTIFY:
		return "EVENT_NOTIFY";
	case LEAPRAID_FUNC_FW_DOWNLOAD:
		return "FW_DOWNLOAD";
	case LEAPRAID_FUNC_FW_UPLOAD:
		return "FW_UPLOAD";
	case LEAPRAID_FUNC_SCSIIO_RAID_PASSTHROUGH:
		return "SCSIIO_RAID_PASSTHROUGH";
	case LEAPRAID_FUNC_SCSI_ENC_PROCESSOR:
		return "SCSI_ENC_PROCESSOR";
	case LEAPRAID_FUNC_SMP_PASSTHROUGH:
		return "SMP_PASSTHROUGH";
	case LEAPRAID_FUNC_SAS_IO_UNIT_CTRL:
		return "SAS_IO_UNIT_CTRL";
	case LEAPRAID_FUNC_SCSIIO_SATA_PASSTHROUGH:
		return "SCSIIO_SATA_PASSTHROUGH";
	case LEAPRAID_FUNC_ADAPTER_UNIT_RESET:
		return "ADAPTER_UNIT_RESET";
	case LEAPRAID_FUNC_HANDSHAKE:
		return "HANDSHAKE";
	case LEAPRAID_FUNC_LOGBUF_INIT:
		return "LOGBUF_INIT";
	default:
		return "UNKNOWN";
	}
}

static const char *leapraid_cfg_action_name(u8 action)
{
	switch (action) {
	case LEAPRAID_CFG_ACT_PAGE_HEADER:
		return "PAGE_HEADER";
	case LEAPRAID_CFG_ACT_PAGE_READ_CUR:
		return "PAGE_READ_CUR";
	case LEAPRAID_CFG_ACT_PAGE_WRITE_CUR:
		return "PAGE_WRITE_CUR";
	default:
		return "UNKNOWN";
	}
}

static const char *leapraid_cfg_page_type_name(u8 page_type)
{
	switch (page_type) {
	case LEAPRAID_CFG_PT_IO_UNIT:
		return "IO_UNIT";
	case LEAPRAID_CFG_PT_ADAPTER:
		return "ADAPTER";
	case LEAPRAID_CFG_PT_BIOS:
		return "BIOS";
	case LEAPRAID_CFG_PT_RAID_VOLUME:
		return "RAID_VOLUME";
	case LEAPRAID_CFG_PT_MANUFACTURING:
		return "MANUFACTURING";
	case LEAPRAID_CFG_PT_RAID_PHYSDISK:
		return "RAID_PHYSDISK";
	case LEAPRAID_CFG_PT_EXTENDED:
		return "EXTENDED";
	default:
		return "UNKNOWN";
	}
}

static const char *leapraid_cfg_ext_page_type_name(u8 ext_page_type)
{
	switch (ext_page_type) {
	case LEAPRAID_CFG_EXTPT_SAS_IO_UNIT:
		return "SAS_IO_UNIT";
	case LEAPRAID_CFG_EXTPT_SAS_EXP:
		return "SAS_EXPANDER";
	case LEAPRAID_CFG_EXTPT_SAS_DEV:
		return "SAS_DEVICE";
	case LEAPRAID_CFG_EXTPT_SAS_PHY:
		return "SAS_PHY";
	case LEAPRAID_CFG_EXTPT_ENC:
		return "ENCLOSURE";
	case LEAPRAID_CFG_EXTPT_RAID_CONFIG:
		return "RAID_CONFIG";
	default:
		return "UNKNOWN";
	}
}

static const char *leapraid_sep_action_name(u8 action)
{
	switch (action) {
	case LEAPRAID_SEP_REQ_ACT_WRITE_STATUS:
		return "WRITE_STATUS";
	default:
		return "UNKNOWN";
	}
}

static const char *leapraid_sas_op_name(u8 op)
{
	switch (op) {
	case LEAPRAID_SAS_OP_PHY_LINK_RESET:
		return "PHY_LINK_RESET";
	case LEAPRAID_SAS_OP_PHY_HARD_RESET:
		return "PHY_HARD_RESET";
	case LEAPRAID_SAS_OP_SET_PARAMETER:
		return "SET_PARAMETER";
	default:
		return "UNKNOWN";
	}
}

static const char *leapraid_tm_type_name(u8 task_type)
{
	switch (task_type) {
	case LEAPRAID_TM_TASKTYPE_ABORT_TASK:
		return "ABORT_TASK";
	case LEAPRAID_TM_TASKTYPE_ABRT_TASK_SET:
		return "ABORT_TASK_SET";
	case LEAPRAID_TM_TASKTYPE_TARGET_RESET:
		return "TARGET_RESET";
	case LEAPRAID_TM_TASKTYPE_LOGICAL_UNIT_RESET:
		return "LOGICAL_UNIT_RESET";
	case LEAPRAID_TM_TASKTYPE_CLEAR_TASK_SET:
		return "CLEAR_TASK_SET";
	case LEAPRAID_TM_TASKTYPE_QUERY_TASK:
		return "QUERY_TASK";
	case LEAPRAID_TM_TASKTYPE_CLEAR_ACA:
		return "CLEAR_ACA";
	case LEAPRAID_TM_TASKTYPE_QUERY_TASK_SET:
		return "QUERY_TASK_SET";
	case LEAPRAID_TM_TASKTYPE_QUERY_ASYNC_EVENT:
		return "QUERY_ASYNC_EVENT";
	default:
		return "UNKNOWN";
	}
}

void leapraid_log_req_context(struct leapraid_adapter *adapter, u16 smid,
			      const void *req_data)
{
	const struct leapraid_req *req = req_data;

	if (!adapter || !adapter->pdev || !req_data)
		return;

	switch (req->func) {
	case LEAPRAID_FUNC_CONFIG_OP: {
		const struct leapraid_cfg_req *cfg_req = req_data;

		dev_err(&adapter->pdev->dev,
			"cfg-req: smid=%u func=0x%02x(%s) action=0x%02x(%s)\n",
			smid, req->func, leapraid_func_name(req->func),
			cfg_req->action,
			leapraid_cfg_action_name(cfg_req->action));
		dev_err(&adapter->pdev->dev,
			"cfg-req: page_type=0x%02x(%s) page_num=%u\n",
			cfg_req->header.page_type,
			leapraid_cfg_page_type_name(cfg_req->header.page_type),
			cfg_req->header.page_num);
		if (cfg_req->header.page_type == LEAPRAID_CFG_PT_EXTENDED)
			dev_err(&adapter->pdev->dev,
				"cfg-req: ext_page_type=0x%02x(%s)\n",
				cfg_req->ext_page_type,
				leapraid_cfg_ext_page_type_name(
					cfg_req->ext_page_type));
		dev_err(&adapter->pdev->dev, "cfg-req: page_addr=0x%08x\n",
			le32_to_cpu(cfg_req->page_addr));
		break;
	}
	case LEAPRAID_FUNC_SCSI_TMF: {
		const struct leapraid_scsi_tm_req *tm_req = req_data;

		dev_err(&adapter->pdev->dev,
			"scsi_tm:: smid=%u func=0x%02x(%s) task=0x%02x(%s)\n",
			smid, req->func, leapraid_func_name(req->func),
			tm_req->task_type,
			leapraid_tm_type_name(tm_req->task_type));
		dev_err(&adapter->pdev->dev,
			"scsi_tm:: dev_hdl=0x%04x task_mid=%u\n",
			le16_to_cpu(tm_req->dev_hdl),
			le16_to_cpu(tm_req->task_mid));
		break;
	}
	case LEAPRAID_FUNC_SCSI_ENC_PROCESSOR: {
		const struct leapraid_sep_req *sep_req = req_data;

		dev_err(&adapter->pdev->dev,
			"sep: smid=%u func=0x%02x(%s) action=0x%02x(%s)\n",
			smid, req->func, leapraid_func_name(req->func),
			sep_req->act,
			leapraid_sep_action_name(sep_req->act));
		dev_err(&adapter->pdev->dev,
			"sep: dev_hdl=0x%04x slot=%u enc_hdl=0x%04x\n",
			le16_to_cpu(sep_req->dev_hdl),
			le16_to_cpu(sep_req->slot),
			le16_to_cpu(sep_req->enc_hdl));
		break;
	}
	case LEAPRAID_FUNC_SAS_IO_UNIT_CTRL: {
		const struct leapraid_io_unit_ctrl_req *io_req = req_data;

		dev_err(&adapter->pdev->dev,
			"ctl_cmd: smid=%u func=0x%02x(%s) action=0x%02x(%s)\n",
			smid, req->func, leapraid_func_name(req->func),
			io_req->op, leapraid_sas_op_name(io_req->op));
		dev_err(&adapter->pdev->dev,
			"ctl_cmd: dev_hdl=0x%04x param=0x%02x\n",
			le16_to_cpu(io_req->dev_hdl),
			io_req->adapter_para);
		break;
	}
	case LEAPRAID_FUNC_SMP_PASSTHROUGH: {
		const struct leapraid_smp_passthrough_req *smp_req = req_data;

		dev_err(&adapter->pdev->dev,
			"smp_cmd: smid=%u func=0x%02x(%s) action=0x%02x\n",
			smid, req->func, leapraid_func_name(req->func),
			smp_req->passthrough_flg);
		dev_err(&adapter->pdev->dev,
			"smp_cmd: port=%u req_len=%u\n",
			smp_req->physical_port,
			le16_to_cpu(smp_req->req_data_len));
		dev_err(&adapter->pdev->dev, "smp_cmd: sas_addr=0x%016llx\n",
			(unsigned long long)le64_to_cpu(smp_req->sas_address));
		break;
	}
	default:
		dev_err(&adapter->pdev->dev,
			"cmd: smid=%u func=0x%02x(%s)\n",
			smid, req->func, leapraid_func_name(req->func));
		break;
	}
}

static void leapraid_build_and_fire_cfg_req(
		struct leapraid_adapter *adapter,
		struct leapraid_cfg_req *leap_mpi_cfgp_req,
		struct leapraid_cfg_rep *leap_mpi_cfgp_rep)
{
	struct leapraid_cfg_req *local_leap_cfg_req;
	u16 smid;

	memset(leap_mpi_cfgp_rep, 0, sizeof(struct leapraid_cfg_rep));
	memset(&adapter->driver_cmds.cfg_op_cmd.reply, 0,
	       sizeof(struct leapraid_cfg_rep));
	adapter->driver_cmds.cfg_op_cmd.status = LEAPRAID_CMD_PENDING;
	smid = adapter->driver_cmds.cfg_op_cmd.inter_taskid;
	local_leap_cfg_req = leapraid_get_task_desc(adapter, smid);
	memcpy(local_leap_cfg_req, leap_mpi_cfgp_req,
	       sizeof(struct leapraid_cfg_req));
	init_completion(&adapter->driver_cmds.cfg_op_cmd.done);
	leapraid_fire_task(adapter, smid);
	wait_for_completion_timeout(&adapter->driver_cmds.cfg_op_cmd.done,
				    LEAPRAID_CFG_OP_TIMEOUT * HZ);
}

static int leapraid_req_cfg_func(struct leapraid_adapter *adapter,
				 struct leapraid_cfg_req *leap_mpi_cfgp_req,
				 struct leapraid_cfg_rep *leap_mpi_cfgp_rep,
				 void *target_cfg_pg, void *real_cfg_pg_addr,
				 u16 target_real_cfg_pg_sz)
{
	u32 adapter_status = UINT_MAX;
	bool issue_reset = false;
	u16 smid;
	u8 retry_cnt;
	int rc;

	retry_cnt = 0;
	mutex_lock(&adapter->driver_cmds.cfg_op_cmd.mutex);
	smid = adapter->driver_cmds.cfg_op_cmd.inter_taskid;
retry:
	if (retry_cnt) {
		if (retry_cnt > LEAPRAID_CFG_REQ_RETRY_TIMES) {
			rc = -EFAULT;
			goto out_cleanup;
		}
		dev_warn(&adapter->pdev->dev,
			 "cfg-req: Retry request, cnt=%u\n", retry_cnt);
	}

	rc = leapraid_check_adapter_is_op(adapter, LEAPRAID_DB_WAIT_OP_SHORT,
					  __func__);
	if (rc) {
		dev_err(&adapter->pdev->dev,
			"cfg-req: Adapter not operational\n");
		goto out_cleanup;
	}

	leapraid_build_and_fire_cfg_req(adapter, leap_mpi_cfgp_req,
					leap_mpi_cfgp_rep);
	if (!(adapter->driver_cmds.cfg_op_cmd.status & LEAPRAID_CMD_DONE)) {
		retry_cnt++;
		if (adapter->driver_cmds.cfg_op_cmd.status &
		    LEAPRAID_CMD_RESET) {
			dev_warn(&adapter->pdev->dev,
				 "cfg-req: CMD fail due to hard reset\n");
			goto retry;
		}

		if (adapter->access_ctrl.shost_recovering ||
		    adapter->access_ctrl.pcie_recovering) {
			dev_err(&adapter->pdev->dev,
				"cfg-req: pending in %s, status=0x%x\n",
				adapter->access_ctrl.shost_recovering ?
				"shost recovery" : "pcie recovery",
				adapter->driver_cmds.cfg_op_cmd.status);
			leapraid_log_req_context(adapter, smid,
						 leap_mpi_cfgp_req);
			issue_reset = false;
			rc = -EFAULT;
		} else {
			dev_err(&adapter->pdev->dev,
				"cfg-req: timeout, status=0x%x, reset\n",
				adapter->driver_cmds.cfg_op_cmd.status);
			leapraid_log_req_context(adapter, smid,
						 leap_mpi_cfgp_req);
			issue_reset = true;
		}

		goto out_cleanup;
	}

	if (adapter->driver_cmds.cfg_op_cmd.status &
	    LEAPRAID_CMD_REPLY_VALID) {
		memcpy(leap_mpi_cfgp_rep,
		       &adapter->driver_cmds.cfg_op_cmd.reply,
		       sizeof(struct leapraid_cfg_rep));
		adapter_status = le16_to_cpu(
			leap_mpi_cfgp_rep->adapter_status) &
			LEAPRAID_ADAPTER_STATUS_MASK;
		if (adapter_status == LEAPRAID_ADAPTER_STATUS_SUCCESS &&
		    target_cfg_pg && real_cfg_pg_addr &&
		    target_real_cfg_pg_sz &&
		    leap_mpi_cfgp_req->action ==
		    LEAPRAID_CFG_ACT_PAGE_READ_CUR)
			memcpy(target_cfg_pg, real_cfg_pg_addr,
			       target_real_cfg_pg_sz);
		if (adapter_status != LEAPRAID_ADAPTER_STATUS_SUCCESS) {
			if (adapter_status !=
			    LEAPRAID_ADAPTER_STATUS_CONFIG_INVALID_PAGE)
				dev_err(&adapter->pdev->dev,
					"cfg-rep: adapter_status=0x%x\n",
					adapter_status);
			rc = -EFAULT;
		}
	} else {
		dev_err(&adapter->pdev->dev, "cfg-rep: Reply invalid\n");
		rc = -EFAULT;
	}

out_cleanup:
	adapter->driver_cmds.cfg_op_cmd.status = LEAPRAID_CMD_NOT_USED;
	mutex_unlock(&adapter->driver_cmds.cfg_op_cmd.mutex);
	if (issue_reset) {
		if (adapter->scan_dev_desc.first_scan_dev_fired) {
			dev_warn(&adapter->pdev->dev,
				 "%s:%d cfg-req: Failure, issuing reset\n",
				 __func__, __LINE__);
			leapraid_hard_reset_handler(adapter, FULL_RESET);
		} else {
			dev_warn(&adapter->pdev->dev,
				 "cfg-req: CMD fail in init, skip reset\n");
		}
		rc = -EFAULT;
	}
	return rc;
}

static int leapraid_request_cfg_pg_header(
		struct leapraid_adapter *adapter,
		struct leapraid_cfg_req *leap_mpi_cfgp_req,
		struct leapraid_cfg_rep *leap_mpi_cfgp_rep)
{
	return leapraid_req_cfg_func(adapter, leap_mpi_cfgp_req,
				     leap_mpi_cfgp_rep, NULL, NULL, 0);
}

static int leapraid_request_cfg_pg(struct leapraid_adapter *adapter,
				   struct leapraid_cfg_req *leap_mpi_cfgp_req,
				   struct leapraid_cfg_rep *leap_mpi_cfgp_rep,
				   void *target_cfg_pg, void *real_cfg_pg_addr,
				   u16 target_real_cfg_pg_sz)
{
	return leapraid_req_cfg_func(adapter, leap_mpi_cfgp_req,
				     leap_mpi_cfgp_rep, target_cfg_pg,
				     real_cfg_pg_addr, target_real_cfg_pg_sz);
}

int leapraid_op_config_page(struct leapraid_adapter *adapter,
			    void *target_cfg_pg, union cfg_param_1 cfgp1,
			    union cfg_param_2 cfgp2,
			    enum config_page_action cfg_op)
{
	struct leapraid_cfg_req leap_mpi_cfgp_req;
	struct leapraid_cfg_rep leap_mpi_cfgp_rep;
	u16 real_cfg_pg_sz;
	void *real_cfg_pg_addr;
	dma_addr_t real_cfg_pg_dma = 0;
	u32 __page_size;
	int rc;

	memset(&leap_mpi_cfgp_req, 0, sizeof(struct leapraid_cfg_req));
	leap_mpi_cfgp_req.func = LEAPRAID_FUNC_CONFIG_OP;
	leap_mpi_cfgp_req.action = LEAPRAID_CFG_ACT_PAGE_HEADER;

	switch (cfg_op) {
	case GET_BIOS_PG3:
		leap_mpi_cfgp_req.header.page_type = LEAPRAID_CFG_PT_BIOS;
		leap_mpi_cfgp_req.header.page_num =
			LEAPRAID_CFG_PAGE_NUM_BIOS3;
		__page_size = sizeof(struct leapraid_bios_page3);
		break;
	case GET_BIOS_PG2:
		leap_mpi_cfgp_req.header.page_type = LEAPRAID_CFG_PT_BIOS;
		leap_mpi_cfgp_req.header.page_num =
			LEAPRAID_CFG_PAGE_NUM_BIOS2;
		__page_size = sizeof(struct leapraid_bios_page2);
		break;
	case GET_MANUFACTURING_PG0:
		leap_mpi_cfgp_req.header.page_type =
			LEAPRAID_CFG_PT_MANUFACTURING;
		leap_mpi_cfgp_req.header.page_num =
			LEAPRAID_CFG_PAGE_NUM_MANU0;
		__page_size = sizeof(struct leapraid_manufacturing_p0);
		break;
	case GET_SAS_DEVICE_PG0:
		leap_mpi_cfgp_req.header.page_type = LEAPRAID_CFG_PT_EXTENDED;
		leap_mpi_cfgp_req.ext_page_type = LEAPRAID_CFG_EXTPT_SAS_DEV;
		leap_mpi_cfgp_req.header.page_num = LEAPRAID_CFG_PAGE_NUM_DEV0;
		__page_size = sizeof(struct leapraid_sas_dev_p0);
		break;
	case GET_SAS_IOUNIT_PG0:
		leap_mpi_cfgp_req.header.page_type = LEAPRAID_CFG_PT_EXTENDED;
		leap_mpi_cfgp_req.ext_page_type =
			LEAPRAID_CFG_EXTPT_SAS_IO_UNIT;
		leap_mpi_cfgp_req.header.page_num =
			LEAPRAID_CFG_PAGE_NUM_IOUNIT0;
		__page_size = cfgp1.size;
		break;
	case GET_SAS_IOUNIT_PG1:
		leap_mpi_cfgp_req.header.page_type = LEAPRAID_CFG_PT_EXTENDED;
		leap_mpi_cfgp_req.ext_page_type =
			LEAPRAID_CFG_EXTPT_SAS_IO_UNIT;
		leap_mpi_cfgp_req.header.page_num =
			LEAPRAID_CFG_PAGE_NUM_IOUNIT1;
		__page_size = cfgp1.size;
		break;
	case GET_SAS_EXPANDER_PG0:
		leap_mpi_cfgp_req.header.page_type = LEAPRAID_CFG_PT_EXTENDED;
		leap_mpi_cfgp_req.ext_page_type = LEAPRAID_CFG_EXTPT_SAS_EXP;
		leap_mpi_cfgp_req.header.page_num = LEAPRAID_CFG_PAGE_NUM_EXP0;
		__page_size = sizeof(struct leapraid_exp_p0);
		break;
	case GET_SAS_EXPANDER_PG1:
		leap_mpi_cfgp_req.header.page_type = LEAPRAID_CFG_PT_EXTENDED;
		leap_mpi_cfgp_req.ext_page_type = LEAPRAID_CFG_EXTPT_SAS_EXP;
		leap_mpi_cfgp_req.header.page_num = LEAPRAID_CFG_PAGE_NUM_EXP1;
		__page_size = sizeof(struct leapraid_exp_p1);
		break;
	case GET_SAS_ENCLOSURE_PG0:
		leap_mpi_cfgp_req.header.page_type = LEAPRAID_CFG_PT_EXTENDED;
		leap_mpi_cfgp_req.ext_page_type = LEAPRAID_CFG_EXTPT_ENC;
		leap_mpi_cfgp_req.header.page_num = LEAPRAID_CFG_PAGE_NUM_ENC0;
		__page_size = sizeof(struct leapraid_enc_p0);
		break;
	case GET_PHY_PG0:
		leap_mpi_cfgp_req.header.page_type = LEAPRAID_CFG_PT_EXTENDED;
		leap_mpi_cfgp_req.ext_page_type = LEAPRAID_CFG_EXTPT_SAS_PHY;
		leap_mpi_cfgp_req.header.page_num = LEAPRAID_CFG_PAGE_NUM_PHY0;
		__page_size = sizeof(struct leapraid_sas_phy_p0);
		break;
	case GET_RAID_VOLUME_PG0:
		leap_mpi_cfgp_req.header.page_type =
			LEAPRAID_CFG_PT_RAID_VOLUME;
		leap_mpi_cfgp_req.header.page_num = LEAPRAID_CFG_PAGE_NUM_VOL0;
		__page_size = cfgp1.size;
		break;
	case GET_RAID_VOLUME_PG1:
		leap_mpi_cfgp_req.header.page_type =
			LEAPRAID_CFG_PT_RAID_VOLUME;
		leap_mpi_cfgp_req.header.page_num = LEAPRAID_CFG_PAGE_NUM_VOL1;
		__page_size = sizeof(struct leapraid_raidvol_p1);
		break;
	case GET_PHY_DISK_PG0:
		leap_mpi_cfgp_req.header.page_type =
			LEAPRAID_CFG_PT_RAID_PHYSDISK;
		leap_mpi_cfgp_req.header.page_num = LEAPRAID_CFG_PAGE_NUM_PD0;
		__page_size = sizeof(struct leapraid_raidpd_p0);
		break;
	default:
		dev_err(&adapter->pdev->dev,
			"Unsupported config page action=%d!\n", cfg_op);
		return -EINVAL;
	}

	leapraid_build_nodata_mpi_sg(adapter,
				     &leap_mpi_cfgp_req.page_buf_sge);
	rc = leapraid_request_cfg_pg_header(adapter,
					    &leap_mpi_cfgp_req,
					    &leap_mpi_cfgp_rep);
	if (rc) {
		dev_err(&adapter->pdev->dev,
			"cfg-req: Header failed rc=%dn", rc);
		return rc;
	}

	if (cfg_op == GET_SAS_DEVICE_PG0 ||
	    cfg_op == GET_SAS_EXPANDER_PG0 ||
	    cfg_op == GET_SAS_ENCLOSURE_PG0 ||
	    cfg_op == GET_RAID_VOLUME_PG1)
		leap_mpi_cfgp_req.page_addr = cpu_to_le32(cfgp1.form |
							  cfgp2.handle);
	else if (cfg_op == GET_PHY_DISK_PG0)
		leap_mpi_cfgp_req.page_addr = cpu_to_le32(cfgp1.form |
							  cfgp2.form_specific);
	else if (cfg_op == GET_RAID_VOLUME_PG0)
		leap_mpi_cfgp_req.page_addr =
			cpu_to_le32(cfgp2.handle |
				    LEAPRAID_RAID_VOL_CFG_PGAD_HDL);
	else if (cfg_op == GET_SAS_EXPANDER_PG1)
		leap_mpi_cfgp_req.page_addr =
			cpu_to_le32(cfgp2.handle |
				    (cfgp1.phy_number <<
				     LEAPRAID_SAS_EXP_CFG_PGAD_PHYNUM_SHIFT) |
				    LEAPRAID_SAS_EXP_CFG_PGAD_HDL_PHY_NUM);
	else if (cfg_op == GET_PHY_PG0)
		leap_mpi_cfgp_req.page_addr = cpu_to_le32(cfgp1.phy_number |
			LEAPRAID_SAS_PHY_CFG_PGAD_PHY_NUMBER);

	leap_mpi_cfgp_req.action = LEAPRAID_CFG_ACT_PAGE_READ_CUR;

	leap_mpi_cfgp_req.header.page_num = leap_mpi_cfgp_rep.header.page_num;
	leap_mpi_cfgp_req.header.page_type =
		leap_mpi_cfgp_rep.header.page_type;
	leap_mpi_cfgp_req.header.page_len = leap_mpi_cfgp_rep.header.page_len;
	leap_mpi_cfgp_req.ext_page_len = leap_mpi_cfgp_rep.ext_page_len;
	leap_mpi_cfgp_req.ext_page_type = leap_mpi_cfgp_rep.ext_page_type;

	real_cfg_pg_sz = (leap_mpi_cfgp_req.header.page_len) ?
		leap_mpi_cfgp_req.header.page_len * sizeof(u32) :
		le16_to_cpu(leap_mpi_cfgp_rep.ext_page_len) * sizeof(u32);
	real_cfg_pg_addr = dma_alloc_coherent(&adapter->pdev->dev,
					      real_cfg_pg_sz,
					      &real_cfg_pg_dma,
					      GFP_KERNEL);
	if (!real_cfg_pg_addr)
		return -ENOMEM;

	if (leap_mpi_cfgp_req.action == LEAPRAID_CFG_ACT_PAGE_WRITE_CUR) {
		leapraid_single_mpi_sg_append(adapter,
					      &leap_mpi_cfgp_req.page_buf_sge,
					      ((LEAPRAID_SGE_FLG_SIMPLE_ONE |
						LEAPRAID_SGE_FLG_LAST_ONE |
						LEAPRAID_SGE_FLG_EOB |
						LEAPRAID_SGE_FLG_EOL |
						LEAPRAID_SGE_FLG_H2C) <<
						LEAPRAID_SGE_FLG_SHIFT) |
					       real_cfg_pg_sz,
					      real_cfg_pg_dma);
		memcpy(real_cfg_pg_addr, target_cfg_pg,
		       min_t(u16, real_cfg_pg_sz, __page_size));
	} else {
		memset(target_cfg_pg, 0, __page_size);
		leapraid_single_mpi_sg_append(adapter,
					      &leap_mpi_cfgp_req.page_buf_sge,
					      ((LEAPRAID_SGE_FLG_SIMPLE_ONE |
						LEAPRAID_SGE_FLG_LAST_ONE |
						LEAPRAID_SGE_FLG_EOB |
						LEAPRAID_SGE_FLG_EOL) <<
						LEAPRAID_SGE_FLG_SHIFT) |
					       real_cfg_pg_sz,
					      real_cfg_pg_dma);
		memset(real_cfg_pg_addr, 0,
		       min_t(u16, real_cfg_pg_sz, __page_size));
	}

	rc = leapraid_request_cfg_pg(adapter,
				     &leap_mpi_cfgp_req,
				     &leap_mpi_cfgp_rep,
				     target_cfg_pg,
				     real_cfg_pg_addr,
				     min_t(u16, real_cfg_pg_sz, __page_size));
	if (rc) {
		u32 status;

		status = le16_to_cpu(leap_mpi_cfgp_rep.adapter_status) &
				     LEAPRAID_ADAPTER_STATUS_MASK;
		if (status != LEAPRAID_ADAPTER_STATUS_CONFIG_INVALID_PAGE)
			dev_err(&adapter->pdev->dev,
				"cfg-req: rc=%d, pg_info: 0x%x, 0x%x, %d\n",
				rc, leap_mpi_cfgp_req.header.page_type,
				leap_mpi_cfgp_req.ext_page_type,
				leap_mpi_cfgp_req.header.page_num);
	}

	if (real_cfg_pg_addr)
		dma_free_coherent(&adapter->pdev->dev,
				  real_cfg_pg_sz,
				  real_cfg_pg_addr,
				  real_cfg_pg_dma);

	return rc;
}

static int leapraid_cfg_find_vol_in_page(
		struct leapraid_raid_cfg_p0 *raid_cfg_p0,
		u16 pd_hdl, u16 *vol_hdl)
{
	u16 elements = raid_cfg_p0->elements_num;
	int i;

	for (i = 0; i < elements; i++) {
		struct leapraid_raid_cfg_p0_element *elem;
		u16 type;

		elem = &raid_cfg_p0->cfg_element[i];

		type = le16_to_cpu(elem->element_flg) &
		       LEAPRAID_RAIDCFG_P0_EFLG_MASK_ELEMENT_TYPE;

		switch (type) {
		case LEAPRAID_RAIDCFG_P0_EFLG_VOL_PHYS_DISK_ELEMENT:
		case LEAPRAID_RAIDCFG_P0_EFLG_OCE_ELEMENT: {
			u16 phys_hdl;

			phys_hdl = le16_to_cpu(elem->phys_disk_dev_hdl);
			if (phys_hdl == pd_hdl) {
				*vol_hdl = le16_to_cpu(elem->vol_dev_hdl);
				return 0;
			}
			break;
		}
		case LEAPRAID_RAIDCFG_P0_EFLG_HOT_SPARE_ELEMENT:
			*vol_hdl = 0;
			return 0;

		default:
			break;
		}
	}

	return -ENOENT;
}

static int leapraid_cfg_get_volume_hdl_dispatch(
		struct leapraid_adapter *adapter,
		struct leapraid_cfg_req *cfg_req,
		struct leapraid_cfg_rep *cfg_rep,
		struct leapraid_raid_cfg_p0 *raid_cfg_p0,
		void *real_cfg_pg_addr,
		u16 real_cfg_pg_sz,
		u16 raid_cfg_p0_sz,
		u16 pd_hdl, u16 *vol_hdl)
{
	u16 adapter_status;
	int config_num;
	int rc;

	config_num = 0xFF;
	while (true) {
		cfg_req->page_addr =
			cpu_to_le32(config_num +
				    LEAPRAID_SAS_CFG_PGAD_GET_NEXT_LOOP);
		rc = leapraid_request_cfg_pg(
			adapter, cfg_req, cfg_rep,
			raid_cfg_p0, real_cfg_pg_addr,
			min_t(u16, real_cfg_pg_sz, raid_cfg_p0_sz));
		adapter_status = le16_to_cpu(cfg_rep->adapter_status) &
			LEAPRAID_ADAPTER_STATUS_MASK;
		if (rc) {
			if (adapter_status ==
			    LEAPRAID_ADAPTER_STATUS_CONFIG_INVALID_PAGE) {
				*vol_hdl = 0;
				return 0;
			}
			return rc;
		}

		if (adapter_status != LEAPRAID_ADAPTER_STATUS_SUCCESS)
			return LEAPRAID_OPERATION_FAILED;

		rc = leapraid_cfg_find_vol_in_page(raid_cfg_p0,
						   pd_hdl,
						   vol_hdl);
		if (rc != -ENOENT)
			return rc;

		config_num = raid_cfg_p0->cfg_num;
	}
	return 0;
}

int leapraid_cfg_get_volume_hdl(struct leapraid_adapter *adapter,
				u16 pd_hdl, u16 *vol_hdl)
{
	struct leapraid_raid_cfg_p0 *raid_cfg_p0;
	struct leapraid_cfg_req cfg_req;
	struct leapraid_cfg_rep cfg_rep;
	dma_addr_t real_cfg_pg_dma = 0;
	void *real_cfg_pg_addr;
	u16 real_cfg_pg_sz;
	int rc, raid_cfg_p0_sz;

	*vol_hdl = 0;
	memset(&cfg_req, 0, sizeof(struct leapraid_cfg_req));
	cfg_req.func = LEAPRAID_FUNC_CONFIG_OP;
	cfg_req.action = LEAPRAID_CFG_ACT_PAGE_HEADER;
	cfg_req.header.page_type = LEAPRAID_CFG_PT_EXTENDED;
	cfg_req.ext_page_type = LEAPRAID_CFG_EXTPT_RAID_CONFIG;
	cfg_req.header.page_num = LEAPRAID_CFG_PAGE_NUM_VOL0;

	leapraid_build_nodata_mpi_sg(adapter, &cfg_req.page_buf_sge);
	rc = leapraid_request_cfg_pg_header(adapter, &cfg_req, &cfg_rep);
	if (rc)
		return rc;

	cfg_req.action = LEAPRAID_CFG_ACT_PAGE_READ_CUR;
	raid_cfg_p0_sz = le16_to_cpu(cfg_rep.ext_page_len) *
		LEAPRAID_CFG_UNIT_SIZE;
	raid_cfg_p0 = kmalloc(raid_cfg_p0_sz, GFP_KERNEL);
	if (!raid_cfg_p0)
		return -ENOMEM;

	real_cfg_pg_sz = (cfg_req.header.page_len) ?
		cfg_req.header.page_len * LEAPRAID_CFG_UNIT_SIZE :
		le16_to_cpu(cfg_rep.ext_page_len) * LEAPRAID_CFG_UNIT_SIZE;

	real_cfg_pg_addr = dma_alloc_coherent(&adapter->pdev->dev,
					      real_cfg_pg_sz, &real_cfg_pg_dma,
					      GFP_KERNEL);
	if (!real_cfg_pg_addr) {
		rc = -ENOMEM;
		goto out_free;
	}

	memset(raid_cfg_p0, 0, raid_cfg_p0_sz);
	leapraid_single_mpi_sg_append(adapter,
				      &cfg_req.page_buf_sge,
				      ((LEAPRAID_SGE_FLG_SIMPLE_ONE |
					LEAPRAID_SGE_FLG_LAST_ONE |
					LEAPRAID_SGE_FLG_EOB |
					LEAPRAID_SGE_FLG_EOL) <<
				       LEAPRAID_SGE_FLG_SHIFT) |
				      real_cfg_pg_sz,
				      real_cfg_pg_dma);
	memset(real_cfg_pg_addr, 0,
	       min_t(u16, real_cfg_pg_sz, raid_cfg_p0_sz));

	rc = leapraid_cfg_get_volume_hdl_dispatch(adapter,
						  &cfg_req, &cfg_rep,
						  raid_cfg_p0,
						  real_cfg_pg_addr,
						  real_cfg_pg_sz,
						  raid_cfg_p0_sz,
						  pd_hdl, vol_hdl);

out_free:
	if (real_cfg_pg_addr)
		dma_free_coherent(&adapter->pdev->dev,
				  real_cfg_pg_sz, real_cfg_pg_addr,
				  real_cfg_pg_dma);
	kfree(raid_cfg_p0);
	return rc;
}

static int leapraid_get_adapter_phys(struct leapraid_adapter *adapter,
				     u8 *nr_phys)
{
	struct leapraid_sas_io_unit_p0 sas_io_unit_page0;
	union cfg_param_1 cfgp1 = {0};
	union cfg_param_2 cfgp2 = {0};
	int rc;

	*nr_phys = 0;
	cfgp1.size = sizeof(struct leapraid_sas_io_unit_p0);
	rc = leapraid_op_config_page(adapter, &sas_io_unit_page0, cfgp1,
				     cfgp2, GET_SAS_IOUNIT_PG0);
	if (rc)
		return rc;

	*nr_phys = sas_io_unit_page0.phy_num;

	return 0;
}

static int leapraid_cfg_get_number_pds(struct leapraid_adapter *adapter,
				       u16 hdl, u8 *num_pds)
{
	union cfg_param_1 cfgp1 = {0};
	union cfg_param_2 cfgp2 = {0};
	struct leapraid_raidvol_p0 raidvol_p0;
	int rc;

	*num_pds = 0;
	cfgp1.size = sizeof(struct leapraid_raidvol_p0);
	cfgp2.handle = hdl;
	rc = leapraid_op_config_page(adapter, &raidvol_p0, cfgp1,
				     cfgp2, GET_RAID_VOLUME_PG0);
	if (!rc)
		*num_pds = raidvol_p0.num_phys_disks;

	return rc;
}

int leapraid_cfg_get_volume_wwid(struct leapraid_adapter *adapter,
				 u16 vol_hdl, u64 *wwid)
{
	union cfg_param_1 cfgp1 = {0};
	union cfg_param_2 cfgp2 = {0};
	struct leapraid_raidvol_p1 raidvol_p1;
	int rc;

	*wwid = 0;
	cfgp1.form = LEAPRAID_RAID_VOL_CFG_PGAD_HDL;
	cfgp2.handle = vol_hdl;
	rc = leapraid_op_config_page(adapter, &raidvol_p1, cfgp1,
				     cfgp2, GET_RAID_VOLUME_PG1);
	if (!rc)
		*wwid = le64_to_cpu(raidvol_p1.wwid);

	return rc;
}

static int leapraid_get_sas_io_unit_page0(
		struct leapraid_adapter *adapter,
		struct leapraid_sas_io_unit_p0 *sas_io_unit_p0,
		u16 sas_iou_pg0_sz)
{
	union cfg_param_1 cfgp1 = {0};
	union cfg_param_2 cfgp2 = {0};

	cfgp1.size = sas_iou_pg0_sz;
	return leapraid_op_config_page(adapter, sas_io_unit_p0, cfgp1,
				       cfgp2, GET_SAS_IOUNIT_PG0);
}

static int leapraid_get_sas_address(struct leapraid_adapter *adapter,
				    u16 hdl, u64 *sas_address)
{
	union cfg_param_1 cfgp1 = {0};
	union cfg_param_2 cfgp2 = {0};
	struct leapraid_sas_dev_p0 sas_dev_p0;

	*sas_address = 0;
	cfgp1.form = LEAPRAID_SAS_DEV_CFG_PGAD_HDL;
	cfgp2.handle = hdl;
	if (leapraid_op_config_page(adapter, &sas_dev_p0, cfgp1,
				    cfgp2, GET_SAS_DEVICE_PG0))
		return -ENXIO;

	if (hdl <= adapter->dev_topo.card.phys_num &&
	    (!(le32_to_cpu(sas_dev_p0.dev_info) & LEAPRAID_DEVTYP_SEP)))
		*sas_address = adapter->dev_topo.card.sas_address;
	else
		*sas_address = le64_to_cpu(sas_dev_p0.sas_address);

	return 0;
}

int leapraid_get_volume_cap(struct leapraid_adapter *adapter,
			    struct leapraid_raid_volume *raid_volume)
{
	union cfg_param_1 cfgp1 = {0};
	union cfg_param_2 cfgp2 = {0};
	struct leapraid_raidvol_p0 *raidvol_p0;
	struct leapraid_sas_dev_p0 sas_dev_p0;
	struct leapraid_raidpd_p0 raidpd_p0;
	u8 num_pds;
	u16 sz;
	int rc = 0;

	if (leapraid_cfg_get_number_pds(adapter, raid_volume->hdl,
					&num_pds) || !num_pds)
		return -EFAULT;

	raid_volume->pd_num = num_pds;
	sz = offsetof(struct leapraid_raidvol_p0, phys_disk) +
		(num_pds * sizeof(struct leapraid_raidvol0_phys_disk));
	raidvol_p0 = kzalloc(sz, GFP_KERNEL);
	if (!raidvol_p0)
		return -ENOMEM;

	cfgp1.size = sz;
	cfgp2.handle = raid_volume->hdl;
	if (leapraid_op_config_page(adapter, raidvol_p0, cfgp1, cfgp2,
				    GET_RAID_VOLUME_PG0)) {
		rc = -EFAULT;
		goto out_cleanup;
	}

	raid_volume->vol_type = raidvol_p0->volume_type;
	cfgp1.form = LEAPRAID_PHYSDISK_CFG_PGAD_PHYSDISKNUM;
	cfgp2.form_specific = raidvol_p0->phys_disk[0].phys_disk_num;
	if (!(leapraid_op_config_page(adapter, &raidpd_p0, cfgp1, cfgp2,
				      GET_PHY_DISK_PG0))) {
		cfgp1.form = LEAPRAID_SAS_DEV_CFG_PGAD_HDL;
		cfgp2.handle = le16_to_cpu(raidpd_p0.dev_hdl);
		if (!(leapraid_op_config_page(adapter, &sas_dev_p0, cfgp1,
					      cfgp2, GET_SAS_DEVICE_PG0)))
			raid_volume->dev_info =
				le32_to_cpu(sas_dev_p0.dev_info);
	}

out_cleanup:
	kfree(raidvol_p0);
	return rc;
}

static bool leapraid_should_skip_poll_work(struct leapraid_adapter *adapter)
{
	unsigned long flags;
	bool skip;

	spin_lock_irqsave(&adapter->reset_desc.adapter_reset_lock, flags);
	skip = adapter->access_ctrl.shost_recovering ||
		adapter->access_ctrl.pcie_recovering ||
		adapter->access_ctrl.host_removing;
	spin_unlock_irqrestore(&adapter->reset_desc.adapter_reset_lock, flags);

	return skip;
}

static void leapraid_fw_log_work(struct work_struct *work)
{
	struct leapraid_adapter *adapter = container_of(work,
		struct leapraid_adapter, fw_log_desc.fw_log_work.work);
	struct leapraid_fw_log_info *infom;
	struct leapraid_reg_base __iomem *iomem_base;
	unsigned long flags;

	if (leapraid_should_skip_poll_work(adapter))
		goto scheduled_timer;

	infom = (struct leapraid_fw_log_info *)
		(adapter->fw_log_desc.fw_log_buffer +
		 LEAPRAID_SYS_LOG_BUF_SIZE);

	iomem_base = adapter->iomem_base;
	if (adapter->fw_log_desc.fw_log_init_flag == 0) {
		infom->user_position =
			leapraid_readl(&iomem_base->host_log_buf_pos);
		infom->adapter_position =
			leapraid_readl(&iomem_base->adapter_log_buf_pos);
		adapter->fw_log_desc.fw_log_init_flag++;
	}

	writel(infom->user_position, &iomem_base->host_log_buf_pos);
	infom->adapter_position =
		leapraid_readl(&iomem_base->adapter_log_buf_pos);

scheduled_timer:
	spin_lock_irqsave(&adapter->reset_desc.adapter_reset_lock, flags);
	if (adapter->fw_log_desc.fw_log_wq)
		queue_delayed_work(
			adapter->fw_log_desc.fw_log_wq,
			&adapter->fw_log_desc.fw_log_work,
			msecs_to_jiffies(LEAPRAID_PCIE_LOG_POLLING_INTERVAL));
	spin_unlock_irqrestore(&adapter->reset_desc.adapter_reset_lock, flags);
}

void leapraid_fw_log_stop(struct leapraid_adapter *adapter)
{
	struct workqueue_struct *wq;
	unsigned long flags;

	if (!adapter->fw_log_desc.open_pcie_trace)
		return;

	spin_lock_irqsave(&adapter->reset_desc.adapter_reset_lock, flags);
	wq = adapter->fw_log_desc.fw_log_wq;
	adapter->fw_log_desc.fw_log_wq = NULL;
	spin_unlock_irqrestore(&adapter->reset_desc.adapter_reset_lock, flags);
	if (wq) {
		if (!cancel_delayed_work_sync(&adapter->fw_log_desc
					      .fw_log_work))
			flush_workqueue(wq);
		destroy_workqueue(wq);
	}
}

void leapraid_fw_log_start(struct leapraid_adapter *adapter)
{
	unsigned long flags;

	if (!adapter->fw_log_desc.open_pcie_trace)
		return;

	if (adapter->fw_log_desc.fw_log_wq)
		return;

	INIT_DELAYED_WORK(&adapter->fw_log_desc.fw_log_work,
			  leapraid_fw_log_work);
	snprintf(adapter->fw_log_desc.fw_log_wq_name,
		 sizeof(adapter->fw_log_desc.fw_log_wq_name),
		 "poll_%s%u_fw_log",
		 LEAPRAID_DRIVER_NAME, adapter->adapter_attr.id);
	adapter->fw_log_desc.fw_log_wq =
		create_singlethread_workqueue(
			adapter->fw_log_desc.fw_log_wq_name);
	if (!adapter->fw_log_desc.fw_log_wq)
		return;

	spin_lock_irqsave(&adapter->reset_desc.adapter_reset_lock, flags);
	if (adapter->fw_log_desc.fw_log_wq)
		queue_delayed_work(
			adapter->fw_log_desc.fw_log_wq,
			&adapter->fw_log_desc.fw_log_work,
			msecs_to_jiffies(LEAPRAID_PCIE_LOG_POLLING_INTERVAL));
	spin_unlock_irqrestore(&adapter->reset_desc.adapter_reset_lock, flags);
}

static void leapraid_check_scheduled_fault_work(struct work_struct *work)
{
	struct leapraid_adapter *adapter;
	unsigned long flags;
	u32 adapter_state;
	int rc;

	adapter = container_of(work, struct leapraid_adapter,
			       reset_desc.fault_reset_work.work);

	if (leapraid_should_skip_poll_work(adapter))
		goto scheduled_timer;

	adapter_state = leapraid_get_adapter_state(adapter);
	if (adapter_state != LEAPRAID_DB_OPERATIONAL) {
		dev_info(&adapter->pdev->dev, "%s:%d: call hard_reset 0x%x\n",
			 __func__, __LINE__, adapter_state);
		rc = leapraid_hard_reset_handler(adapter, FULL_RESET);
		adapter_state = leapraid_get_adapter_state(adapter);
		if (rc && adapter_state != LEAPRAID_DB_OPERATIONAL) {
			dev_err(&adapter->pdev->dev,
				"%s: Hard reset failed, state=0x%x rc=%d\n",
				__func__, adapter_state, rc);
			return;
		}
	}

scheduled_timer:
	spin_lock_irqsave(&adapter->reset_desc.adapter_reset_lock, flags);
	if (adapter->reset_desc.fault_reset_wq)
		queue_delayed_work(
			adapter->reset_desc.fault_reset_wq,
			&adapter->reset_desc.fault_reset_work,
			msecs_to_jiffies(LEAPRAID_FAULT_POLLING_INTERVAL));
	spin_unlock_irqrestore(&adapter->reset_desc.adapter_reset_lock, flags);
}

void leapraid_check_scheduled_fault_start(struct leapraid_adapter *adapter)
{
	unsigned long flags;

	if (adapter->reset_desc.fault_reset_wq)
		return;

	INIT_DELAYED_WORK(&adapter->reset_desc.fault_reset_work,
			  leapraid_check_scheduled_fault_work);
	snprintf(adapter->reset_desc.fault_reset_wq_name,
		 sizeof(adapter->reset_desc.fault_reset_wq_name),
		 "poll_%s%u_status",
		 LEAPRAID_DRIVER_NAME, adapter->adapter_attr.id);
	adapter->reset_desc.fault_reset_wq =
		create_singlethread_workqueue(
			adapter->reset_desc.fault_reset_wq_name);
	if (!adapter->reset_desc.fault_reset_wq) {
		dev_err(&adapter->pdev->dev,
			"Create single thread workqueue failed!\n");
		return;
	}

	spin_lock_irqsave(&adapter->reset_desc.adapter_reset_lock, flags);
	if (adapter->reset_desc.fault_reset_wq)
		queue_delayed_work(
			adapter->reset_desc.fault_reset_wq,
			&adapter->reset_desc.fault_reset_work,
			msecs_to_jiffies(LEAPRAID_FAULT_POLLING_INTERVAL));
	spin_unlock_irqrestore(&adapter->reset_desc.adapter_reset_lock, flags);
}

void leapraid_check_scheduled_fault_stop(struct leapraid_adapter *adapter)
{
	struct workqueue_struct *wq;
	unsigned long flags;

	spin_lock_irqsave(&adapter->reset_desc.adapter_reset_lock, flags);
	wq = adapter->reset_desc.fault_reset_wq;
	adapter->reset_desc.fault_reset_wq = NULL;
	spin_unlock_irqrestore(&adapter->reset_desc.adapter_reset_lock, flags);

	if (!wq)
		return;

	if (!cancel_delayed_work_sync(&adapter->reset_desc.fault_reset_work))
		flush_workqueue(wq);
	destroy_workqueue(wq);
}

static void leapraid_overheat_work(struct work_struct *work)
{
	struct leapraid_overheat_desc *desc;
	struct leapraid_adapter *adapter;
	struct workqueue_struct *wq;
	struct Scsi_Host *shost;
	struct pci_dev *pdev;
	unsigned long flags;

	desc = container_of(work, struct leapraid_overheat_desc,
			    fault_overheat_work);
	adapter = container_of(desc, struct leapraid_adapter, overheat_desc);
	pdev = adapter->pdev;
	shost = pci_get_drvdata(pdev);

	if (!shost) {
		dev_err(&pdev->dev,
			"Overheat processing failed: invalid host/adapter\n");
		atomic_set(&adapter->overheat_desc.thermal_alert, 0);
		wake_up(&adapter->scan_dev_desc.wait_driver_loading);
		return;
	}

	if (adapter->access_ctrl.host_removing) {
		atomic_set(&adapter->overheat_desc.thermal_alert, 0);
		wake_up(&adapter->scan_dev_desc.wait_driver_loading);
		return;
	}

	adapter->access_ctrl.host_removing = 1;
	adapter->access_ctrl.shost_recover_async = 0;
	adapter->scan_dev_desc.scan_start = 0;
	adapter->scan_dev_desc.wait_scan_dev_done = 0;
	adapter->scan_dev_desc.driver_loading = 0;
	wake_up(&adapter->access_ctrl.shost_recover_wq);
	wake_up(&adapter->scan_dev_desc.wait_driver_loading);

	leapraid_mask_int(adapter);

	leapraid_check_scheduled_fault_stop(adapter);
	leapraid_fw_log_stop(adapter);
	leapraid_mq_polling_pause(adapter);

	leapraid_clean_active_cmds(adapter);
	spin_lock_irqsave(&adapter->fw_evt_s.fw_evt_lock, flags);
	wq = adapter->fw_evt_s.fw_evt_thread;
	adapter->fw_evt_s.fw_evt_thread = NULL;
	spin_unlock_irqrestore(&adapter->fw_evt_s.fw_evt_lock, flags);
	if (wq)
		destroy_workqueue(wq);

	sas_remove_host(shost);
	leapraid_cleanup_lists(adapter);
	atomic_set(&adapter->overheat_desc.thermal_alert, 0);
	wake_up(&adapter->scan_dev_desc.wait_driver_loading);
	dev_err(&pdev->dev, "%s: Suspend adapter due to overheat\n", __func__);
}

static void leapraid_overheat_init(struct leapraid_adapter *adapter)
{
	if (adapter->overheat_desc.fault_overheat_wq)
		return;

	atomic_set(&adapter->overheat_desc.thermal_alert, 0);
	snprintf(adapter->overheat_desc.fault_overheat_wq_name,
		 sizeof(adapter->overheat_desc.fault_overheat_wq_name),
		 "driver_%s%u_overheat",
		 LEAPRAID_DRIVER_NAME,
		 adapter->adapter_attr.id);
	adapter->overheat_desc.fault_overheat_wq =
		create_singlethread_workqueue(
			adapter->overheat_desc.fault_overheat_wq_name);
	if (!adapter->overheat_desc.fault_overheat_wq) {
		dev_err(&adapter->pdev->dev,
			"Failed to create overheat workqueue\n");
		return;
	}

	INIT_WORK(&adapter->overheat_desc.fault_overheat_work,
		  leapraid_overheat_work);
}

void leapraid_overheat_cleanup(struct leapraid_adapter *adapter)
{
	struct workqueue_struct *wq;

	wq = xchg(&adapter->overheat_desc.fault_overheat_wq, NULL);
	if (!wq)
		return;

	cancel_work_sync(&adapter->overheat_desc.fault_overheat_work);
	destroy_workqueue(wq);
}

static void leapraid_fw_work(struct leapraid_adapter *adapter,
			     struct leapraid_fw_evt_work *fw_evt);

static void leapraid_fw_evt_free(struct kref *r)
{
	struct leapraid_fw_evt_work *fw_evt;

	fw_evt = container_of(r, struct leapraid_fw_evt_work, refcnt);

	kfree(fw_evt->evt_data);
	kfree(fw_evt);
}

static void leapraid_fw_evt_get(struct leapraid_fw_evt_work *fw_evt)
{
	kref_get(&fw_evt->refcnt);
}

static void leapraid_fw_evt_put(struct leapraid_fw_evt_work *fw_work)
{
	kref_put(&fw_work->refcnt, leapraid_fw_evt_free);
}

static struct leapraid_fw_evt_work *leapraid_alloc_fw_evt_work(void)
{
	struct leapraid_fw_evt_work *fw_evt = kzalloc_obj(*fw_evt, GFP_ATOMIC);

	if (fw_evt)
		kref_init(&fw_evt->refcnt);

	return fw_evt;
}

static void leapraid_run_fw_evt_work(struct work_struct *work)
{
	struct leapraid_fw_evt_work *fw_evt =
		container_of(work, struct leapraid_fw_evt_work, work);

	leapraid_fw_work(fw_evt->adapter, fw_evt);
}

static void leapraid_fw_evt_add(struct leapraid_adapter *adapter,
				struct leapraid_fw_evt_work *fw_evt)
{
	unsigned long flags;

	spin_lock_irqsave(&adapter->fw_evt_s.fw_evt_lock, flags);
	if (adapter->access_ctrl.host_removing ||
	    adapter->access_ctrl.pcie_recovering ||
	    !adapter->fw_evt_s.fw_evt_thread) {
		spin_unlock_irqrestore(&adapter->fw_evt_s.fw_evt_lock, flags);
		return;
	}

	leapraid_fw_evt_get(fw_evt);
	INIT_LIST_HEAD(&fw_evt->list);
	list_add_tail(&fw_evt->list, &adapter->fw_evt_s.fw_evt_list);
	INIT_WORK(&fw_evt->work, leapraid_run_fw_evt_work);
	leapraid_fw_evt_get(fw_evt);
	queue_work(adapter->fw_evt_s.fw_evt_thread, &fw_evt->work);
	spin_unlock_irqrestore(&adapter->fw_evt_s.fw_evt_lock, flags);
}

static void leapraid_del_fw_evt_from_list(struct leapraid_adapter *adapter,
					  struct leapraid_fw_evt_work *fw_evt)
{
	unsigned long flags;

	spin_lock_irqsave(&adapter->fw_evt_s.fw_evt_lock, flags);
	if (!list_empty(&fw_evt->list)) {
		list_del_init(&fw_evt->list);
		leapraid_fw_evt_put(fw_evt);
	}
	spin_unlock_irqrestore(&adapter->fw_evt_s.fw_evt_lock, flags);
}

static struct leapraid_fw_evt_work *leapraid_next_fw_evt(
		struct leapraid_adapter *adapter)
{
	struct leapraid_fw_evt_work *fw_evt = NULL;
	unsigned long flags;

	spin_lock_irqsave(&adapter->fw_evt_s.fw_evt_lock, flags);
	if (!list_empty(&adapter->fw_evt_s.fw_evt_list)) {
		fw_evt = list_first_entry(&adapter->fw_evt_s.fw_evt_list,
					  struct leapraid_fw_evt_work, list);
		list_del_init(&fw_evt->list);
	}
	spin_unlock_irqrestore(&adapter->fw_evt_s.fw_evt_lock, flags);
	return fw_evt;
}

void leapraid_clean_active_fw_evt(struct leapraid_adapter *adapter)
{
	struct leapraid_fw_evt_work *fw_evt;
	unsigned long flags;
	bool in_fw_evt_context;
	bool rc;

	if ((list_empty(&adapter->fw_evt_s.fw_evt_list) &&
	     !adapter->fw_evt_s.cur_evt) || !adapter->fw_evt_s.fw_evt_thread)
		return;

	adapter->fw_evt_s.fw_evt_cleanup = 1;
	if (adapter->access_ctrl.shost_recovering &&
	    adapter->fw_evt_s.cur_evt)
		adapter->fw_evt_s.cur_evt->ignore = 1;

	while ((fw_evt = leapraid_next_fw_evt(adapter))) {
		rc = cancel_work_sync(&fw_evt->work);
		if (rc)
			leapraid_fw_evt_put(fw_evt);
		leapraid_fw_evt_put(fw_evt);
	}

	spin_lock_irqsave(&adapter->fw_evt_s.fw_evt_lock, flags);
	fw_evt = adapter->fw_evt_s.cur_evt;
	if (fw_evt) {
		in_fw_evt_context = adapter->fw_evt_s.cur_evt_task == current;
		leapraid_fw_evt_get(fw_evt);
	}
	spin_unlock_irqrestore(&adapter->fw_evt_s.fw_evt_lock, flags);
	if (fw_evt) {
		if (!in_fw_evt_context)
			cancel_work_sync(&fw_evt->work);
		leapraid_fw_evt_put(fw_evt);
	}

	adapter->fw_evt_s.fw_evt_cleanup = 0;
}

static void leapraid_internal_dev_ublk(struct scsi_device *sdev,
				       struct leapraid_sdev_priv *sdev_priv)
{
	int rc;

	sdev_printk(KERN_WARNING, sdev,
		    "hdl 0x%04x: Now internal unblkg dev\n",
		    sdev_priv->starget_priv->hdl);
	sdev_priv->block = 0;
	rc = scsi_internal_device_unblock_nowait(sdev, SDEV_RUNNING);
	if (rc == -EINVAL) {
		sdev_printk(KERN_WARNING, sdev,
			    "hdl 0x%04x: unblkg failed, rc=%d\n",
			    sdev_priv->starget_priv->hdl, rc);
		sdev_priv->block = 1;
		rc = scsi_internal_device_block_nowait(sdev);
		if (rc)
			sdev_printk(KERN_WARNING, sdev,
				    "hdl 0x%04x: Earlier ublkg err, rc=%d\n",
				    sdev_priv->starget_priv->hdl, rc);

		sdev_priv->block = 0;
		rc = scsi_internal_device_unblock_nowait(sdev, SDEV_RUNNING);
		if (rc)
			sdev_printk(KERN_WARNING, sdev,
				    "hdl 0x%04x: ublkg failed again, rc=%d\n",
				    sdev_priv->starget_priv->hdl, rc);
	}
}

static void leapraid_ublk_io_dev(struct leapraid_adapter *adapter,
				 u64 sas_addr,
				 struct leapraid_card_port *card_port)
{
	struct leapraid_sdev_priv *sdev_priv;
	struct scsi_device *sdev;

	shost_for_each_device(sdev, adapter->shost) {
		sdev_priv = sdev->hostdata;
		if (!sdev_priv || !sdev_priv->starget_priv)
			continue;

		if (sdev_priv->starget_priv->sas_address != sas_addr)
			continue;

		if (sdev_priv->starget_priv->card_port != card_port)
			continue;

		if (sdev_priv->block)
			leapraid_internal_dev_ublk(sdev, sdev_priv);

		scsi_device_set_state(sdev, SDEV_OFFLINE);
	}
}

static void leapraid_ublk_io_all_dev(struct leapraid_adapter *adapter)
{
	struct leapraid_sdev_priv *sdev_priv;
	struct leapraid_starget_priv *stgt_priv;
	struct scsi_device *sdev;

	shost_for_each_device(sdev, adapter->shost) {
		sdev_priv = sdev->hostdata;

		if (!sdev_priv)
			continue;

		stgt_priv = sdev_priv->starget_priv;
		if (!stgt_priv || stgt_priv->deleted)
			continue;

		if (!sdev_priv->block)
			continue;

		sdev_printk(KERN_WARNING, sdev, "hdl 0x%04x: blkg...\n",
			    sdev_priv->starget_priv->hdl);
		leapraid_internal_dev_ublk(sdev, sdev_priv);
		continue;
	}
}

static void leapraid_internal_dev_blk(
		struct scsi_device *sdev,
		struct leapraid_sdev_priv *sdev_priv)
{
	int rc;

	sdev_printk(KERN_INFO, sdev, "Internal blkg hdl 0x%04x\n",
		    sdev_priv->starget_priv->hdl);
	sdev_priv->block = 1;
	rc = scsi_internal_device_block_nowait(sdev);
	if (rc == -EINVAL)
		sdev_printk(KERN_WARNING, sdev,
			    "hdl 0x%04x: blkg failed, rc=%d\n",
			    rc, sdev_priv->starget_priv->hdl);
}

static void leapraid_imm_blkio_to_end_dev(struct leapraid_adapter *adapter,
					  struct leapraid_sas_port *sas_port)
{
	struct leapraid_sdev_priv *sdev_priv;
	struct leapraid_sas_dev *sas_dev;
	struct scsi_device *sdev;
	unsigned long flags;

	spin_lock_irqsave(&adapter->dev_topo.sas_dev_lock, flags);
	sas_dev = leapraid_hold_lock_get_sas_dev_by_addr(
		adapter,
		sas_port->remote_identify.sas_address,
		sas_port->card_port);

	if (sas_dev) {
		shost_for_each_device(sdev, adapter->shost) {
			sdev_priv = sdev->hostdata;
			if (!sdev_priv)
				continue;

			if (sdev_priv->starget_priv->hdl != sas_dev->hdl)
				continue;

			if (sdev_priv->block)
				continue;

			if (sas_dev && sas_dev->pend_sas_rphy_add)
				continue;

			if (sdev_priv->sep) {
				sdev_printk(KERN_INFO, sdev,
					    "skip dev blk: sep hdl 0x%04x\n",
					    sdev_priv->starget_priv->hdl);
				continue;
			}

			leapraid_internal_dev_blk(sdev, sdev_priv);
		}

		leapraid_sdev_put(sas_dev);
	}
	spin_unlock_irqrestore(&adapter->dev_topo.sas_dev_lock, flags);
}

static void leapraid_imm_blkio_set_end_dev_blk_hdls(
		struct leapraid_adapter *adapter,
		struct leapraid_topo_node *topo_node_exp)
{
	struct leapraid_sas_port *sas_port;

	list_for_each_entry(sas_port,
			    &topo_node_exp->sas_port_list, port_list) {
		if (sas_port->remote_identify.device_type == SAS_END_DEVICE)
			leapraid_imm_blkio_to_end_dev(adapter, sas_port);
	}
}

static void leapraid_imm_blkio_to_kids_attchd_to_ex(
		struct leapraid_adapter *adapter,
		struct leapraid_topo_node *topo_node_exp);

static void leapraid_imm_blkio_to_sib_exp(
		struct leapraid_adapter *adapter,
		struct leapraid_topo_node *topo_node_exp)
{
	struct leapraid_topo_node *topo_node_exp_sib;
	struct leapraid_sas_port *sas_port;

	list_for_each_entry(sas_port,
			    &topo_node_exp->sas_port_list, port_list) {
		if (sas_port->remote_identify.device_type ==
			 SAS_EDGE_EXPANDER_DEVICE ||
		    sas_port->remote_identify.device_type ==
			 SAS_FANOUT_EXPANDER_DEVICE) {
			topo_node_exp_sib =
				leapraid_exp_find_by_sas_address(
					adapter,
					sas_port->remote_identify.sas_address,
					sas_port->card_port);
			leapraid_imm_blkio_to_kids_attchd_to_ex(
				adapter,
				topo_node_exp_sib);
		}
	}
}

static void leapraid_imm_blkio_to_kids_attchd_to_ex(
				struct leapraid_adapter *adapter,
				struct leapraid_topo_node *topo_node_exp)
{
	if (!topo_node_exp)
		return;

	leapraid_imm_blkio_set_end_dev_blk_hdls(adapter, topo_node_exp);

	leapraid_imm_blkio_to_sib_exp(adapter, topo_node_exp);
}

static void leapraid_report_sdev_directly(struct leapraid_adapter *adapter,
					  struct leapraid_sas_dev *sas_dev)
{
	struct leapraid_sas_port *sas_port;

	sas_port = leapraid_transport_port_add(adapter,
					       sas_dev->hdl,
					       sas_dev->parent_sas_addr,
					       sas_dev->card_port);
	if (!sas_port) {
		leapraid_sas_dev_remove(adapter, sas_dev);
		return;
	}

	if (!sas_dev->starget) {
		if (!adapter->scan_dev_desc.driver_loading) {
			leapraid_transport_port_remove(
				adapter,
				sas_dev->sas_addr,
				sas_dev->parent_sas_addr,
				sas_dev->card_port);
			leapraid_sas_dev_remove(adapter, sas_dev);
		}
		return;
	}
}

static struct leapraid_sas_dev *leapraid_init_sas_dev(
		struct leapraid_adapter *adapter,
		struct leapraid_sas_dev_p0 *sas_dev_pg0,
		struct leapraid_card_port *card_port, u16 hdl,
		u64 parent_sas_addr, u64 sas_addr, u32 dev_info)
{
	struct leapraid_sas_dev *sas_dev;
	struct leapraid_enc_node *enc_dev;
	unsigned long flags;

	sas_dev = kzalloc_obj(*sas_dev);
	if (!sas_dev)
		return NULL;

	kref_init(&sas_dev->refcnt);
	sas_dev->hdl = hdl;
	sas_dev->dev_info = dev_info;
	sas_dev->sas_addr = sas_addr;
	sas_dev->card_port = card_port;
	sas_dev->parent_sas_addr = parent_sas_addr;
	sas_dev->phy = sas_dev_pg0->phy_num;
	sas_dev->enc_hdl = le16_to_cpu(sas_dev_pg0->enc_hdl);
	sas_dev->dev_name = le64_to_cpu(sas_dev_pg0->dev_name);
	sas_dev->port_connection = sas_dev_pg0->max_port_connections;
	sas_dev->slot = sas_dev->enc_hdl ? le16_to_cpu(sas_dev_pg0->slot) : 0;
	if (le16_to_cpu(sas_dev_pg0->flg) &
	    LEAPRAID_SAS_DEV_P0_FLG_ENC_LEVEL_VALID) {
		sas_dev->enc_level = sas_dev_pg0->enc_level;
		memcpy(sas_dev->connector_name,
		       sas_dev_pg0->connector_name,
		       LEAPRAID_SAS_DEV_P0_CON_NAME_LEN);
		sas_dev->connector_name[LEAPRAID_SAS_DEV_P0_CON_NAME_LEN] =
			'\0';
	} else {
		sas_dev->enc_level = 0;
		sas_dev->connector_name[0] = '\0';
	}
	if (sas_dev->enc_hdl) {
		spin_lock_irqsave(&adapter->dev_topo.enc_lock, flags);
		enc_dev = leapraid_enc_find_by_hdl(adapter, sas_dev->enc_hdl);
		if (enc_dev)
			sas_dev->enc_lid = le64_to_cpu(enc_dev->pg0.enc_lid);
		spin_unlock_irqrestore(&adapter->dev_topo.enc_lock, flags);
	}
	dev_info(&adapter->pdev->dev,
		 "add dev: hdl=0x%x, SAS addr=0x%016llx, port connect=0x%x\n",
		 hdl, sas_dev->sas_addr, sas_dev->port_connection);

	return sas_dev;
}

static void leapraid_add_dev(struct leapraid_adapter *adapter, u16 hdl)
{
	union cfg_param_1 cfgp1 = {0};
	union cfg_param_2 cfgp2 = {0};
	struct leapraid_sas_dev_p0 sas_dev_pg0;
	struct leapraid_card_port *card_port;
	struct leapraid_sas_dev *sas_dev;
	unsigned long flags;
	u64 parent_sas_addr;
	u32 dev_info;
	u64 sas_addr;
	u8 port_id;

	cfgp1.form = LEAPRAID_SAS_DEV_CFG_PGAD_HDL;
	cfgp2.handle = hdl;
	if (leapraid_op_config_page(adapter, &sas_dev_pg0,
				    cfgp1, cfgp2, GET_SAS_DEVICE_PG0))
		return;

	dev_info = le32_to_cpu(sas_dev_pg0.dev_info);
	if (!(leapraid_is_end_dev(dev_info)))
		return;

	sas_addr = le64_to_cpu(sas_dev_pg0.sas_address);
	if (!(le16_to_cpu(sas_dev_pg0.flg) &
	      LEAPRAID_SAS_DEV_P0_FLG_DEV_PRESENT))
		return;

	port_id = sas_dev_pg0.physical_port;
	card_port = leapraid_get_port_by_id(adapter, port_id, false);
	if (!card_port)
		return;

	sas_dev = leapraid_get_sas_dev_by_addr(adapter, sas_addr, card_port);
	if (sas_dev) {
		leapraid_sdev_put(sas_dev);
		return;
	}

	if (leapraid_get_sas_address(adapter,
				     le16_to_cpu(sas_dev_pg0.parent_dev_hdl),
				     &parent_sas_addr))
		return;

	sas_dev = leapraid_init_sas_dev(adapter, &sas_dev_pg0, card_port,
					hdl, parent_sas_addr, sas_addr,
					dev_info);
	if (!sas_dev)
		return;

	if (adapter->scan_dev_desc.wait_scan_dev_done) {
		spin_lock_irqsave(&adapter->dev_topo.sas_dev_lock, flags);
		leapraid_sdev_get(sas_dev);
		list_add_tail(&sas_dev->list,
			      &adapter->dev_topo.sas_dev_init_list);
		leapraid_check_boot_dev(adapter, sas_dev, 0);
		spin_unlock_irqrestore(&adapter->dev_topo.sas_dev_lock, flags);
	} else {
		spin_lock_irqsave(&adapter->dev_topo.sas_dev_lock, flags);
		leapraid_sdev_get(sas_dev);
		list_add_tail(&sas_dev->list, &adapter->dev_topo.sas_dev_list);
		spin_unlock_irqrestore(&adapter->dev_topo.sas_dev_lock, flags);
		leapraid_report_sdev_directly(adapter, sas_dev);
	}
	leapraid_sdev_put(sas_dev);
}

static void leapraid_remove_device(struct leapraid_adapter *adapter,
				   struct leapraid_sas_dev *sas_dev)
{
	struct leapraid_starget_priv *starget_priv;

	leapraid_clear_cached_boot_dev(adapter, sas_dev, 0);
	if (sas_dev->led_on) {
		leapraid_set_led(adapter, sas_dev, false);
		sas_dev->led_on = 0;
	}

	if (sas_dev->starget && sas_dev->starget->hostdata) {
		starget_priv = sas_dev->starget->hostdata;
		starget_priv->deleted = 1;
		leapraid_ublk_io_dev(adapter,
				     sas_dev->sas_addr, sas_dev->card_port);
		starget_priv->hdl = LEAPRAID_INVALID_DEV_HANDLE;
	}

	leapraid_transport_port_remove(adapter,
				       sas_dev->sas_addr,
				       sas_dev->parent_sas_addr,
				       sas_dev->card_port);

	dev_info(&adapter->pdev->dev,
		 "remove dev: hdl=0x%04x, SAS addr=0x%016llx\n",
		 sas_dev->hdl, (unsigned long long)sas_dev->sas_addr);
}

static struct leapraid_vphy *leapraid_alloc_vphy(
		struct leapraid_adapter *adapter,
		u8 port_id, u8 phy_num)
{
	struct leapraid_card_port *port;
	struct leapraid_vphy *vphy;

	port = leapraid_get_port_by_id(adapter, port_id, false);
	if (!port)
		return NULL;

	vphy = leapraid_get_vphy_by_phy(port, phy_num);
	if (vphy)
		return vphy;
	vphy = kzalloc_obj(*vphy);
	if (!vphy)
		return NULL;

	if (!port->vphys_mask)
		INIT_LIST_HEAD(&port->vphys_list);

	port->vphys_mask |= BIT(phy_num);
	vphy->phy_mask |= BIT(phy_num);
	list_add_tail(&vphy->list, &port->vphys_list);
	return vphy;
}

static int leapraid_add_port_to_card_port_list(
					struct leapraid_adapter *adapter,
					u8 port_id, bool refresh)
{
	struct leapraid_card_port *card_port;

	card_port = leapraid_get_port_by_id(adapter, port_id, false);
	if (card_port)
		return 0;
	card_port = kzalloc_obj(*card_port);
	if (!card_port)
		return -ENOMEM;

	card_port->port_id = port_id;
	dev_dbg(&adapter->pdev->dev,
		"port: %d is added to card_port list\n",
		card_port->port_id);

	if (refresh && adapter->access_ctrl.shost_recovering)
		card_port->flg = LEAPRAID_CARD_PORT_FLG_NEW;
	list_add_tail(&card_port->list, &adapter->dev_topo.card_port_list);
	return 0;
}

static int leapraid_add_card_phy(struct leapraid_adapter *adapter,
				 struct leapraid_sas_io_unit_p0 *iou,
				 int i)
{
	struct leapraid_sas_phy_p0 phy_pg0;
	union cfg_param_1 cfgp1 = {0};
	union cfg_param_2 cfgp2 = {0};
	struct leapraid_topo_node *card = &adapter->dev_topo.card;
	u8 port_id;

	cfgp1.phy_number = i;

	if (leapraid_op_config_page(adapter, &phy_pg0,
				    cfgp1, cfgp2, GET_PHY_PG0))
		return -EINVAL;

	port_id = iou->phy_info[i].port;

	if (leapraid_add_port_to_card_port_list(adapter, port_id, false))
		return -EINVAL;

	if ((le32_to_cpu(phy_pg0.phy_info) &
	     LEAPRAID_SAS_PHYINFO_VPHY) &&
	    ((phy_pg0.neg_link_rate >>
	      LEAPRAID_SAS_NEG_LINK_RATE_SHIFT) >=
	     LEAPRAID_SAS_NEG_LINK_RATE_1_5)) {
		if (!leapraid_alloc_vphy(adapter, port_id, i))
			return -ENOMEM;

		card->card_phy[i].vphy = 1;
	}

	card->card_phy[i].hdl = card->hdl;
	card->card_phy[i].phy_id = i;
	card->card_phy[i].card_port =
		leapraid_get_port_by_id(adapter, port_id, false);

	leapraid_transport_add_card_phy(adapter,
					&card->card_phy[i],
					&phy_pg0,
					card->parent_dev);

	return 0;
}

static int leapraid_refresh_card_phy(struct leapraid_adapter *adapter,
				     struct leapraid_sas_io_unit_p0 *iou,
				     int i)
{
	struct leapraid_topo_node *card = &adapter->dev_topo.card;
	struct leapraid_sas_phy_p0 phy_pg0;
	union cfg_param_1 cfgp1 = {0};
	union cfg_param_2 cfgp2 = {0};
	u16 attached_hdl;
	u32 dev_info;
	u8 link_rate;
	u8 port_id;

	link_rate = iou->phy_info[i].neg_link_rate >>
		LEAPRAID_SAS_NEG_LINK_RATE_SHIFT;
	port_id = iou->phy_info[i].port;
	if (leapraid_add_port_to_card_port_list(adapter, port_id, true))
		return -EINVAL;

	dev_info = le32_to_cpu(iou->phy_info[i].controller_phy_dev_info);
	if (dev_info & LEAPRAID_DEVTYP_SEP &&
	    link_rate >= LEAPRAID_SAS_NEG_LINK_RATE_1_5) {
		cfgp1.phy_number = i;

		if (leapraid_op_config_page(adapter, &phy_pg0,
					    cfgp1, cfgp2, GET_PHY_PG0))
			return 0;

		if (le32_to_cpu(phy_pg0.phy_info) &
		    LEAPRAID_SAS_PHYINFO_VPHY) {
			if (!leapraid_alloc_vphy(adapter, port_id, i))
				return -ENOMEM;
			card->card_phy[i].vphy = 1;
		}
	}

	card->card_phy[i].hdl = card->hdl;

	attached_hdl = le16_to_cpu(iou->phy_info[i].attached_dev_hdl);
	if (attached_hdl && link_rate < LEAPRAID_SAS_NEG_LINK_RATE_1_5)
		link_rate = LEAPRAID_SAS_NEG_LINK_RATE_1_5;

	card->card_phy[i].card_port =
		leapraid_get_port_by_id(adapter, port_id, false);

	if (!card->card_phy[i].phy) {
		cfgp1.phy_number = i;
		if (leapraid_op_config_page(adapter, &phy_pg0,
					    cfgp1, cfgp2, GET_PHY_PG0))
			return 0;

		card->card_phy[i].phy_id = i;

		leapraid_transport_add_card_phy(adapter,
						&card->card_phy[i],
						&phy_pg0,
						card->parent_dev);
		return 0;
	}

	leapraid_transport_update_links(adapter,
					card->sas_address,
					attached_hdl,
					i,
					link_rate,
					card->card_phy[i].card_port);

	return 0;
}

static void leapraid_sas_host_add(struct leapraid_adapter *adapter,
				  bool refresh)
{
	union cfg_param_1 cfgp1 = {0};
	union cfg_param_2 cfgp2 = {0};
	struct leapraid_sas_dev_p0 sas_dev_pg0;
	struct leapraid_enc_p0 enc_pg0;
	struct leapraid_sas_io_unit_p0 *sas_iou_pg0;
	u16 sas_iou_pg0_sz;
	u8 phys_num;
	int i;
	int rc;

	if (!refresh) {
		if (leapraid_get_adapter_phys(adapter, &phys_num) || !phys_num)
			return;

		adapter->dev_topo.card.card_phy =
			kzalloc_objs(struct leapraid_card_phy, phys_num);
		if (!adapter->dev_topo.card.card_phy)
			return;

		adapter->dev_topo.card.phys_num = phys_num;
	}

	sas_iou_pg0_sz =
		offsetof(struct leapraid_sas_io_unit_p0, phy_info) +
			 (adapter->dev_topo.card.phys_num *
			  sizeof(struct leapraid_sas_io_unit0_phy_info));
	sas_iou_pg0 = kzalloc(sas_iou_pg0_sz, GFP_KERNEL);
	if (!sas_iou_pg0)
		return;

	if (leapraid_get_sas_io_unit_page0(adapter,
					   sas_iou_pg0,
					   sas_iou_pg0_sz))
		goto out_free;

	adapter->dev_topo.card.parent_dev = &adapter->shost->shost_gendev;
	adapter->dev_topo.card.hdl =
		le16_to_cpu(sas_iou_pg0->phy_info[0].controller_dev_hdl);
	for (i = 0; i < adapter->dev_topo.card.phys_num; i++) {
		if (!refresh) /* add */
			rc = leapraid_add_card_phy(adapter, sas_iou_pg0, i);
		else /* refresh */
			rc = leapraid_refresh_card_phy(adapter,
						       sas_iou_pg0,
						       i);
		if (rc)
			goto out_free;
	}

	if (!refresh) {
		cfgp1.form = LEAPRAID_SAS_DEV_CFG_PGAD_HDL;
		cfgp2.handle = adapter->dev_topo.card.hdl;
		if (leapraid_op_config_page(adapter, &sas_dev_pg0, cfgp1,
					    cfgp2, GET_SAS_DEVICE_PG0))
			goto out_free;

		adapter->dev_topo.card.enc_hdl =
			le16_to_cpu(sas_dev_pg0.enc_hdl);
		adapter->dev_topo.card.sas_address =
			le64_to_cpu(sas_dev_pg0.sas_address);
		dev_info(&adapter->pdev->dev,
			 "add host: hdl=0x%04x, SAS addr=0x%016llx, phy=%d\n",
			 adapter->dev_topo.card.hdl,
			 (unsigned long long)adapter->dev_topo.card.sas_address,
			 adapter->dev_topo.card.phys_num);

		if (adapter->dev_topo.card.enc_hdl) {
			cfgp1.form = LEAPRAID_SAS_ENC_CFG_PGAD_HDL;
			cfgp2.handle = adapter->dev_topo.card.enc_hdl;
			if (!(leapraid_op_config_page(adapter, &enc_pg0,
						      cfgp1, cfgp2,
						      GET_SAS_ENCLOSURE_PG0)))
				adapter->dev_topo.card.enc_lid =
					le64_to_cpu(enc_pg0.enc_lid);
		}
	}
out_free:
	kfree(sas_iou_pg0);
}

static int leapraid_internal_exp_add(struct leapraid_adapter *adapter,
				     struct leapraid_exp_p0 *exp_pg0,
				     union cfg_param_1 *cfgp1,
				     union cfg_param_2 *cfgp2,
				     u16 hdl)
{
	struct leapraid_topo_node *topo_node_exp;
	struct leapraid_sas_port *sas_port = NULL;
	struct leapraid_enc_node *enc_dev;
	struct leapraid_exp_p1 exp_pg1;
	int ret;
	int rc;
	unsigned long flags;
	u8 port_id;
	u16 parent_handle;
	u64 sas_addr_parent;
	int i;

	port_id = exp_pg0->physical_port;
	parent_handle = le16_to_cpu(exp_pg0->parent_dev_hdl);

	rc = leapraid_get_sas_address(adapter,
				      parent_handle, &sas_addr_parent);
	if (rc)
		return rc;
	topo_node_exp = kzalloc_obj(*topo_node_exp);
	if (!topo_node_exp)
		return -ENOMEM;

	topo_node_exp->hdl = hdl;
	topo_node_exp->phys_num = exp_pg0->phy_num;
	topo_node_exp->sas_address_parent = sas_addr_parent;
	topo_node_exp->sas_address = le64_to_cpu(exp_pg0->sas_address);
	topo_node_exp->card_port =
		leapraid_get_port_by_id(adapter, port_id, false);
	if (!topo_node_exp->card_port) {
		rc = -EPERM;
		goto out_fail;
	}

	dev_info(&adapter->pdev->dev,
		 "add exp: saddr=0x%016llx, hdl=0x%04x, phdl=0x%04x, phy=%d\n",
		 (unsigned long long)topo_node_exp->sas_address,
		 hdl, parent_handle,
		 topo_node_exp->phys_num);
	if (!topo_node_exp->phys_num) {
		dev_err(&adapter->pdev->dev,
			"%s: Invalid PHY num from exp_pg0\n", __func__);
		rc = -EPERM;
		goto out_fail;
	}

	topo_node_exp->card_phy =
		kzalloc_objs(struct leapraid_card_phy, topo_node_exp->phys_num);
	if (!topo_node_exp->card_phy) {
		dev_err(&adapter->pdev->dev,
			"%s: Failed to alloc expander phy array, count=%u\n",
			__func__, topo_node_exp->phys_num);
		rc = -EPERM;
		goto out_fail;
	}

	INIT_LIST_HEAD(&topo_node_exp->sas_port_list);
	sas_port = leapraid_transport_port_add(adapter, hdl, sas_addr_parent,
					       topo_node_exp->card_port);
	if (!sas_port) {
		rc = -EPERM;
		goto out_fail;
	}

	topo_node_exp->parent_dev = &sas_port->rphy->dev;
	topo_node_exp->rphy = sas_port->rphy;
	for (i = 0; i < topo_node_exp->phys_num; i++) {
		cfgp1->phy_number = i;
		cfgp2->handle = hdl;
		if (leapraid_op_config_page(adapter, &exp_pg1, *cfgp1, *cfgp2,
					    GET_SAS_EXPANDER_PG1)) {
			dev_err(&adapter->pdev->dev,
				"%s: Failed to get exp_pg1, phy=%d\n",
				__func__, i);
			rc = -EPERM;
			goto out_fail;
		}

		topo_node_exp->card_phy[i].hdl = hdl;
		topo_node_exp->card_phy[i].phy_id = i;
		topo_node_exp->card_phy[i].card_port =
			leapraid_get_port_by_id(adapter, port_id, false);
		ret = leapraid_transport_add_exp_phy(
				adapter,
				&topo_node_exp->card_phy[i],
				&exp_pg1,
				topo_node_exp->parent_dev);
		if (ret) {
			rc = -EPERM;
			goto out_fail;
		}
	}

	if (topo_node_exp->enc_hdl) {
		spin_lock_irqsave(&adapter->dev_topo.enc_lock, flags);
		enc_dev = leapraid_enc_find_by_hdl(adapter,
						   topo_node_exp->enc_hdl);
		if (enc_dev)
			topo_node_exp->enc_lid =
				le64_to_cpu(enc_dev->pg0.enc_lid);
		spin_unlock_irqrestore(&adapter->dev_topo.enc_lock, flags);
	}

	spin_lock_irqsave(&adapter->dev_topo.topo_node_lock, flags);
	list_add_tail(&topo_node_exp->list, &adapter->dev_topo.exp_list);
	spin_unlock_irqrestore(&adapter->dev_topo.topo_node_lock, flags);
	return 0;

out_fail:
	if (sas_port)
		leapraid_transport_port_remove(adapter,
					       topo_node_exp->sas_address,
					       sas_addr_parent,
					       topo_node_exp->card_port);
	kfree(topo_node_exp->card_phy);
	kfree(topo_node_exp);
	return rc;
}

static int leapraid_exp_add(struct leapraid_adapter *adapter, u16 hdl)
{
	union cfg_param_1 cfgp1 = {0};
	union cfg_param_2 cfgp2 = {0};
	struct leapraid_topo_node *topo_node_exp;
	struct leapraid_exp_p0 exp_pg0;
	u16 parent_handle;
	u64 sas_addr, sas_addr_parent;
	unsigned long flags;
	u8 port_id;
	int rc;

	if (!hdl) {
		dev_warn(&adapter->pdev->dev, "%s: Invalid hdl\n", __func__);
		return -EPERM;
	}

	if (adapter->access_ctrl.shost_recovering ||
	    adapter->access_ctrl.pcie_recovering) {
		dev_warn(&adapter->pdev->dev,
			 "%s: Failed, shost_recovering=%d pcie_recovering=%d\n",
			 __func__, adapter->access_ctrl.shost_recovering,
			adapter->access_ctrl.pcie_recovering);
		return -EPERM;
	}

	cfgp1.form = LEAPRAID_SAS_EXP_CFD_PGAD_HDL;
	cfgp2.handle = hdl;
	if (leapraid_op_config_page(adapter, &exp_pg0, cfgp1, cfgp2,
				    GET_SAS_EXPANDER_PG0))
		return -EPERM;

	parent_handle = le16_to_cpu(exp_pg0.parent_dev_hdl);
	if (leapraid_get_sas_address(adapter, parent_handle, &sas_addr_parent))
		return -EPERM;

	port_id = exp_pg0.physical_port;
	if (sas_addr_parent != adapter->dev_topo.card.sas_address) {
		spin_lock_irqsave(&adapter->dev_topo.topo_node_lock, flags);
		topo_node_exp =
			leapraid_exp_find_by_sas_address(
				adapter,
				sas_addr_parent,
				leapraid_get_port_by_id(adapter,
							port_id,
							false));
		spin_unlock_irqrestore(&adapter->dev_topo.topo_node_lock,
				       flags);
		if (!topo_node_exp) {
			rc = leapraid_exp_add(adapter, parent_handle);
			if (rc != 0)
				return rc;
		}
	}

	spin_lock_irqsave(&adapter->dev_topo.topo_node_lock, flags);
	sas_addr = le64_to_cpu(exp_pg0.sas_address);
	topo_node_exp =
		leapraid_exp_find_by_sas_address(
			adapter,
			sas_addr,
			leapraid_get_port_by_id(adapter, port_id, false));
	spin_unlock_irqrestore(&adapter->dev_topo.topo_node_lock, flags);

	if (topo_node_exp)
		return 0;

	return leapraid_internal_exp_add(adapter, &exp_pg0, &cfgp1,
					 &cfgp2, hdl);
}

static void leapraid_exp_node_rm(struct leapraid_adapter *adapter,
				 struct leapraid_topo_node *topo_node_exp)
{
	struct leapraid_sas_port *sas_port, *sas_port_next;
	unsigned long flags;
	int port_id;

	list_for_each_entry_safe(sas_port, sas_port_next,
				 &topo_node_exp->sas_port_list,
				 port_list) {
		if (adapter->access_ctrl.shost_recovering)
			return;

		switch (sas_port->remote_identify.device_type) {
		case SAS_END_DEVICE:
			leapraid_sas_dev_remove_by_sas_address(
				adapter,
				sas_port->remote_identify.sas_address,
				sas_port->card_port);
			break;
		case SAS_EDGE_EXPANDER_DEVICE:
		case SAS_FANOUT_EXPANDER_DEVICE:
			leapraid_exp_rm(
				adapter,
				sas_port->remote_identify.sas_address,
				sas_port->card_port);
			break;
		default:
			break;
		}
	}

	port_id = topo_node_exp->card_port->port_id;
	leapraid_transport_port_remove(adapter, topo_node_exp->sas_address,
				       topo_node_exp->sas_address_parent,
				       topo_node_exp->card_port);
	dev_info(&adapter->pdev->dev,
		 "removing exp: port=%d, SAS addr=0x%016llx, hdl=0x%04x\n",
		 port_id, (unsigned long long)topo_node_exp->sas_address,
		 topo_node_exp->hdl);
	spin_lock_irqsave(&adapter->dev_topo.topo_node_lock, flags);
	list_del(&topo_node_exp->list);
	spin_unlock_irqrestore(&adapter->dev_topo.topo_node_lock, flags);
	kfree(topo_node_exp->card_phy);
	kfree(topo_node_exp);
}

void leapraid_exp_rm(struct leapraid_adapter *adapter, u64 sas_addr,
		     struct leapraid_card_port *port)
{
	struct leapraid_topo_node *topo_node_exp;
	unsigned long flags;

	if (adapter->access_ctrl.shost_recovering)
		return;

	if (!port)
		return;

	spin_lock_irqsave(&adapter->dev_topo.topo_node_lock, flags);
	topo_node_exp = leapraid_exp_find_by_sas_address(adapter,
							 sas_addr,
							 port);
	spin_unlock_irqrestore(&adapter->dev_topo.topo_node_lock, flags);

	if (topo_node_exp)
		leapraid_exp_node_rm(adapter, topo_node_exp);
}

static void leapraid_internal_sas_topo_chg_evt(
		struct leapraid_adapter *adapter,
		struct leapraid_card_port *card_port,
		struct leapraid_topo_node *topo_node_exp,
		struct leapraid_fw_evt_work *fw_evt,
		u64 sas_addr, u8 max_phys)
{
	struct leapraid_evt_data_sas_topo_change_list *evt_data;
	u8 phy_number;
	u8 link_rate;
	u16 reason_code;
	u16 hdl;
	int i;

	evt_data = fw_evt->evt_data;
	for (i = 0; i < evt_data->entry_num; i++) {
		if (fw_evt->ignore)
			return;

		if (adapter->access_ctrl.host_removing ||
		    adapter->access_ctrl.pcie_recovering)
			return;

		phy_number = evt_data->start_phy_num + i;
		if (phy_number >= max_phys)
			continue;

		reason_code = evt_data->phy[i].phy_status &
			LEAPRAID_EVT_SAS_TOPO_RC_MASK;

		hdl = le16_to_cpu(evt_data->phy[i].attached_dev_hdl);
		if (!hdl ||
		    hdl > adapter->adapter_attr.features.max_dev_handle) {
			dev_warn(&adapter->pdev->dev,
				 "%s: Invalid device handle\n", __func__);
			continue;
		}
		link_rate = evt_data->phy[i].link_rate >>
			LEAPRAID_SAS_NEG_LINK_RATE_SHIFT;
		switch (reason_code) {
		case LEAPRAID_EVT_SAS_TOPO_RC_TARG_ADDED:
			if (adapter->access_ctrl.shost_recovering)
				break;
			leapraid_transport_update_links(adapter, sas_addr,
							hdl, phy_number,
							link_rate, card_port);
			if (link_rate < LEAPRAID_SAS_NEG_LINK_RATE_1_5)
				break;
			leapraid_add_dev(adapter, hdl);
			break;
		case LEAPRAID_EVT_SAS_TOPO_RC_TARG_NOT_RESPONDING:
			leapraid_sas_dev_remove_by_hdl(adapter, hdl);
			break;
		}
	}

	if (evt_data->exp_status == LEAPRAID_EVT_SAS_TOPO_ES_NOT_RESPONDING &&
	    topo_node_exp)
		leapraid_exp_rm(adapter, sas_addr, card_port);
}

static void leapraid_sas_topo_chg_evt(struct leapraid_adapter *adapter,
				      struct leapraid_fw_evt_work *fw_evt)
{
	struct leapraid_topo_node *topo_node_exp;
	struct leapraid_card_port *card_port;
	struct leapraid_evt_data_sas_topo_change_list *evt_data;
	u16 phdl;
	u8 max_phys;
	u64 sas_addr;
	unsigned long flags;

	if (adapter->access_ctrl.shost_recovering ||
	    adapter->access_ctrl.host_removing ||
	    adapter->access_ctrl.pcie_recovering)
		return;

	evt_data = fw_evt->evt_data;
	leapraid_sas_host_add(adapter, adapter->dev_topo.card.phys_num > 0);

	if (fw_evt->ignore)
		return;

	phdl = le16_to_cpu(evt_data->exp_dev_hdl);
	card_port = leapraid_get_port_by_id(adapter,
					    evt_data->physical_port,
					    false);
	if (evt_data->exp_status == LEAPRAID_EVT_SAS_TOPO_ES_ADDED &&
	    leapraid_exp_add(adapter, phdl) != 0)
		return;

	spin_lock_irqsave(&adapter->dev_topo.topo_node_lock, flags);
	topo_node_exp = leapraid_exp_find_by_hdl(adapter, phdl);
	if (topo_node_exp) {
		sas_addr = topo_node_exp->sas_address;
		max_phys = topo_node_exp->phys_num;
		card_port = topo_node_exp->card_port;
	} else if (phdl < adapter->dev_topo.card.phys_num) {
		sas_addr = adapter->dev_topo.card.sas_address;
		max_phys = adapter->dev_topo.card.phys_num;
	} else {
		spin_unlock_irqrestore(&adapter->dev_topo.topo_node_lock,
				       flags);
		return;
	}
	spin_unlock_irqrestore(&adapter->dev_topo.topo_node_lock, flags);

	leapraid_internal_sas_topo_chg_evt(adapter, card_port,
					   topo_node_exp, fw_evt,
					   sas_addr, max_phys);
}

static void leapraid_reprobe_lun(struct scsi_device *sdev, void *no_uld_attach)
{
	sdev->no_uld_attach = no_uld_attach ? 1 : 0;
	sdev_printk(KERN_INFO, sdev,
		    "%s RAID component to upper layer\n",
		    sdev->no_uld_attach ? "hide" : "expose");
	WARN_ON(scsi_device_reprobe(sdev));
}

static void leapraid_sas_pd_add(
		struct leapraid_adapter *adapter,
		struct leapraid_evt_data_ir_change *evt_data)
{
	union cfg_param_1 cfgp1 = {0};
	union cfg_param_2 cfgp2 = {0};
	struct leapraid_sas_dev_p0 sas_dev_p0;
	struct leapraid_sas_dev *sas_dev;
	u64 sas_address;
	u16 parent_hdl;
	u16 hdl;

	hdl = le16_to_cpu(evt_data->phys_disk_dev_hdl);
	if (!hdl || hdl > adapter->adapter_attr.features.max_dev_handle) {
		dev_warn(&adapter->pdev->dev,
			 "%s: Invalid device handle\n", __func__);
		return;
	}

	set_bit(hdl, adapter->dev_topo.pd_hdls);
	sas_dev = leapraid_get_sas_dev_by_hdl(adapter, hdl);
	if (sas_dev) {
		leapraid_sdev_put(sas_dev);
		dev_warn(&adapter->pdev->dev,
			 "Dev handle 0x%x already exists\n", hdl);
		return;
	}

	cfgp1.form = LEAPRAID_SAS_DEV_CFG_PGAD_HDL;
	cfgp2.handle = hdl;
	if (leapraid_op_config_page(adapter, &sas_dev_p0, cfgp1, cfgp2,
				    GET_SAS_DEVICE_PG0)) {
		dev_warn(&adapter->pdev->dev, "Failed to read dev page0\n");
		return;
	}

	parent_hdl = le16_to_cpu(sas_dev_p0.parent_dev_hdl);
	if (!leapraid_get_sas_address(adapter, parent_hdl, &sas_address))
		leapraid_transport_update_links(adapter, sas_address, hdl,
						sas_dev_p0.phy_num,
						LEAPRAID_SAS_NEG_LINK_RATE_1_5,
			 leapraid_get_port_by_id(adapter,
						 sas_dev_p0.physical_port,
						 false));
	leapraid_add_dev(adapter, hdl);
}

static void leapraid_sas_pd_delete(
		struct leapraid_adapter *adapter,
		struct leapraid_evt_data_ir_change *evt_data)
{
	u16 hdl;

	hdl = le16_to_cpu(evt_data->phys_disk_dev_hdl);
	leapraid_sas_dev_remove_by_hdl(adapter, hdl);
}

static void leapraid_sas_pd_hide(
		struct leapraid_adapter *adapter,
		struct leapraid_evt_data_ir_change *evt_data)
{
	struct leapraid_starget_priv *starget_priv;
	struct scsi_target *starget = NULL;
	struct leapraid_sas_dev *sas_dev;
	unsigned long flags;
	u64 volume_wwid = 0;
	u16 volume_hdl;
	u16 hdl;

	hdl = le16_to_cpu(evt_data->phys_disk_dev_hdl);
	if (!hdl || hdl > adapter->adapter_attr.features.max_dev_handle) {
		dev_warn(&adapter->pdev->dev,
			 "%s: Invalid device handle\n", __func__);
		return;
	}
	leapraid_cfg_get_volume_hdl(adapter, hdl, &volume_hdl);
	if (volume_hdl)
		leapraid_cfg_get_volume_wwid(adapter,
					     volume_hdl,
					     &volume_wwid);

	spin_lock_irqsave(&adapter->dev_topo.sas_dev_lock, flags);
	sas_dev = leapraid_hold_lock_get_sas_dev_by_hdl(adapter, hdl);
	if (!sas_dev) {
		spin_unlock_irqrestore(&adapter->dev_topo.sas_dev_lock, flags);
		return;
	}

	set_bit(hdl, adapter->dev_topo.pd_hdls);
	if (sas_dev->starget && sas_dev->starget->hostdata) {
		starget = sas_dev->starget;
		starget_priv = starget->hostdata;
		starget_priv->flg |= LEAPRAID_TGT_FLG_RAID_MEMBER;
		sas_dev->volume_hdl = volume_hdl;
		sas_dev->volume_wwid = volume_wwid;
	}
	spin_unlock_irqrestore(&adapter->dev_topo.sas_dev_lock, flags);
	if (starget) {
		starget_for_each_device(starget,
					(void *)LEAPRAID_NO_ULD_ATTACH_FLAG,
					leapraid_reprobe_lun);
	}

	leapraid_sdev_put(sas_dev);
}

static void leapraid_sas_pd_expose(
		struct leapraid_adapter *adapter,
		struct leapraid_evt_data_ir_change *evt_data)
{
	struct leapraid_starget_priv *starget_priv;
	struct scsi_target *starget = NULL;
	struct leapraid_sas_dev *sas_dev;
	unsigned long flags;
	u16 hdl;

	hdl = le16_to_cpu(evt_data->phys_disk_dev_hdl);
	if (!hdl || hdl > adapter->adapter_attr.features.max_dev_handle) {
		dev_warn(&adapter->pdev->dev,
			 "%s: Invalid device handle\n", __func__);
		return;
	}

	spin_lock_irqsave(&adapter->dev_topo.sas_dev_lock, flags);
	sas_dev = leapraid_hold_lock_get_sas_dev_by_hdl(adapter, hdl);
	if (!sas_dev) {
		dev_warn(&adapter->pdev->dev,
			 "%s:%d: sas_dev not found, hdl=0x%x\n",
			 __func__, __LINE__, hdl);
		spin_unlock_irqrestore(&adapter->dev_topo.sas_dev_lock, flags);
		return;
	}

	sas_dev->volume_hdl = 0;
	sas_dev->volume_wwid = 0;
	clear_bit(hdl, adapter->dev_topo.pd_hdls);
	if (sas_dev->starget && sas_dev->starget->hostdata) {
		starget = sas_dev->starget;
		starget_priv = starget->hostdata;
		starget_priv->flg &= ~LEAPRAID_TGT_FLG_RAID_MEMBER;
		sas_dev->led_on = 0;
	}
	spin_unlock_irqrestore(&adapter->dev_topo.sas_dev_lock, flags);

	if (starget) {
		starget_for_each_device(starget,
					NULL,
					leapraid_reprobe_lun);
	}

	leapraid_sdev_put(sas_dev);
}

static void leapraid_sas_vol_visibility(
		struct leapraid_adapter *adapter,
		struct leapraid_evt_data_ir_change *evt_data)
{
	struct leapraid_raid_volume *raid_volume;
	struct scsi_device *sdev;
	bool reprobe_flg = false;
	bool sdev_held = false;
	unsigned long flags;
	u16 hdl;
	u8 rc;

	hdl = le16_to_cpu(evt_data->vol_dev_hdl);
	rc = evt_data->reason_code;
	raid_volume = leapraid_raid_volume_find_by_hdl(adapter, hdl);
	if (!raid_volume) {
		dev_warn(&adapter->pdev->dev,
			 "%s:%d: Volume handle 0x%x not found\n",
			 __func__, __LINE__, hdl);
		return;
	}

	spin_lock_irqsave(&adapter->dev_topo.raid_volume_lock, flags);
	sdev = raid_volume->sdev;
	if (!sdev) {
		spin_unlock_irqrestore(&adapter->dev_topo.raid_volume_lock,
				       flags);
		leapraid_raid_volume_put(raid_volume);
		dev_warn(&adapter->pdev->dev,
			 "%s:%d: Volume handle 0x%x has no sdev\n",
			 __func__, __LINE__, hdl);
		return;
	}

	if (sdev->no_uld_attach &&
	    rc == LEAPRAID_EVT_IR_RC_VOLUME_UNHIDE) {
		sdev->no_uld_attach = 0;
		reprobe_flg = true;
	} else if (!sdev->no_uld_attach &&
		   rc == LEAPRAID_EVT_IR_RC_VOLUME_HIDE) {
		sdev->no_uld_attach = 1;
		reprobe_flg = true;
	}

	if (reprobe_flg && !scsi_device_get(sdev))
		sdev_held = true;
	spin_unlock_irqrestore(&adapter->dev_topo.raid_volume_lock, flags);
	leapraid_raid_volume_put(raid_volume);

	if (!reprobe_flg) {
		dev_warn(&adapter->pdev->dev,
			 "%s:rc(0x%x): Request matches, skipping\n",
			 __func__, rc);
		return;
	}

	if (!sdev_held) {
		dev_warn(&adapter->pdev->dev,
			 "%s: Failed to hold sdev for reprobe, hdl=0x%x\n",
			 __func__, hdl);
		return;
	}

	if (sdev->no_uld_attach)
		sdev_printk(KERN_INFO, sdev, "hide vol\n");
	else
		sdev_printk(KERN_INFO, sdev, "unhide vol\n");

	WARN_ON(scsi_device_reprobe(sdev));
	scsi_device_put(sdev);
}

static void leapraid_sas_volume_add(
		struct leapraid_adapter *adapter,
		struct leapraid_evt_data_ir_change *evt_data)
{
	struct leapraid_raid_volume *raid_volume;
	unsigned long flags;
	u64 wwid;
	u16 hdl;

	hdl = le16_to_cpu(evt_data->vol_dev_hdl);

	if (leapraid_cfg_get_volume_wwid(adapter, hdl, &wwid)) {
		dev_warn(&adapter->pdev->dev, "Failed to read volume page1\n");
		return;
	}

	if (!wwid) {
		dev_warn(&adapter->pdev->dev, "Invalid WWID(handle=0x%x)\n",
			 hdl);
		return;
	}

	raid_volume = leapraid_raid_volume_find_by_wwid(adapter, wwid);
	if (raid_volume) {
		dev_warn(&adapter->pdev->dev,
			 "Volume handle 0x%x already exists\n", hdl);
		leapraid_raid_volume_put(raid_volume);
		return;
	}

	raid_volume = kzalloc_obj(*raid_volume);
	if (!raid_volume)
		return;

	INIT_LIST_HEAD(&raid_volume->list);
	kref_init(&raid_volume->refcnt);
	raid_volume->id = adapter->dev_topo.sas_id++;
	raid_volume->channel = RAID_CHANNEL;
	raid_volume->hdl = hdl;
	raid_volume->wwid = wwid;
	leapraid_raid_volume_add(adapter, raid_volume);
	if (!adapter->scan_dev_desc.wait_scan_dev_done) {
		if (scsi_add_device(adapter->shost, RAID_CHANNEL,
				    raid_volume->id, 0))
			leapraid_raid_volume_remove(adapter, raid_volume);
		dev_info(&adapter->pdev->dev,
			 "add RAID volume: hdl=0x%x, wwid=0x%llx\n", hdl, wwid);
	} else {
		spin_lock_irqsave(&adapter->dev_topo.raid_volume_lock, flags);
		leapraid_check_boot_dev(adapter, raid_volume, RAID_CHANNEL);
		spin_unlock_irqrestore(&adapter->dev_topo.raid_volume_lock,
				       flags);
	}
	leapraid_raid_volume_put(raid_volume);
}

static void leapraid_sas_volume_delete_by_ptr(
		struct leapraid_adapter *adapter,
		struct leapraid_raid_volume *raid_volume)
{
	struct leapraid_starget_priv *starget_priv;
	struct scsi_target *starget = NULL;
	unsigned long flags;
	bool in_list;

	if (!raid_volume)
		return;

	spin_lock_irqsave(&adapter->dev_topo.raid_volume_lock, flags);
	in_list = !list_empty(&raid_volume->list);
	if (in_list && raid_volume->starget) {
		starget = raid_volume->starget;
		starget_priv = starget->hostdata;
		if (starget_priv)
			starget_priv->deleted = 1;
	}
	spin_unlock_irqrestore(&adapter->dev_topo.raid_volume_lock, flags);

	if (!in_list)
		return;

	dev_info(&adapter->pdev->dev,
		 "delete RAID volume: hdl=0x%x, wwid=0x%llx\n",
		 raid_volume->hdl, raid_volume->wwid);
	leapraid_raid_volume_remove(adapter, raid_volume);

	if (starget)
		scsi_remove_target(&starget->dev);
}

static void leapraid_sas_volume_delete(struct leapraid_adapter *adapter,
				       u16 hdl)
{
	struct leapraid_raid_volume *raid_volume;

	raid_volume = leapraid_raid_volume_find_by_hdl(adapter, hdl);
	if (!raid_volume) {
		dev_warn(&adapter->pdev->dev,
			 "%s:%d: Volume handle 0x%x not found\n",
			 __func__, __LINE__, hdl);
		return;
	}

	leapraid_sas_volume_delete_by_ptr(adapter, raid_volume);
	leapraid_raid_volume_put(raid_volume);
}

static void leapraid_sas_ir_chg_evt(struct leapraid_adapter *adapter,
				    struct leapraid_fw_evt_work *fw_evt)
{
	struct leapraid_evt_data_ir_change *evt_data;

	evt_data = fw_evt->evt_data;

	switch (evt_data->reason_code) {
	case LEAPRAID_EVT_IR_RC_VOLUME_ADD:
		leapraid_sas_volume_add(adapter, evt_data);
		break;
	case LEAPRAID_EVT_IR_RC_VOLUME_DELETE:
		leapraid_sas_volume_delete(adapter,
					   le16_to_cpu(evt_data->vol_dev_hdl));
		break;
	case LEAPRAID_EVT_IR_RC_PD_HIDDEN_TO_ADD:
		leapraid_sas_pd_add(adapter, evt_data);
		break;
	case LEAPRAID_EVT_IR_RC_PD_UNHIDDEN_TO_DELETE:
		leapraid_sas_pd_delete(adapter, evt_data);
		break;
	case LEAPRAID_EVT_IR_RC_PD_CREATED_TO_HIDE:
		leapraid_sas_pd_hide(adapter, evt_data);
		break;
	case LEAPRAID_EVT_IR_RC_PD_DELETED_TO_EXPOSE:
		leapraid_sas_pd_expose(adapter, evt_data);
		break;
	case LEAPRAID_EVT_IR_RC_VOLUME_HIDE:
	case LEAPRAID_EVT_IR_RC_VOLUME_UNHIDE:
		leapraid_sas_vol_visibility(adapter, evt_data);
		break;
	default:
		break;
	}
}

static void leapraid_sas_enc_dev_stat_add_node(
		struct leapraid_adapter *adapter, u16 hdl)
{
	union cfg_param_1 cfgp1 = {0};
	union cfg_param_2 cfgp2 = {0};
	struct leapraid_enc_node *enc_node;
	struct leapraid_enc_node *enc_exist;
	unsigned long flags;
	int rc;

	enc_node = kzalloc_obj(*enc_node);
	if (!enc_node)
		return;

	cfgp1.form = LEAPRAID_SAS_ENC_CFG_PGAD_HDL;
	cfgp2.handle = hdl;
	rc = leapraid_op_config_page(adapter, &enc_node->pg0, cfgp1, cfgp2,
				     GET_SAS_ENCLOSURE_PG0);
	if (rc) {
		kfree(enc_node);
		return;
	}

	spin_lock_irqsave(&adapter->dev_topo.enc_lock, flags);
	enc_exist = leapraid_enc_find_by_hdl(adapter, hdl);
	if (enc_exist) {
		spin_unlock_irqrestore(&adapter->dev_topo.enc_lock, flags);
		kfree(enc_node);
		return;
	}
	list_add_tail(&enc_node->list, &adapter->dev_topo.enc_list);
	spin_unlock_irqrestore(&adapter->dev_topo.enc_lock, flags);
}

static void leapraid_sas_enc_dev_stat_del_node(
		struct leapraid_adapter *adapter, u16 hdl)
{
	struct leapraid_enc_node *enc_node;
	unsigned long flags;

	if (!hdl)
		return;

	spin_lock_irqsave(&adapter->dev_topo.enc_lock, flags);
	enc_node = leapraid_enc_find_by_hdl(adapter, hdl);
	if (enc_node)
		list_del(&enc_node->list);
	spin_unlock_irqrestore(&adapter->dev_topo.enc_lock, flags);

	kfree(enc_node);
}

static void leapraid_sas_enc_dev_stat_chg_evt(
		struct leapraid_adapter *adapter,
		struct leapraid_fw_evt_work *fw_evt)
{
	struct leapraid_evt_data_sas_enc_dev_status_change *evt_data;
	u16 enc_hdl;

	if (adapter->access_ctrl.shost_recovering)
		return;

	evt_data = fw_evt->evt_data;
	enc_hdl = le16_to_cpu(evt_data->enc_hdl);
	switch (evt_data->reason_code) {
	case LEAPRAID_EVT_SAS_ENCL_RC_ADDED:
		if (enc_hdl)
			leapraid_sas_enc_dev_stat_add_node(adapter, enc_hdl);
		break;
	case LEAPRAID_EVT_SAS_ENCL_RC_NOT_RESPONDING:
		leapraid_sas_enc_dev_stat_del_node(adapter, enc_hdl);
		break;
	default:
		break;
	}
}

static void leapraid_remove_unresp_sas_end_dev(
		struct leapraid_adapter *adapter)
{
	struct leapraid_sas_dev *sas_dev, *sas_dev_next;
	unsigned long flags;
	LIST_HEAD(head);

	spin_lock_irqsave(&adapter->dev_topo.sas_dev_lock, flags);
	list_for_each_entry_safe(sas_dev, sas_dev_next,
				 &adapter->dev_topo.sas_dev_init_list, list) {
		if (sas_dev->rphy || sas_dev->pend_sas_rphy_add)
			continue;

		list_del_init(&sas_dev->list);
		leapraid_clear_cached_boot_dev(adapter, sas_dev, 0);
		leapraid_sdev_put(sas_dev);
	}
	list_for_each_entry_safe(sas_dev, sas_dev_next,
				 &adapter->dev_topo.sas_dev_list, list) {
		if (!sas_dev->resp)
			list_move_tail(&sas_dev->list, &head);
		else
			sas_dev->resp = 0;
	}
	spin_unlock_irqrestore(&adapter->dev_topo.sas_dev_lock, flags);

	list_for_each_entry_safe(sas_dev, sas_dev_next, &head, list) {
		leapraid_remove_device(adapter, sas_dev);
		list_del_init(&sas_dev->list);
		leapraid_sdev_put(sas_dev);
	}

	dev_warn(&adapter->pdev->dev,
		 "Unresponsive SAS end devices removed\n");
}

static void leapraid_remove_unresp_raid_volumes(
		struct leapraid_adapter *adapter)
{
	unsigned long flags;
	struct leapraid_raid_volume *raid_volume, *raid_volume_next;
	LIST_HEAD(head);

	spin_lock_irqsave(&adapter->dev_topo.raid_volume_lock, flags);
	list_for_each_entry_safe(raid_volume, raid_volume_next,
				 &adapter->dev_topo.raid_volume_list, list) {
		if (!raid_volume->resp)
			list_move_tail(&raid_volume->list, &head);
		else
			raid_volume->resp = 0;
	}
	spin_unlock_irqrestore(&adapter->dev_topo.raid_volume_lock, flags);

	list_for_each_entry_safe(raid_volume, raid_volume_next, &head, list) {
		leapraid_sas_volume_delete_by_ptr(adapter, raid_volume);
	}

	dev_warn(&adapter->pdev->dev,
		 "Unresponsive RAID volumes removed\n");
}

static void leapraid_remove_unresp_sas_exp(struct leapraid_adapter *adapter)
{
	struct leapraid_topo_node *topo_node_exp, *topo_node_exp_next;
	unsigned long flags;
	LIST_HEAD(head);

	spin_lock_irqsave(&adapter->dev_topo.topo_node_lock, flags);
	list_for_each_entry_safe(topo_node_exp, topo_node_exp_next,
				 &adapter->dev_topo.exp_list, list) {
		if (!topo_node_exp->resp)
			list_move_tail(&topo_node_exp->list, &head);
		else
			topo_node_exp->resp = 0;
	}
	spin_unlock_irqrestore(&adapter->dev_topo.topo_node_lock, flags);

	list_for_each_entry_safe(topo_node_exp, topo_node_exp_next,
				 &head, list)
		leapraid_exp_node_rm(adapter, topo_node_exp);

	dev_warn(&adapter->pdev->dev,
		 "Unresponsive SAS expanders removed\n");
}

static void leapraid_remove_unresp_dev(struct leapraid_adapter *adapter)
{
	leapraid_remove_unresp_sas_end_dev(adapter);
	if (adapter->adapter_attr.raid_support)
		leapraid_remove_unresp_raid_volumes(adapter);
	leapraid_remove_unresp_sas_exp(adapter);
	leapraid_ublk_io_all_dev(adapter);
}

static void leapraid_del_dirty_vphy(struct leapraid_adapter *adapter)
{
	struct leapraid_card_port *card_port, *card_port_next;
	struct leapraid_vphy *vphy, *vphy_next;

	list_for_each_entry_safe(card_port, card_port_next,
				 &adapter->dev_topo.card_port_list, list) {
		if (!card_port->vphys_mask)
			continue;

		list_for_each_entry_safe(vphy, vphy_next,
					 &card_port->vphys_list, list) {
			if (!(vphy->flg & LEAPRAID_VPHY_FLG_DIRTY))
				continue;

			card_port->vphys_mask &= ~vphy->phy_mask;
			list_del(&vphy->list);
			kfree(vphy);
		}

		if (!card_port->vphys_mask && !card_port->sas_address)
			card_port->flg |= LEAPRAID_CARD_PORT_FLG_DIRTY;
	}
}

static void leapraid_del_dirty_card_port(struct leapraid_adapter *adapter)
{
	struct leapraid_card_port *card_port, *card_port_next;

	list_for_each_entry_safe(card_port, card_port_next,
				 &adapter->dev_topo.card_port_list, list) {
		if (!(card_port->flg & LEAPRAID_CARD_PORT_FLG_DIRTY) ||
		    card_port->flg & LEAPRAID_CARD_PORT_FLG_NEW)
			continue;

		list_del(&card_port->list);
		kfree(card_port);
	}
}

static void leapraid_update_dev_qdepth(struct leapraid_adapter *adapter)
{
	struct leapraid_sdev_priv *sdev_priv;
	struct leapraid_sas_dev *sas_dev;
	struct leapraid_adapter_attr *attr;
	struct scsi_device *sdev;
	u16 qdepth;

	attr = &adapter->adapter_attr;
	shost_for_each_device(sdev, adapter->shost) {
		sdev_priv = sdev->hostdata;
		if (!sdev_priv || !sdev_priv->starget_priv)
			continue;
		sas_dev = sdev_priv->starget_priv->sas_dev;
		if (sas_dev && sas_dev->dev_info & LEAPRAID_DEVTYP_SSP_TGT)
			qdepth = (sas_dev->port_connection > 1) ?
				attr->wideport_max_queue_depth :
				attr->narrowport_max_queue_depth;
		else if (sas_dev && sas_dev->dev_info &
			 LEAPRAID_DEVTYP_SATA_DEV)
			qdepth = attr->sata_max_queue_depth;
		else
			continue;

		leapraid_change_queue_depth(sdev, qdepth);
	}
}

static void leapraid_update_exp_links(struct leapraid_adapter *adapter,
				      struct leapraid_topo_node *topo_node_exp,
				      u16 hdl)
{
	union cfg_param_1 cfgp1 = {0};
	union cfg_param_2 cfgp2 = {0};
	struct leapraid_exp_p1 exp_p1;
	int i;

	cfgp2.handle = hdl;
	for (i = 0; i < topo_node_exp->phys_num; i++) {
		cfgp1.phy_number = i;
		if (leapraid_op_config_page(adapter, &exp_p1, cfgp1, cfgp2,
					    GET_SAS_EXPANDER_PG1))
			return;

		leapraid_transport_update_links(
			adapter,
			topo_node_exp->sas_address,
			le16_to_cpu(exp_p1.attached_dev_hdl),
			i,
			exp_p1.neg_link_rate >>
				LEAPRAID_SAS_NEG_LINK_RATE_SHIFT,
			topo_node_exp->card_port);
	}
}

static void leapraid_scan_exp_after_reset(struct leapraid_adapter *adapter)
{
	union cfg_param_1 cfgp1 = {0};
	union cfg_param_2 cfgp2 = {0};
	struct leapraid_topo_node *topo_node_exp;
	struct leapraid_exp_p0 exp_p0;
	unsigned long flags;
	u16 hdl;
	u8 port_id;

	cfgp1.form = LEAPRAID_SAS_CFG_PGAD_GET_NEXT_LOOP;
	for (hdl = 0xFFFF, cfgp2.handle = hdl;
	     !leapraid_op_config_page(adapter, &exp_p0, cfgp1, cfgp2,
				      GET_SAS_EXPANDER_PG0);
	     cfgp2.handle = hdl) {
		hdl = le16_to_cpu(exp_p0.dev_hdl);
		port_id = exp_p0.physical_port;
		spin_lock_irqsave(&adapter->dev_topo.topo_node_lock, flags);
		topo_node_exp =
			leapraid_exp_find_by_sas_address(
				adapter,
				le64_to_cpu(exp_p0.sas_address),
				leapraid_get_port_by_id(adapter,
							port_id,
							false));
		spin_unlock_irqrestore(&adapter->dev_topo.topo_node_lock,
				       flags);

		if (topo_node_exp) {
			leapraid_update_exp_links(adapter, topo_node_exp, hdl);
		} else {
			leapraid_exp_add(adapter, hdl);

			dev_info(&adapter->pdev->dev,
				 "add exp: hdl=0x%04x, SAS addr=0x%016llx\n",
				 hdl,
				 (unsigned long long)le64_to_cpu(
					exp_p0.sas_address));
		}
	}
}

static void leapraid_scan_phy_disks_after_reset(
		struct leapraid_adapter *adapter)
{
	union cfg_param_1 cfgp1 = {0};
	union cfg_param_2 cfgp2 = {0};
	union cfg_param_1 cfgp1_extra = {0};
	union cfg_param_2 cfgp2_extra = {0};
	struct leapraid_sas_dev_p0 sas_dev_p0;
	struct leapraid_raidpd_p0 raidpd_p0;
	struct leapraid_sas_dev *sas_dev;
	u8 phys_disk_num, port_id;
	u16 hdl, parent_hdl;
	u64 sas_addr;

	cfgp1.form = LEAPRAID_SAS_CFG_PGAD_GET_NEXT_LOOP;
	for (phys_disk_num = 0xFF, cfgp2.form_specific = phys_disk_num;
	     !leapraid_op_config_page(adapter, &raidpd_p0,
				      cfgp1, cfgp2, GET_PHY_DISK_PG0);
	     cfgp2.form_specific = phys_disk_num) {
		phys_disk_num = raidpd_p0.phys_disk_num;
		hdl = le16_to_cpu(raidpd_p0.dev_hdl);
		sas_dev = leapraid_get_sas_dev_by_hdl(adapter, hdl);
		if (sas_dev) {
			leapraid_sdev_put(sas_dev);
			continue;
		}

		cfgp1_extra.form = LEAPRAID_SAS_DEV_CFG_PGAD_HDL;
		cfgp2_extra.handle = hdl;
		if (leapraid_op_config_page(adapter, &sas_dev_p0, cfgp1_extra,
					    cfgp2_extra, GET_SAS_DEVICE_PG0) !=
					    0)
			continue;

		parent_hdl = le16_to_cpu(sas_dev_p0.parent_dev_hdl);
		if (!leapraid_get_sas_address(adapter,
					      parent_hdl,
					      &sas_addr)) {
			port_id = sas_dev_p0.physical_port;
			leapraid_transport_update_links(
				adapter, sas_addr, hdl,
				sas_dev_p0.phy_num,
				LEAPRAID_SAS_NEG_LINK_RATE_1_5,
				leapraid_get_port_by_id(
					adapter, port_id, false));
			if (!hdl || hdl >
			    adapter->adapter_attr.features.max_dev_handle) {
				dev_warn(&adapter->pdev->dev,
					 "%s: Invalid device handle\n",
					 __func__);
			} else {
				set_bit(hdl, adapter->dev_topo.pd_hdls);
				leapraid_add_dev(adapter, hdl);

				dev_info(&adapter->pdev->dev,
					 "add pd hdl=0x%04x saddr=0x%016llx\n",
					 hdl,
					 (unsigned long long)le64_to_cpu(
						sas_dev_p0.sas_address));
			}
		}
	}
}

static void leapraid_scan_vol_after_reset(struct leapraid_adapter *adapter)
{
	union cfg_param_1 cfgp1 = {0};
	union cfg_param_2 cfgp2 = {0};
	union cfg_param_1 cfgp1_extra = {0};
	union cfg_param_2 cfgp2_extra = {0};
	struct leapraid_evt_data_ir_change evt_data;
	struct leapraid_raid_volume *raid_volume;
	struct leapraid_raidvol_p1 *vol_p1;
	struct leapraid_raidvol_p0 *vol_p0;
	u16 hdl;

	vol_p0 = kzalloc_obj(*vol_p0);
	if (!vol_p0)
		return;

	vol_p1 = kzalloc_obj(*vol_p1);
	if (!vol_p1) {
		kfree(vol_p0);
		return;
	}

	cfgp1.form = LEAPRAID_SAS_CFG_PGAD_GET_NEXT_LOOP;
	for (hdl = 0xFFFF, cfgp2.handle = hdl;
	     !leapraid_op_config_page(adapter, vol_p1, cfgp1,
				      cfgp2, GET_RAID_VOLUME_PG1);
	     cfgp2.handle = hdl) {
		hdl = le16_to_cpu(vol_p1->dev_hdl);
		raid_volume = leapraid_raid_volume_find_by_wwid(
				adapter,
				le64_to_cpu(vol_p1->wwid));
		if (raid_volume) {
			leapraid_raid_volume_put(raid_volume);
			continue;
		}

		cfgp1_extra.size = sizeof(struct leapraid_raidvol_p0);
		cfgp2_extra.handle = hdl;
		if (leapraid_op_config_page(adapter, vol_p0, cfgp1_extra,
					    cfgp2_extra, GET_RAID_VOLUME_PG0))
			continue;

		if (vol_p0->volume_state == LEAPRAID_VOL_STATE_OPTIMAL ||
		    vol_p0->volume_state == LEAPRAID_VOL_STATE_ONLINE ||
		    vol_p0->volume_state == LEAPRAID_VOL_STATE_DEGRADED) {
			memset(&evt_data, 0,
			       sizeof(struct leapraid_evt_data_ir_change));
			evt_data.reason_code = LEAPRAID_EVT_IR_RC_VOLUME_ADD;
			evt_data.vol_dev_hdl = vol_p1->dev_hdl;
			leapraid_sas_volume_add(adapter, &evt_data);
			dev_info(&adapter->pdev->dev,
				 "add volume: hdl=0x%04x\n",
				 vol_p1->dev_hdl);
		}
	}

	kfree(vol_p0);
	kfree(vol_p1);
}

static void leapraid_scan_sas_dev_after_reset(struct leapraid_adapter *adapter)
{
	union cfg_param_1 cfgp1 = {0};
	union cfg_param_2 cfgp2 = {0};
	struct leapraid_sas_dev_p0 sas_dev_p0;
	struct leapraid_sas_dev *sas_dev;
	u16 hdl, parent_hdl;
	u64 sas_address;
	u8 port_id;

	cfgp1.form = LEAPRAID_SAS_CFG_PGAD_GET_NEXT_LOOP;
	for (hdl = 0xFFFF, cfgp2.handle = hdl;
	     !leapraid_op_config_page(adapter, &sas_dev_p0, cfgp1, cfgp2,
				      GET_SAS_DEVICE_PG0);
	     cfgp2.handle = hdl) {
		hdl = le16_to_cpu(sas_dev_p0.dev_hdl);
		if (!hdl ||
		    hdl > adapter->adapter_attr.features.max_dev_handle) {
			dev_warn(&adapter->pdev->dev,
				 "%s: Invalid device handle\n", __func__);
			continue;
		}

		if (!(leapraid_is_end_dev(le32_to_cpu(sas_dev_p0.dev_info))))
			continue;

		port_id = sas_dev_p0.physical_port;
		sas_dev = leapraid_get_sas_dev_by_addr(
				adapter,
				le64_to_cpu(sas_dev_p0.sas_address),
				leapraid_get_port_by_id(
					adapter,
					port_id,
					false));
		if (sas_dev) {
			leapraid_sdev_put(sas_dev);
			continue;
		}

		parent_hdl = le16_to_cpu(sas_dev_p0.parent_dev_hdl);
		if (!leapraid_get_sas_address(adapter, parent_hdl,
					      &sas_address)) {
			leapraid_transport_update_links(
				adapter,
				sas_address,
				hdl,
				sas_dev_p0.phy_num,
				LEAPRAID_SAS_NEG_LINK_RATE_1_5,
				leapraid_get_port_by_id(adapter,
							port_id,
							false));
			leapraid_add_dev(adapter, hdl);
			dev_info(&adapter->pdev->dev,
				 "Add SAS dev: hdl=0x%04x, saddr=0x%016llx\n",
				 hdl,
				 (unsigned long long)le64_to_cpu(
					sas_dev_p0.sas_address));
		}
	}
}

static void leapraid_scan_all_dev_after_reset(struct leapraid_adapter *adapter)
{
	leapraid_sas_host_add(adapter, adapter->dev_topo.card.phys_num > 0);
	leapraid_scan_exp_after_reset(adapter);
	if (adapter->adapter_attr.raid_support) {
		leapraid_scan_phy_disks_after_reset(adapter);
		leapraid_scan_vol_after_reset(adapter);
	}
	leapraid_scan_sas_dev_after_reset(adapter);
}

static void leapraid_hardreset_async_logic(struct leapraid_adapter *adapter)
{
	unsigned long flags;

	leapraid_remove_unresp_dev(adapter);
	leapraid_del_dirty_vphy(adapter);
	leapraid_del_dirty_card_port(adapter);
	leapraid_update_dev_qdepth(adapter);
	leapraid_scan_all_dev_after_reset(adapter);

	if (adapter->scan_dev_desc.driver_loading)
		leapraid_scan_dev_done(adapter);
	spin_lock_irqsave(&adapter->reset_desc.adapter_reset_lock, flags);
	adapter->access_ctrl.shost_recover_async = 0;
	spin_unlock_irqrestore(&adapter->reset_desc.adapter_reset_lock, flags);
	wake_up(&adapter->access_ctrl.shost_recover_wq);
}

static int leapraid_send_enc_cmd(struct leapraid_adapter *adapter,
				 struct leapraid_sep_rep *sep_rep,
				 struct leapraid_sep_req *sep_req)
{
	void *req;
	bool reset_flg = false;
	int rc;
	u16 smid;

	mutex_lock(&adapter->driver_cmds.enc_cmd.mutex);
	rc = leapraid_check_adapter_is_op(adapter, LEAPRAID_DB_WAIT_OP_SHORT,
					  __func__);
	if (rc)
		goto unlock;

	adapter->driver_cmds.enc_cmd.status = LEAPRAID_CMD_PENDING;
	smid = adapter->driver_cmds.enc_cmd.inter_taskid;
	req = leapraid_get_task_desc(adapter, smid);
	memset(req, 0, LEAPRAID_REQUEST_SIZE);
	memcpy(req, sep_req, sizeof(struct leapraid_sep_req));
	init_completion(&adapter->driver_cmds.enc_cmd.done);
	leapraid_fire_task(adapter, smid);
	wait_for_completion_timeout(&adapter->driver_cmds.enc_cmd.done,
				    LEAPRAID_ENC_CMD_TIMEOUT * HZ);
	if (!(adapter->driver_cmds.enc_cmd.status & LEAPRAID_CMD_DONE)) {
		dev_err(&adapter->pdev->dev,
			"%s: SEP command timeout, status=0x%x\n",
			__func__, adapter->driver_cmds.enc_cmd.status);
		leapraid_log_req_context(adapter, smid, sep_req);
		reset_flg =
			leapraid_check_reset(
				adapter->driver_cmds.enc_cmd.status);
		rc = -EFAULT;
		goto do_hard_reset;
	}

	if (adapter->driver_cmds.enc_cmd.status & LEAPRAID_CMD_REPLY_VALID)
		memcpy(sep_rep, &adapter->driver_cmds.enc_cmd.reply,
		       sizeof(struct leapraid_sep_rep));
do_hard_reset:
	if (reset_flg) {
		dev_info(&adapter->pdev->dev, "%s:%d: call hard_reset\n",
			 __func__, __LINE__);
		leapraid_hard_reset_handler(adapter, FULL_RESET);
	}

	adapter->driver_cmds.enc_cmd.status = LEAPRAID_CMD_NOT_USED;
unlock:
	mutex_unlock(&adapter->driver_cmds.enc_cmd.mutex);
	return rc;
}

static void leapraid_set_led(struct leapraid_adapter *adapter,
			     struct leapraid_sas_dev *sas_dev, bool on)
{
	struct leapraid_sep_rep sep_rep;
	struct leapraid_sep_req sep_req;

	if (!sas_dev)
		return;

	memset(&sep_req, 0, sizeof(struct leapraid_sep_req));
	memset(&sep_rep, 0, sizeof(struct leapraid_sep_rep));
	sep_req.func = LEAPRAID_FUNC_SCSI_ENC_PROCESSOR;
	sep_req.act = LEAPRAID_SEP_REQ_ACT_WRITE_STATUS;
	if (on) {
		u32 status = LEAPRAID_SEP_REQ_SLOTSTATUS_PREDICTED_FAULT;

		sep_req.slot_status = cpu_to_le32(status);
		sep_req.dev_hdl = cpu_to_le16(sas_dev->hdl);
		sep_req.flg = LEAPRAID_SEP_REQ_FLG_DEVHDL_ADDRESS;
		if (leapraid_send_enc_cmd(adapter, &sep_rep, &sep_req)) {
			leapraid_sdev_put(sas_dev);
			return;
		}

		sas_dev->led_on = 1;
		leapraid_sdev_put(sas_dev);
	} else {
		sep_req.slot_status = 0;
		sep_req.slot = cpu_to_le16(sas_dev->slot);
		sep_req.dev_hdl = 0;
		sep_req.enc_hdl = cpu_to_le16(sas_dev->enc_hdl);
		sep_req.flg = LEAPRAID_SEP_REQ_FLG_ENCLOSURE_SLOT_ADDRESS;
		leapraid_send_enc_cmd(adapter, &sep_rep, &sep_req);
	}
}

static int leapraid_wait_adapter_recovery(struct leapraid_adapter *adapter)
{
	unsigned long flags;

	while (leapraid_shost_in_recovery(adapter->shost) ||
	       READ_ONCE(adapter->access_ctrl.shost_recovering)) {
		if (READ_ONCE(adapter->access_ctrl.host_removing) ||
		    READ_ONCE(adapter->fw_evt_s.fw_evt_cleanup)) {
			spin_lock_irqsave(
				&adapter->reset_desc.adapter_reset_lock,
				flags);
			adapter->access_ctrl.shost_recover_async = 0;
			spin_unlock_irqrestore(
				&adapter->reset_desc.adapter_reset_lock,
				flags);

			wake_up(&adapter->access_ctrl.shost_recover_wq);

			dev_warn(&adapter->pdev->dev,
				 "%s: Failed, shost %d, host %d\n",
				 __func__,
				 adapter->access_ctrl.shost_recovering,
				 adapter->access_ctrl.host_removing);

			return -EFAULT;
		}

		wait_event_timeout(
			adapter->access_ctrl.recovery_waitq,
			(!leapraid_shost_in_recovery(adapter->shost) &&
			 !READ_ONCE(adapter->access_ctrl.shost_recovering)),
			msecs_to_jiffies(1000));
	}

	return 0;
}

static void leapraid_fw_work(struct leapraid_adapter *adapter,
			     struct leapraid_fw_evt_work *fw_evt)
{
	struct leapraid_sas_dev *sas_dev;
	unsigned long flags;

	spin_lock_irqsave(&adapter->fw_evt_s.fw_evt_lock, flags);
	adapter->fw_evt_s.cur_evt = fw_evt;
	adapter->fw_evt_s.cur_evt_task = current;
	spin_unlock_irqrestore(&adapter->fw_evt_s.fw_evt_lock, flags);
	leapraid_del_fw_evt_from_list(adapter, fw_evt);
	if (adapter->access_ctrl.host_removing ||
	    adapter->access_ctrl.pcie_recovering) {
		spin_lock_irqsave(&adapter->fw_evt_s.fw_evt_lock, flags);
		leapraid_fw_evt_put(fw_evt);
		adapter->fw_evt_s.cur_evt = NULL;
		adapter->fw_evt_s.cur_evt_task = NULL;
		spin_unlock_irqrestore(&adapter->fw_evt_s.fw_evt_lock, flags);
		return;
	}
	switch (fw_evt->evt_type) {
	case LEAPRAID_EVT_SAS_TOPO_CHANGE_LIST:
		leapraid_sas_topo_chg_evt(adapter, fw_evt);
		break;
	case LEAPRAID_EVT_IR_CHANGE:
		leapraid_sas_ir_chg_evt(adapter, fw_evt);
		break;
	case LEAPRAID_EVT_SAS_ENCL_DEV_STATUS_CHANGE:
		leapraid_sas_enc_dev_stat_chg_evt(adapter, fw_evt);
		break;
	case LEAPRAID_EVT_REMOVE_DEAD_DEV:
		if (leapraid_wait_adapter_recovery(adapter))
			goto out_cleanup;

		leapraid_hardreset_async_logic(adapter);
		break;
	case LEAPRAID_EVT_TURN_ON_PFA_LED:
		sas_dev = leapraid_get_sas_dev_by_hdl(adapter,
						      fw_evt->dev_handle);
		leapraid_set_led(adapter, sas_dev, true);
		break;
	case LEAPRAID_EVT_SCAN_DEV_DONE:
		adapter->scan_dev_desc.scan_start = 0;
		break;
	default:
		break;
	}
out_cleanup:
	spin_lock_irqsave(&adapter->fw_evt_s.fw_evt_lock, flags);
	leapraid_fw_evt_put(fw_evt);
	adapter->fw_evt_s.cur_evt = NULL;
	adapter->fw_evt_s.cur_evt_task = NULL;
	spin_unlock_irqrestore(&adapter->fw_evt_s.fw_evt_lock, flags);
}

static void leapraid_sas_dev_stat_chg_evt(
		struct leapraid_adapter *adapter,
		struct leapraid_evt_data_sas_dev_status_change *event_data)
{
	struct leapraid_starget_priv *starget_priv;
	struct leapraid_sas_dev *sas_dev;
	u64 sas_address;
	unsigned long flags;

	switch (event_data->reason_code) {
	case LEAPRAID_EVT_SAS_DEV_STAT_RC_INTERNAL_DEV_RESET:
	case LEAPRAID_EVT_SAS_DEV_STAT_RC_CMP_INTERNAL_DEV_RESET:
		break;
	default:
		return;
	}

	spin_lock_irqsave(&adapter->dev_topo.sas_dev_lock, flags);

	sas_address = le64_to_cpu(event_data->sas_address);
	sas_dev = leapraid_hold_lock_get_sas_dev_by_addr(
			adapter,
			sas_address,
			leapraid_get_port_by_id(adapter,
						event_data->physical_port,
						false));
	if (!sas_dev || !sas_dev->starget)
		goto out_unlock;

	starget_priv = sas_dev->starget->hostdata;
	if (starget_priv) {
		switch (event_data->reason_code) {
		case LEAPRAID_EVT_SAS_DEV_STAT_RC_INTERNAL_DEV_RESET:
			starget_priv->tm_busy = 1;
			break;
		case LEAPRAID_EVT_SAS_DEV_STAT_RC_CMP_INTERNAL_DEV_RESET:
			starget_priv->tm_busy = 0;
			break;
		}
	}

out_unlock:
	if (sas_dev)
		leapraid_sdev_put(sas_dev);
	spin_unlock_irqrestore(&adapter->dev_topo.sas_dev_lock, flags);
}

static void leapraid_set_volume_delete_flag(struct leapraid_adapter *adapter,
					    u16 handle)
{
	struct leapraid_raid_volume *raid_volume;
	struct leapraid_starget_priv *sas_target_priv_data;
	unsigned long flags;

	raid_volume = leapraid_raid_volume_find_by_hdl(adapter, handle);
	if (raid_volume) {
		spin_lock_irqsave(&adapter->dev_topo.raid_volume_lock, flags);
		if (raid_volume->starget &&
		    raid_volume->starget->hostdata) {
			sas_target_priv_data = raid_volume->starget->hostdata;
			sas_target_priv_data->deleted = 1;
		}
		spin_unlock_irqrestore(&adapter->dev_topo.raid_volume_lock,
				       flags);
		leapraid_raid_volume_put(raid_volume);
	}
}

static void leapraid_check_ir_change_evt(
		struct leapraid_adapter *adapter,
		struct leapraid_evt_data_ir_change *evt_data)
{
	u16 phys_disk_dev_hdl;

	switch (evt_data->reason_code) {
	case LEAPRAID_EVT_IR_RC_VOLUME_DELETE:
		leapraid_set_volume_delete_flag(
			adapter,
			le16_to_cpu(evt_data->vol_dev_hdl));
		break;
	case LEAPRAID_EVT_IR_RC_PD_UNHIDDEN_TO_DELETE:
		phys_disk_dev_hdl =
			le16_to_cpu(evt_data->phys_disk_dev_hdl);
		if (!phys_disk_dev_hdl ||
		    phys_disk_dev_hdl >
		    adapter->adapter_attr.features.max_dev_handle) {
			dev_warn(&adapter->pdev->dev,
				 "%s: Invalid device handle\n", __func__);
		} else {
			clear_bit(phys_disk_dev_hdl,
				  adapter->dev_topo.pd_hdls);
			leapraid_tgt_rst_send(adapter, phys_disk_dev_hdl);
		}
		break;
	}
}

static void leapraid_topo_del_evts_process_exp_status(
		struct leapraid_adapter *adapter,
		struct leapraid_evt_data_sas_topo_change_list *evt_data)
{
	struct leapraid_fw_evt_work *fw_evt = NULL;
	struct leapraid_evt_data_sas_topo_change_list *loc_evt_data;
	unsigned long flags;
	u16 exp_hdl;

	exp_hdl = le16_to_cpu(evt_data->exp_dev_hdl);

	switch (evt_data->exp_status) {
	case LEAPRAID_EVT_SAS_TOPO_ES_NOT_RESPONDING:
		spin_lock_irqsave(&adapter->fw_evt_s.fw_evt_lock, flags);
		list_for_each_entry(fw_evt,
				    &adapter->fw_evt_s.fw_evt_list, list) {
			if (fw_evt->evt_type !=
			    LEAPRAID_EVT_SAS_TOPO_CHANGE_LIST ||
			    fw_evt->ignore)
				continue;

			loc_evt_data = fw_evt->evt_data;
			if ((loc_evt_data->exp_status ==
			     LEAPRAID_EVT_SAS_TOPO_ES_ADDED ||
			     loc_evt_data->exp_status ==
			     LEAPRAID_EVT_SAS_TOPO_ES_RESPONDING) &&
			    le16_to_cpu(loc_evt_data->exp_dev_hdl) == exp_hdl)
				fw_evt->ignore = 1;
		}
		spin_unlock_irqrestore(&adapter->fw_evt_s.fw_evt_lock, flags);
		break;
	default:
		break;
	}
}

static void leapraid_check_topo_del_evts(
		struct leapraid_adapter *adapter,
		struct leapraid_evt_data_sas_topo_change_list *evt_data)
{
	int reason_code;
	u16 hdl;
	int i;

	for (i = 0; i < evt_data->entry_num; i++) {
		hdl = le16_to_cpu(evt_data->phy[i].attached_dev_hdl);
		if (!hdl)
			continue;

		reason_code = evt_data->phy[i].phy_status &
				LEAPRAID_EVT_SAS_TOPO_RC_MASK;
		if (reason_code ==
		    LEAPRAID_EVT_SAS_TOPO_RC_TARG_NOT_RESPONDING)
			leapraid_tgt_not_responding(adapter, hdl);
	}
	leapraid_topo_del_evts_process_exp_status(adapter, evt_data);
}

static bool leapraid_async_evt_validate(
		struct leapraid_adapter *adapter,
		struct leapraid_evt_notify_rep *event_notify_rep)
{
	size_t msg_len;
	size_t evt_sz;
	size_t evt_avail;
	u16 evt;

	msg_len = event_notify_rep->msg_len * sizeof(u32);
	if (msg_len > LEAPRAID_REPLY_SIZE ||
	    msg_len < offsetof(struct leapraid_evt_notify_rep, evt_data)) {
		dev_warn(&adapter->pdev->dev,
			 "%s: Invalid async event msg_len=%zu\n",
			 __func__, msg_len);
		return false;
	}

	evt_sz = le16_to_cpu(event_notify_rep->evt_data_len) * sizeof(u32);
	evt_avail = msg_len -
		    offsetof(struct leapraid_evt_notify_rep, evt_data);
	if (evt_sz > evt_avail) {
		dev_warn(&adapter->pdev->dev,
			 "%s: Invalid async event evt_data_len=%zu\n",
			 __func__, evt_sz);
		return false;
	}

	evt = le16_to_cpu(event_notify_rep->evt);
	switch (evt) {
	case LEAPRAID_EVT_SAS_DEV_STATUS_CHANGE:
		if (evt_sz <
		    sizeof(struct leapraid_evt_data_sas_dev_status_change))
			goto invalid_evt_sz;
		break;
	case LEAPRAID_EVT_IR_CHANGE:
		if (evt_sz < sizeof(struct leapraid_evt_data_ir_change))
			goto invalid_evt_sz;
		break;
	case LEAPRAID_EVT_SAS_TOPO_CHANGE_LIST:
	{
		struct leapraid_evt_data_sas_topo_change_list *evt_data =
			(void *)event_notify_rep->evt_data;
		size_t hdr_sz;

		hdr_sz =
			offsetof(struct leapraid_evt_data_sas_topo_change_list,
				 phy);
		if (evt_sz < hdr_sz)
			goto invalid_evt_sz;

		if (evt_data->entry_num >
		    (evt_sz - hdr_sz) / sizeof(evt_data->phy[0]))
			goto invalid_evt_sz;
		break;
	}
	case LEAPRAID_EVT_SAS_ENCL_DEV_STATUS_CHANGE:
		if (evt_sz <
		    sizeof(struct leapraid_evt_data_sas_enc_dev_status_change))
			goto invalid_evt_sz;
		break;
	default:
		break;
	}

	return true;

invalid_evt_sz:
	dev_warn(&adapter->pdev->dev,
		 "%s: Invalid async event size=%zu for evt=0x%x\n",
		 __func__, evt_sz, evt);
	return false;
}

static bool leapraid_async_process_evt(
		struct leapraid_adapter *adapter,
		struct leapraid_evt_notify_rep *event_notify_rep)
{
	u16 evt = le16_to_cpu(event_notify_rep->evt);
	bool exit_flag = false;

	if (adapter->access_ctrl.host_removing ||
	    adapter->access_ctrl.pcie_recovering)
		return true;

	switch (evt) {
	case LEAPRAID_EVT_SAS_DEV_STATUS_CHANGE:
		leapraid_sas_dev_stat_chg_evt(
			adapter,
			(struct leapraid_evt_data_sas_dev_status_change
				*)event_notify_rep->evt_data);
		break;
	case LEAPRAID_EVT_IR_CHANGE:
		leapraid_check_ir_change_evt(
			adapter,
			(struct leapraid_evt_data_ir_change
				*)event_notify_rep->evt_data);
		break;
	case LEAPRAID_EVT_SAS_TOPO_CHANGE_LIST:
		leapraid_check_topo_del_evts(
			adapter,
			(struct leapraid_evt_data_sas_topo_change_list
				*)event_notify_rep->evt_data);
		if (adapter->access_ctrl.shost_recovering) {
			exit_flag = true;
			return exit_flag;
		}
		break;
	case LEAPRAID_EVT_SAS_ENCL_DEV_STATUS_CHANGE:
		break;
	default:
		exit_flag = true;
		return exit_flag;
	}

	return exit_flag;
}

static void leapraid_async_evt_cb_enqueue(
		struct leapraid_adapter *adapter,
		struct leapraid_evt_notify_rep *evt_notify_rep)
{
	struct leapraid_fw_evt_work *fw_evt;
	u16 evt_sz;

	fw_evt = leapraid_alloc_fw_evt_work();
	if (!fw_evt)
		return;

	evt_sz = le16_to_cpu(evt_notify_rep->evt_data_len) * sizeof(u32);
	fw_evt->evt_data = kmemdup(evt_notify_rep->evt_data,
				   evt_sz, GFP_ATOMIC);
	if (!fw_evt->evt_data) {
		leapraid_fw_evt_put(fw_evt);
		return;
	}
	fw_evt->adapter = adapter;
	fw_evt->evt_type = le16_to_cpu(evt_notify_rep->evt);
	leapraid_fw_evt_add(adapter, fw_evt);
	leapraid_fw_evt_put(fw_evt);
}

static void leapraid_async_evt_cb(struct leapraid_adapter *adapter,
				  u8 msix_index, u32 rep_paddr)
{
	struct leapraid_evt_notify_rep *evt_notify_rep;

	if (adapter->access_ctrl.host_removing ||
	    adapter->access_ctrl.pcie_recovering)
		return;

	evt_notify_rep = leapraid_get_reply_vaddr(adapter, rep_paddr);
	if (unlikely(!evt_notify_rep))
		return;

	if (!leapraid_async_evt_validate(adapter, evt_notify_rep))
		return;

	if (leapraid_async_process_evt(adapter, evt_notify_rep))
		return;

	leapraid_async_evt_cb_enqueue(adapter, evt_notify_rep);
}

static void leapraid_handle_async_event(struct leapraid_adapter *adapter,
					u8 msix_index, u32 reply)
{
	struct leapraid_evt_notify_rep *leap_mpi_rep =
		leapraid_get_reply_vaddr(adapter, reply);

	if (!leap_mpi_rep)
		return;

	if (leap_mpi_rep->func != LEAPRAID_FUNC_EVENT_NOTIFY)
		return;

	leapraid_async_evt_cb(adapter, msix_index, reply);
}

void leapraid_async_turn_on_led(struct leapraid_adapter *adapter, u16 handle)
{
	struct leapraid_fw_evt_work *fw_event;

	fw_event = leapraid_alloc_fw_evt_work();
	if (!fw_event)
		return;

	fw_event->dev_handle = handle;
	fw_event->adapter = adapter;
	fw_event->evt_type = LEAPRAID_EVT_TURN_ON_PFA_LED;
	leapraid_fw_evt_add(adapter, fw_event);
	leapraid_fw_evt_put(fw_event);
}

static void leapraid_hardreset_barrier(struct leapraid_adapter *adapter)
{
	struct leapraid_fw_evt_work *fw_event;

	fw_event = leapraid_alloc_fw_evt_work();
	if (!fw_event)
		return;

	fw_event->adapter = adapter;
	fw_event->evt_type = LEAPRAID_EVT_REMOVE_DEAD_DEV;
	leapraid_fw_evt_add(adapter, fw_event);
	leapraid_fw_evt_put(fw_event);
}

static void leapraid_scan_dev_complete(struct leapraid_adapter *adapter)
{
	struct leapraid_fw_evt_work *fw_evt;

	fw_evt = leapraid_alloc_fw_evt_work();
	if (!fw_evt)
		return;

	fw_evt->evt_type = LEAPRAID_EVT_SCAN_DEV_DONE;
	fw_evt->adapter = adapter;
	leapraid_fw_evt_add(adapter, fw_evt);
	leapraid_fw_evt_put(fw_evt);
}

static void leapraid_handle_scan_cb(struct leapraid_adapter *adapter,
				    struct leapraid_driver_cmd *cmd,
				    struct leapraid_rep *rep)
{
	u16 status;

	cmd->status &= ~LEAPRAID_CMD_PENDING;

	status = le16_to_cpu(rep->adapter_status) &
		 LEAPRAID_ADAPTER_STATUS_MASK;
	if (status != LEAPRAID_ADAPTER_STATUS_SUCCESS)
		adapter->scan_dev_desc.scan_dev_failed = 1;

	if (!cmd->async_scan_dev) {
		complete(&cmd->done);
		return;
	}

	if (status == LEAPRAID_ADAPTER_STATUS_SUCCESS)
		leapraid_scan_dev_complete(adapter);
	else
		adapter->scan_dev_desc.scan_start_failed = status;
}

static void leapraid_handle_ctl_cb(struct leapraid_adapter *adapter,
				   struct leapraid_rep *rep,
				   u16 taskid)
{
	struct leapraid_scsiio_rep *scsiio_reply;

	if (rep->function != LEAPRAID_FUNC_SCSIIO &&
	    rep->function != LEAPRAID_FUNC_SCSIIO_RAID_PASSTHROUGH)
		return;

	scsiio_reply = (struct leapraid_scsiio_rep *)rep;
	if (!(scsiio_reply->scsi_state &
	      LEAPRAID_SCSI_STATE_AUTOSENSE_VALID))
		return;

	memcpy(&adapter->driver_cmds.ctl_cmd.sense,
	       leapraid_get_sense_buffer(adapter, taskid),
	       min_t(u32, SCSI_SENSE_BUFFERSIZE,
		     le32_to_cpu(scsiio_reply->sense_count)));
}

static bool leapraid_driver_cmds_done(struct leapraid_adapter *adapter,
				      u16 taskid, u8 msix_index,
				      u32 rep_paddr, u8 cb_idx)
{
	struct leapraid_rep *leap_mpi_rep =
		leapraid_get_reply_vaddr(adapter, rep_paddr);
	struct leapraid_driver_cmd *sp_cmd, *_sp_cmd = NULL;
	u8 reply_len;

	list_for_each_entry(sp_cmd, &adapter->driver_cmds.special_cmd_list,
			    list)
		if (cb_idx == sp_cmd->cb_idx) {
			_sp_cmd = sp_cmd;
			break;
		}

	if (WARN_ON(!_sp_cmd))
		return true;
	if (WARN_ON(_sp_cmd->status == LEAPRAID_CMD_NOT_USED))
		return true;
	if (WARN_ON(taskid != _sp_cmd->hp_taskid &&
		    taskid != _sp_cmd->taskid &&
		    taskid != _sp_cmd->inter_taskid))
		return true;
	_sp_cmd->status |= LEAPRAID_CMD_DONE;
	if (leap_mpi_rep) {
		reply_len = leap_mpi_rep->msg_len * sizeof(u32);
		if (reply_len > LEAPRAID_REPLY_SIZE)
			reply_len = LEAPRAID_REPLY_SIZE;
		memcpy(&_sp_cmd->reply, leap_mpi_rep, reply_len);
		_sp_cmd->status |= LEAPRAID_CMD_REPLY_VALID;

		if (_sp_cmd->cb_idx == LEAPRAID_SCAN_DEV_CB_IDX) {
			leapraid_handle_scan_cb(adapter, _sp_cmd,
						leap_mpi_rep);
			return true;
		}

		if (_sp_cmd->cb_idx == LEAPRAID_CTL_CB_IDX)
			leapraid_handle_ctl_cb(adapter, leap_mpi_rep, taskid);
	}

	_sp_cmd->status &= ~LEAPRAID_CMD_PENDING;
	complete(&_sp_cmd->done);

	return true;
}

static void leapraid_complete_task(struct leapraid_adapter *adapter,
				   u16 taskid, u8 msix_idx, u32 rep)
{
	bool scsiio_task = taskid <= adapter->shost->can_queue;

	if (scsiio_task) {
		if (leapraid_scsiio_done(adapter, taskid, msix_idx, rep))
			leapraid_free_taskid(adapter, taskid);
		return;
	}

	if (leapraid_driver_cmds_done(adapter, taskid, msix_idx, rep,
				      leapraid_get_cb_idx(adapter, taskid)))
		leapraid_free_taskid(adapter, taskid);
}

static void leapraid_request_descript_handler(
		struct leapraid_adapter *adapter,
		union leapraid_rep_desc_union *rpf,
		u8 req_desc_type, u8 msix_idx)
{
	u32 rep;
	u16 taskid;

	rep = 0;
	taskid = le16_to_cpu(rpf->dflt_rep.taskid);
	switch (req_desc_type) {
	case LEAPRAID_RPY_DESC_FLG_FP_SCSI_IO_SUCCESS:
	case LEAPRAID_RPY_DESC_FLG_SCSI_IO_SUCCESS:
		leapraid_complete_task(adapter, taskid, msix_idx, 0);
		break;
	case LEAPRAID_RPY_DESC_FLG_ADDRESS_REPLY:
		rep = le32_to_cpu(rpf->addr_rep.rep_frame_addr);
		if (rep > (u32)adapter->mem_desc.rep_msg_dma +
			   adapter->adapter_attr.rep_msg_qd *
			   LEAPRAID_REPLY_SIZE ||
		    rep < (u32)adapter->mem_desc.rep_msg_dma)
			rep = 0;

		if (taskid)
			leapraid_complete_task(adapter, taskid, msix_idx, rep);
		else
			leapraid_handle_async_event(adapter, msix_idx, rep);

		if (!rep)
			return;

		adapter->rep_msg_host_idx =
			(adapter->rep_msg_host_idx ==
				(adapter->adapter_attr.rep_msg_qd - 1)) ?
					0 : adapter->rep_msg_host_idx + 1;
		adapter->mem_desc.rep_msg_addr[adapter->rep_msg_host_idx] =
			cpu_to_le32(rep);
		wmb(); /* Make sure that all write ops are in order */
		writel(adapter->rep_msg_host_idx,
		       &adapter->iomem_base->rep_msg_host_idx);

		break;
	default:
		break;
	}
}

int leapraid_rep_queue_handler(struct leapraid_rq *rq)
{
	struct leapraid_adapter *adapter = rq->adapter;
	union leapraid_rep_desc_union *rep_desc;
	u8 req_desc_type;
	u64 finish_cmds;
	u8 msix_idx;

	msix_idx = rq->msix_idx;
	finish_cmds = 0;
	if (!atomic_add_unless(&rq->busy, LEAPRAID_BUSY_LIMIT,
			       LEAPRAID_BUSY_LIMIT))
		return finish_cmds;

	rep_desc = &rq->rep_desc[rq->rep_post_host_idx];
	req_desc_type = rep_desc->dflt_rep.rep_flg &
				LEAPRAID_RPY_DESC_FLG_TYPE_MASK;
	if (req_desc_type == LEAPRAID_RPY_DESC_FLG_UNUSED) {
		atomic_dec(&rq->busy);
		return finish_cmds;
	}

	for (;;) {
		if (rep_desc->u.low == UINT_MAX ||
		    rep_desc->u.high == UINT_MAX)
			break;

		leapraid_request_descript_handler(adapter, rep_desc,
						  req_desc_type, msix_idx);
		dev_dbg(&adapter->pdev->dev,
			"LEAPRAID_SCSIIO: Handled Desc taskid %d, msix %d\n",
			rep_desc->dflt_rep.taskid, msix_idx);
		rep_desc->words = cpu_to_le64(ULLONG_MAX);
		rq->rep_post_host_idx =
			(rq->rep_post_host_idx ==
				(adapter->adapter_attr.rep_desc_qd -
					LEAPRAID_BUSY_LIMIT)) ?
					0 : rq->rep_post_host_idx + 1;
		req_desc_type =
			rq->rep_desc[rq->rep_post_host_idx].dflt_rep.rep_flg &
					LEAPRAID_RPY_DESC_FLG_TYPE_MASK;
		finish_cmds++;
		if (req_desc_type == LEAPRAID_RPY_DESC_FLG_UNUSED)
			break;
		rep_desc = rq->rep_desc + rq->rep_post_host_idx;
	}

	if (!finish_cmds) {
		atomic_dec(&rq->busy);
		return finish_cmds;
	}

	wmb(); /* Make sure that all write ops are in order */
	writel(rq->rep_post_host_idx | ((msix_idx & LEAPRAID_MSIX_GROUP_MASK) <<
					LEAPRAID_RPHI_MSIX_IDX_SHIFT),
	       &adapter->iomem_base->rep_post_reg_idx[
		       msix_idx / LEAPRAID_MSIX_GROUP_SIZE].idx);
	atomic_dec(&rq->busy);
	return finish_cmds;
}

static irqreturn_t leapraid_irq_handler(int irq, void *bus_id)
{
	struct leapraid_rq *rq = bus_id;
	struct leapraid_adapter *adapter = rq->adapter;

	dev_dbg(&adapter->pdev->dev,
		"LEAPRAID_SCSIIO: Receive a interrupt, irq %d msix %d\n",
		irq, rq->msix_idx);

	if (adapter->mask_int)
		return IRQ_NONE;

	return (leapraid_rep_queue_handler(rq) > 0 ?
		 IRQ_HANDLED : IRQ_NONE);
}

void leapraid_sync_irqs(struct leapraid_adapter *adapter, bool poll)
{
	struct leapraid_int_rq *int_rq;
	struct leapraid_blk_mq_poll_rq *blk_mq_poll_rq;
	unsigned int i;

	if (!adapter->notification_desc.msix_enable)
		return;

	if (adapter->access_ctrl.shost_recovering ||
	    adapter->access_ctrl.host_removing ||
	    adapter->access_ctrl.pcie_recovering)
		return;

	for (i = 0; i < adapter->notification_desc.iopoll_qdex; i++) {
		int_rq = &adapter->notification_desc.int_rqs[i];
		if (adapter->access_ctrl.shost_recovering ||
		    adapter->access_ctrl.host_removing ||
		    adapter->access_ctrl.pcie_recovering)
			return;

		if (int_rq->rq.msix_idx == 0)
			continue;

		synchronize_irq(pci_irq_vector(adapter->pdev,
					       int_rq->rq.msix_idx));
		if (poll)
			leapraid_rep_queue_handler(&int_rq->rq);
	}

	for (i = 0; i < adapter->notification_desc.iopoll_qcnt; i++) {
		blk_mq_poll_rq =
			&adapter->notification_desc.blk_mq_poll_rqs[i];
		if (adapter->access_ctrl.shost_recovering ||
		    adapter->access_ctrl.host_removing ||
		    adapter->access_ctrl.pcie_recovering)
			return;

		if (blk_mq_poll_rq->rq.msix_idx == 0)
			continue;

		leapraid_rep_queue_handler(&blk_mq_poll_rq->rq);
	}
}

static void leapraid_sync_irqs_for_cleanup(struct leapraid_adapter *adapter)
{
	struct leapraid_notification_desc *desc = &adapter->notification_desc;
	struct leapraid_int_rq *int_rq;
	unsigned int i;

	if (!adapter->mask_int && leapraid_pci_active(adapter))
		leapraid_mask_int(adapter);

	for (i = 0; i < desc->int_rqs_allocated; i++) {
		int_rq = &desc->int_rqs[i];
		synchronize_irq(pci_irq_vector(adapter->pdev,
					       int_rq->rq.msix_idx));
	}
}

void leapraid_mq_polling_pause(struct leapraid_adapter *adapter)
{
	struct leapraid_notification_desc *desc;
	int iopoll_q_count;
	int qid;

	desc = &adapter->notification_desc;
	iopoll_q_count = adapter->adapter_attr.rq_cnt - desc->iopoll_qdex;

	for (qid = 0; qid < iopoll_q_count; qid++)
		atomic_set(&desc->blk_mq_poll_rqs[qid].pause, 1);

	for (qid = 0; qid < iopoll_q_count; qid++) {
		while (atomic_read(&desc->blk_mq_poll_rqs[qid].busy)) {
			cpu_relax();
			udelay(LEAPRAID_IO_POLL_DELAY_US);
		}
	}
}

void leapraid_mq_polling_resume(struct leapraid_adapter *adapter)
{
	struct leapraid_notification_desc *desc;
	int iopoll_q_count;
	int qid;

	desc = &adapter->notification_desc;
	iopoll_q_count = adapter->adapter_attr.rq_cnt - desc->iopoll_qdex;

	for (qid = 0; qid < iopoll_q_count; qid++)
		atomic_set(&desc->blk_mq_poll_rqs[qid].pause, 0);
}

static int leapraid_unlock_host_diag(struct leapraid_adapter *adapter,
				     u32 *host_diag)
{
	const u32 unlock_seq[] = {
		LEAPRAID_WRSEQ_FLUSH_KEY_VALUE,
		LEAPRAID_WRSEQ_1ST_KEY_VALUE,
		LEAPRAID_WRSEQ_2ND_KEY_VALUE,
		LEAPRAID_WRSEQ_3RD_KEY_VALUE,
		LEAPRAID_WRSEQ_4TH_KEY_VALUE,
		LEAPRAID_WRSEQ_5TH_KEY_VALUE,
		LEAPRAID_WRSEQ_6TH_KEY_VALUE
	};

	const int max_retries = LEAPRAID_UNLOCK_RETRY_LIMIT;
	int retry = 0;
	unsigned int i;

	*host_diag = 0;
	while (retry++ <= max_retries) {
		for (i = 0; i < ARRAY_SIZE(unlock_seq); i++)
			writel(unlock_seq[i], &adapter->iomem_base->ws);

		msleep(LEAPRAID_UNLOCK_SLEEP_MS);

		*host_diag = leapraid_readl(&adapter->iomem_base->host_diag);
		if (*host_diag & LEAPRAID_DIAG_WRITE_ENABLE)
			return 0;
	}

	dev_err(&adapter->pdev->dev, "Try host reset timeout!\n");
	return -EFAULT;
}

static int leapraid_host_diag_reset(struct leapraid_adapter *adapter)
{
	u32 host_diag;

	pci_cfg_access_lock(adapter->pdev);

	mutex_lock(&adapter->reset_desc.host_diag_mutex);
	if (leapraid_unlock_host_diag(adapter, &host_diag))
		goto out_cleanup;

	writel(host_diag | LEAPRAID_DIAG_RESET,
	       &adapter->iomem_base->host_diag);

	msleep(LEAPRAID_MSLEEP_EXTRA_LONG_MS);
	msleep(LEAPRAID_MSLEEP_NORMAL_MS);

	host_diag = leapraid_readl(&adapter->iomem_base->host_diag);
	if (host_diag == LEAPRAID_INVALID_HOST_DIAG_VAL ||
	    host_diag & LEAPRAID_DIAG_RESET)
		goto out_cleanup;

	writel(0x0, &adapter->iomem_base->ws);
	mutex_unlock(&adapter->reset_desc.host_diag_mutex);
	if (!leapraid_wait_adapter_ready(adapter))
		goto out_failed;

	pci_cfg_access_unlock(adapter->pdev);
	return 0;

out_cleanup:
	mutex_unlock(&adapter->reset_desc.host_diag_mutex);
out_failed:
	pci_cfg_access_unlock(adapter->pdev);
	dev_err(&adapter->pdev->dev, "Host diag failed\n");
	return -EFAULT;
}

static int leapraid_find_matching_port(
		struct leapraid_card_port *card_port_table,
		u8 count, u8 port_id, u64 sas_addr)
{
	int i;

	for (i = 0; i < count; i++)
		if (card_port_table[i].port_id == port_id &&
		    card_port_table[i].sas_address == sas_addr)
			return i;

	return LEAPRAID_INVALID_INDEX;
}

static u8 leapraid_fill_card_port_table(
		struct leapraid_adapter *adapter,
		struct leapraid_sas_io_unit_p0 *sas_iounit_p0,
		struct leapraid_card_port *new_card_port_table)
{
	u8 port_entry_num = 0, port_id;
	u16 attached_hdl;
	u64 attached_sas_addr;
	int i, idx;

	for (i = 0; i < adapter->dev_topo.card.phys_num; i++) {
		if (sas_iounit_p0->phy_info[i].neg_link_rate >>
		    LEAPRAID_SAS_NEG_LINK_RATE_SHIFT <
		    LEAPRAID_SAS_NEG_LINK_RATE_1_5)
			continue;

		attached_hdl = le16_to_cpu(sas_iounit_p0->phy_info[i]
					   .attached_dev_hdl);
		if (leapraid_get_sas_address(adapter,
					     attached_hdl,
					     &attached_sas_addr) != 0)
			continue;

		port_id = sas_iounit_p0->phy_info[i].port;

		idx = leapraid_find_matching_port(new_card_port_table,
						  port_entry_num,
						  port_id,
						  attached_sas_addr);
		if (idx >= 0) {
			new_card_port_table[idx].phy_mask |= BIT(i);
		} else {
			new_card_port_table[port_entry_num].port_id = port_id;
			new_card_port_table[port_entry_num].phy_mask = BIT(i);
			new_card_port_table[port_entry_num].sas_address =
				attached_sas_addr;
			port_entry_num++;
		}
	}

	return port_entry_num;
}

static u8 leapraid_set_new_card_port_table_after_reset(
		struct leapraid_adapter *adapter,
		struct leapraid_card_port *new_card_port_table)
{
	union cfg_param_1 cfgp1 = {0};
	union cfg_param_2 cfgp2 = {0};
	struct leapraid_sas_io_unit_p0 *sas_iounit_p0;
	u8 port_entry_num = 0;
	u16 sz;

	sz = offsetof(struct leapraid_sas_io_unit_p0, phy_info) +
		(adapter->dev_topo.card.phys_num *
		 sizeof(struct leapraid_sas_io_unit0_phy_info));
	sas_iounit_p0 = kzalloc(sz, GFP_KERNEL);
	if (!sas_iounit_p0)
		return port_entry_num;

	cfgp1.size = sz;
	if (leapraid_op_config_page(adapter, sas_iounit_p0, cfgp1, cfgp2,
				    GET_SAS_IOUNIT_PG0) != 0)
		goto out_free;

	port_entry_num = leapraid_fill_card_port_table(adapter,
						       sas_iounit_p0,
						       new_card_port_table);
out_free:
	kfree(sas_iounit_p0);
	return port_entry_num;
}

static void leapraid_update_existing_port(struct leapraid_adapter *adapter,
					  struct leapraid_card_port *new_table,
					  int entry_idx, int port_entry_num)
{
	struct leapraid_card_port *matched_card_port;
	int matched_code;
	int count, lcount = 0;
	u64 sas_addr;
	int i;

	matched_code = leapraid_check_card_port(adapter,
						&new_table[entry_idx],
						&matched_card_port,
						&count);

	if (!matched_card_port)
		return;

	if (matched_code == SAME_PORT_WITH_PARTIALLY_CHANGED_PHYS ||
	    matched_code == SAME_ADDR_WITH_PARTIALLY_CHANGED_PHYS) {
		leapraid_add_or_del_phys_from_existing_port(adapter,
							    matched_card_port,
							    new_table,
							    entry_idx,
							    port_entry_num);
	} else if (matched_code == SAME_ADDR_ONLY) {
		sas_addr = new_table[entry_idx].sas_address;
		for (i = 0; i < port_entry_num; i++)
			if (new_table[i].sas_address == sas_addr)
				lcount++;

		if (count > 1 || lcount > 1)
			return;

		leapraid_add_or_del_phys_from_existing_port(adapter,
							    matched_card_port,
							    new_table,
							    entry_idx,
							    port_entry_num);
	}

	if (matched_card_port->port_id != new_table[entry_idx].port_id)
		matched_card_port->port_id = new_table[entry_idx].port_id;

	matched_card_port->flg &= ~LEAPRAID_CARD_PORT_FLG_DIRTY;
	matched_card_port->phy_mask = new_table[entry_idx].phy_mask;
}

static void leapraid_update_card_port_after_reset(
		struct leapraid_adapter *adapter)
{
	struct leapraid_card_port *new_card_port_table;
	struct leapraid_card_port *matched_card_port;
	u8 port_entry_num;
	u8 nr_phys;
	int i;

	if (leapraid_get_adapter_phys(adapter, &nr_phys) || !nr_phys)
		return;

	if (!adapter->dev_topo.card.card_phy) {
		adapter->dev_topo.card.card_phy =
			kzalloc_objs(struct leapraid_card_phy, nr_phys);
		if (!adapter->dev_topo.card.card_phy)
			return;
	}

	adapter->dev_topo.card.phys_num = nr_phys;

	new_card_port_table = kzalloc_objs(struct leapraid_card_port,
					   adapter->dev_topo.card.phys_num);
	if (!new_card_port_table)
		return;

	port_entry_num =
		leapraid_set_new_card_port_table_after_reset(
			adapter,
			new_card_port_table);
	if (!port_entry_num)
		goto out_free_port_table;

	list_for_each_entry(matched_card_port,
			    &adapter->dev_topo.card_port_list, list)
		matched_card_port->flg |= LEAPRAID_CARD_PORT_FLG_DIRTY;

	matched_card_port = NULL;
	for (i = 0; i < port_entry_num; i++)
		leapraid_update_existing_port(adapter,
					      new_card_port_table,
					      i, port_entry_num);
out_free_port_table:
	kfree(new_card_port_table);
}

static bool leapraid_is_valid_vphy(
		struct leapraid_adapter *adapter,
		struct leapraid_sas_io_unit_p0 *sas_io_unit_p0,
		int phy_index)
{
	union cfg_param_1 cfgp1 = {0};
	union cfg_param_2 cfgp2 = {0};
	struct leapraid_sas_phy_p0 phy_p0;
	u32 dev_info;

	if (sas_io_unit_p0->phy_info[phy_index].neg_link_rate >>
	    LEAPRAID_SAS_NEG_LINK_RATE_SHIFT <
	    LEAPRAID_SAS_NEG_LINK_RATE_1_5)
		return false;

	dev_info = le32_to_cpu(sas_io_unit_p0->phy_info[phy_index]
			       .controller_phy_dev_info);
	if (!(dev_info & LEAPRAID_DEVTYP_SEP))
		return false;

	cfgp1.phy_number = phy_index;
	if (leapraid_op_config_page(adapter, &phy_p0, cfgp1, cfgp2,
				    GET_PHY_PG0))
		return false;

	if (!(le32_to_cpu(phy_p0.phy_info) & LEAPRAID_SAS_PHYINFO_VPHY))
		return false;

	return true;
}

static void leapraid_update_vphy_binding(struct leapraid_adapter *adapter,
					 struct leapraid_card_port *card_port,
					 struct leapraid_vphy *vphy,
					 int phy_index, u8 may_new_port_id,
					 u64 attached_sas_addr)
{
	struct leapraid_card_port *may_new_card_port;
	struct leapraid_sas_dev *sas_dev;

	may_new_card_port = leapraid_get_port_by_id(adapter,
						    may_new_port_id,
						    true);
	if (!may_new_card_port) {
		may_new_card_port = kzalloc_obj(*may_new_card_port);
		if (!may_new_card_port)
			return;
		may_new_card_port->port_id = may_new_port_id;
		dev_err(&adapter->pdev->dev,
			"%s: New card port %p added, port=%d\n",
			__func__, may_new_card_port, may_new_port_id);
		list_add_tail(&may_new_card_port->list,
			      &adapter->dev_topo.card_port_list);
	}

	if (card_port != may_new_card_port) {
		if (!may_new_card_port->vphys_mask)
			INIT_LIST_HEAD(&may_new_card_port->vphys_list);
		may_new_card_port->vphys_mask |= BIT(phy_index);
		card_port->vphys_mask &= ~BIT(phy_index);
		list_move(&vphy->list, &may_new_card_port->vphys_list);

		sas_dev = leapraid_get_sas_dev_by_addr(adapter,
						       attached_sas_addr,
						       card_port);
		if (sas_dev) {
			sas_dev->card_port = may_new_card_port;
			leapraid_sdev_put(sas_dev);
		}
	}

	if (may_new_card_port->flg & LEAPRAID_CARD_PORT_FLG_DIRTY) {
		may_new_card_port->sas_address = 0;
		may_new_card_port->phy_mask = 0;
		may_new_card_port->flg &= ~LEAPRAID_CARD_PORT_FLG_DIRTY;
	}
	vphy->flg &= ~LEAPRAID_VPHY_FLG_DIRTY;
}

static void leapraid_update_vphys_after_reset(struct leapraid_adapter *adapter)
{
	union cfg_param_1 cfgp1 = {0};
	union cfg_param_2 cfgp2 = {0};
	struct leapraid_sas_io_unit_p0 *sas_iounit_p0;
	struct leapraid_card_port *card_port, *card_port_next;
	struct leapraid_vphy *vphy, *vphy_next;
	u64 attached_sas_addr;
	u16 sz;
	u16 attached_hdl;
	bool found = false;
	u8 port_id;
	int i;

	list_for_each_entry_safe(card_port, card_port_next,
				 &adapter->dev_topo.card_port_list, list) {
		if (!card_port->vphys_mask)
			continue;

		list_for_each_entry_safe(vphy, vphy_next,
					 &card_port->vphys_list, list)
			vphy->flg |= LEAPRAID_VPHY_FLG_DIRTY;
	}

	sz = offsetof(struct leapraid_sas_io_unit_p0, phy_info) +
		(adapter->dev_topo.card.phys_num *
		 sizeof(struct leapraid_sas_io_unit0_phy_info));
	sas_iounit_p0 = kzalloc(sz, GFP_KERNEL);
	if (!sas_iounit_p0)
		return;

	cfgp1.size = sz;
	if (leapraid_op_config_page(adapter, sas_iounit_p0, cfgp1, cfgp2,
				    GET_SAS_IOUNIT_PG0) != 0)
		goto out_free;

	for (i = 0; i < adapter->dev_topo.card.phys_num; i++) {
		if (!leapraid_is_valid_vphy(adapter, sas_iounit_p0, i))
			continue;

		attached_hdl =
			le16_to_cpu(sas_iounit_p0->phy_info[i]
					.attached_dev_hdl);
		if (leapraid_get_sas_address(adapter, attached_hdl,
					     &attached_sas_addr) != 0)
			continue;

		found = false;
		card_port = NULL;
		card_port_next = NULL;
		list_for_each_entry_safe(card_port, card_port_next,
					 &adapter->dev_topo.card_port_list,
					 list) {
			if (!card_port->vphys_mask)
				continue;

			list_for_each_entry_safe(vphy, vphy_next,
						 &card_port->vphys_list,
						 list) {
				if (!(vphy->flg & LEAPRAID_VPHY_FLG_DIRTY))
					continue;

				if (vphy->sas_address != attached_sas_addr)
					continue;

				if (!(vphy->phy_mask & BIT(i)))
					vphy->phy_mask = BIT(i);

				port_id = sas_iounit_p0->phy_info[i].port;

				leapraid_update_vphy_binding(adapter,
							     card_port,
							     vphy,
							     i,
							     port_id,
							     attached_sas_addr);

				found = true;
				break;
			}
			if (found)
				break;
		}
	}
out_free:
	kfree(sas_iounit_p0);
}

static void leapraid_mark_all_dev_deleted(struct leapraid_adapter *adapter)
{
	struct leapraid_sdev_priv *sdev_priv;
	struct scsi_device *sdev;

	shost_for_each_device(sdev, adapter->shost) {
		sdev_priv = sdev->hostdata;
		if (sdev_priv && sdev_priv->starget_priv)
			sdev_priv->starget_priv->deleted = 1;
	}
}

static void leapraid_free_enc_list(struct leapraid_adapter *adapter)
{
	struct leapraid_enc_node *enc_dev, *enc_dev_next;
	LIST_HEAD(free_list);
	unsigned long flags;

	spin_lock_irqsave(&adapter->dev_topo.enc_lock, flags);
	list_splice_init(&adapter->dev_topo.enc_list, &free_list);
	spin_unlock_irqrestore(&adapter->dev_topo.enc_lock, flags);

	list_for_each_entry_safe(enc_dev, enc_dev_next, &free_list, list) {
		list_del(&enc_dev->list);
		kfree(enc_dev);
	}
}

static void leapraid_rebuild_enc_list_after_reset(
		struct leapraid_adapter *adapter)
{
	union cfg_param_1 cfgp1 = {0};
	union cfg_param_2 cfgp2 = {0};
	struct leapraid_enc_node *enc_node;
	u16 enc_hdl;
	unsigned long flags;
	int rc;

	leapraid_free_enc_list(adapter);

	cfgp1.form = LEAPRAID_SAS_CFG_PGAD_GET_NEXT_LOOP;
	for (enc_hdl = 0xFFFF; ; enc_hdl = le16_to_cpu(enc_node->pg0.enc_hdl)) {
		enc_node = kzalloc_obj(*enc_node);
		if (!enc_node)
			return;

		cfgp2.handle = enc_hdl;
		rc = leapraid_op_config_page(adapter, &enc_node->pg0, cfgp1,
					     cfgp2, GET_SAS_ENCLOSURE_PG0);
		if (rc) {
			kfree(enc_node);
			return;
		}

		spin_lock_irqsave(&adapter->dev_topo.enc_lock, flags);
		list_add_tail(&enc_node->list, &adapter->dev_topo.enc_list);
		spin_unlock_irqrestore(&adapter->dev_topo.enc_lock, flags);
	}
}

static void leapraid_mark_resp_sas_dev(struct leapraid_adapter *adapter,
				       struct leapraid_sas_dev_p0 *sas_dev_p0)
{
	struct leapraid_starget_priv *starget_priv;
	struct leapraid_enc_node *enc_node = NULL;
	struct leapraid_card_port *card_port;
	struct leapraid_sas_dev *sas_dev;
	struct scsi_target *starget;
	unsigned long flags;
	unsigned long enc_flags = 0;
	u16 enc_hdl;
	u64 enc_lid = 0;

	card_port = leapraid_get_port_by_id(adapter, sas_dev_p0->physical_port,
					    false);
	enc_hdl = le16_to_cpu(sas_dev_p0->enc_hdl);
	if (enc_hdl) {
		spin_lock_irqsave(&adapter->dev_topo.enc_lock, enc_flags);
		enc_node = leapraid_enc_find_by_hdl(adapter, enc_hdl);
		if (enc_node)
			enc_lid = le64_to_cpu(enc_node->pg0.enc_lid);
		spin_unlock_irqrestore(&adapter->dev_topo.enc_lock, enc_flags);
		if (!enc_node)
			dev_info(&adapter->pdev->dev,
				 "enc hdl 0x%04x has no matched enc dev\n",
				 enc_hdl);
	}

	spin_lock_irqsave(&adapter->dev_topo.sas_dev_lock, flags);
	list_for_each_entry(sas_dev, &adapter->dev_topo.sas_dev_list, list) {
		if (sas_dev->sas_addr != le64_to_cpu(sas_dev_p0->sas_address) ||
		    sas_dev->slot != le16_to_cpu(sas_dev_p0->slot) ||
		    sas_dev->card_port != card_port)
			continue;

		sas_dev->resp = 1;
		starget = sas_dev->starget;
		if (starget && starget->hostdata) {
			starget_priv = starget->hostdata;
			starget_priv->tm_busy = 0;
			starget_priv->deleted = 0;
		} else {
			starget_priv = NULL;
		}

		if (starget) {
			starget_printk(
				KERN_INFO, starget,
				"dev: hdl=0x%04x saddr=0x%016llx port_id=%d\n",
				sas_dev->hdl,
				(unsigned long long)sas_dev->sas_addr,
				sas_dev->card_port->port_id);
			if (sas_dev->enc_hdl != 0)
				starget_printk(
					KERN_INFO,
					starget,
					"enc info: enc_lid=0x%016llx slot=%d\n",
					(unsigned long long)sas_dev->enc_lid,
					sas_dev->slot);
		}

		if (le16_to_cpu(sas_dev_p0->flg) &
		    LEAPRAID_SAS_DEV_P0_FLG_ENC_LEVEL_VALID) {
			sas_dev->enc_level = sas_dev_p0->enc_level;
			memcpy(sas_dev->connector_name,
			       sas_dev_p0->connector_name,
			       LEAPRAID_SAS_DEV_P0_CON_NAME_LEN);
			sas_dev->connector_name[
				LEAPRAID_SAS_DEV_P0_CON_NAME_LEN] = '\0';
		} else {
			sas_dev->enc_level = 0;
			sas_dev->connector_name[0] = '\0';
		}

		sas_dev->enc_hdl = enc_hdl;
		sas_dev->enc_lid = enc_lid;

		if (sas_dev->hdl == le16_to_cpu(sas_dev_p0->dev_hdl))
			goto unlock;

		dev_info(&adapter->pdev->dev,
			 "hdl changed: 0x%04x -> 0x%04x\n",
			 sas_dev->hdl, sas_dev_p0->dev_hdl);
		sas_dev->hdl = le16_to_cpu(sas_dev_p0->dev_hdl);
		if (starget_priv)
			starget_priv->hdl = le16_to_cpu(sas_dev_p0->dev_hdl);

		goto unlock;
	}
unlock:
	spin_unlock_irqrestore(&adapter->dev_topo.sas_dev_lock, flags);
}

static void leapraid_search_resp_sas_dev(struct leapraid_adapter *adapter)
{
	union cfg_param_1 cfgp1 = {0};
	union cfg_param_2 cfgp2 = {0};
	struct leapraid_sas_dev_p0 sas_dev_p0;
	u32 device_info;

	if (list_empty(&adapter->dev_topo.sas_dev_list))
		return;

	cfgp1.form = LEAPRAID_SAS_CFG_PGAD_GET_NEXT_LOOP;
	for (cfgp2.handle = 0xFFFF;
	     !leapraid_op_config_page(adapter, &sas_dev_p0,
				      cfgp1, cfgp2, GET_SAS_DEVICE_PG0);
	     cfgp2.handle = le16_to_cpu(sas_dev_p0.dev_hdl)) {
		device_info = le32_to_cpu(sas_dev_p0.dev_info);
		if (!(leapraid_is_end_dev(device_info)))
			continue;

		leapraid_mark_resp_sas_dev(adapter, &sas_dev_p0);
	}
}

static void leapraid_mark_resp_raid_volume(struct leapraid_adapter *adapter,
					   u64 wwid, u16 hdl)
{
	struct leapraid_raid_volume *raid_volume;
	struct leapraid_starget_priv *starget_priv = NULL;
	struct scsi_target *starget;
	unsigned long flags;
	u64 volume_wwid;
	u16 old_hdl;

	raid_volume = leapraid_raid_volume_find_by_wwid(adapter, wwid);
	if (!raid_volume)
		return;

	spin_lock_irqsave(&adapter->dev_topo.raid_volume_lock, flags);
	if (!raid_volume->starget) {
		spin_unlock_irqrestore(&adapter->dev_topo.raid_volume_lock,
				       flags);
		leapraid_raid_volume_put(raid_volume);
		return;
	}

	starget = raid_volume->starget;
	if (starget->hostdata) {
		starget_priv = starget->hostdata;
		starget_priv->deleted = 0;
	}

	raid_volume->resp = 1;
	volume_wwid = raid_volume->wwid;
	old_hdl = raid_volume->hdl;
	if (old_hdl != hdl) {
		raid_volume->hdl = hdl;
		if (starget_priv)
			starget_priv->hdl = hdl;
	}
	spin_unlock_irqrestore(&adapter->dev_topo.raid_volume_lock, flags);

	starget_printk(KERN_INFO, starget,
		       "raid volume: hdl=0x%04x, wwid=0x%016llx\n",
		       hdl, (unsigned long long)volume_wwid);
	if (old_hdl != hdl)
		dev_info(&adapter->pdev->dev,
			 "hdl changed: 0x%04x -> 0x%04x\n",
			 old_hdl, hdl);

	leapraid_raid_volume_put(raid_volume);
}

static void leapraid_search_resp_raid_volume(struct leapraid_adapter *adapter)
{
	union cfg_param_1 cfgp1 = {0};
	union cfg_param_1 cfgp1_extra = {0};
	union cfg_param_2 cfgp2 = {0};
	union cfg_param_2 cfgp2_extra = {0};
	struct leapraid_raidvol_p1 raidvol_p1;
	struct leapraid_raidvol_p0 raidvol_p0;
	struct leapraid_raidpd_p0 raidpd_p0;
	unsigned long flags;
	u16 hdl;
	u8 phys_disk_num;
	bool is_empty;

	if (!adapter->adapter_attr.raid_support)
		return;

	spin_lock_irqsave(&adapter->dev_topo.raid_volume_lock, flags);
	is_empty = list_empty(&adapter->dev_topo.raid_volume_list);
	spin_unlock_irqrestore(&adapter->dev_topo.raid_volume_lock, flags);
	if (is_empty)
		return;

	cfgp1.form = LEAPRAID_SAS_CFG_PGAD_GET_NEXT_LOOP;
	for (hdl = 0xFFFF, cfgp2.handle = hdl;
	     !leapraid_op_config_page(adapter, &raidvol_p1, cfgp1, cfgp2,
				      GET_RAID_VOLUME_PG1);
	     cfgp2.handle = hdl) {
		hdl = le16_to_cpu(raidvol_p1.dev_hdl);
		cfgp1_extra.size = sizeof(struct leapraid_raidvol_p0);
		cfgp2_extra.handle = hdl;
		if (leapraid_op_config_page(adapter, &raidvol_p0, cfgp1_extra,
					    cfgp2_extra, GET_RAID_VOLUME_PG0))
			continue;

		if (raidvol_p0.volume_state == LEAPRAID_VOL_STATE_OPTIMAL ||
		    raidvol_p0.volume_state == LEAPRAID_VOL_STATE_ONLINE ||
		    raidvol_p0.volume_state == LEAPRAID_VOL_STATE_DEGRADED)
			leapraid_mark_resp_raid_volume(
				adapter,
				le64_to_cpu(raidvol_p1.wwid),
				hdl);
	}

	memset(adapter->dev_topo.pd_hdls, 0, adapter->dev_topo.pd_hdls_sz);
	cfgp1.form = LEAPRAID_SAS_CFG_PGAD_GET_NEXT_LOOP;
	for (phys_disk_num = 0xFF, cfgp2.form_specific = phys_disk_num;
	     !leapraid_op_config_page(adapter, &raidpd_p0, cfgp1, cfgp2,
				      GET_PHY_DISK_PG0);
	     cfgp2.form_specific = phys_disk_num) {
		phys_disk_num = raidpd_p0.phys_disk_num;
		hdl = le16_to_cpu(raidpd_p0.dev_hdl);
		if (!hdl ||
		    hdl > adapter->adapter_attr.features.max_dev_handle)
			dev_warn(&adapter->pdev->dev,
				 "%s: Invalid device handle\n", __func__);
		else
			set_bit(hdl, adapter->dev_topo.pd_hdls);
	}
}

static void leapraid_mark_resp_exp(struct leapraid_adapter *adapter,
				   struct leapraid_exp_p0 *exp_pg0)
{
	struct leapraid_enc_node *enc_node = NULL;
	struct leapraid_topo_node *topo_node_exp;
	u16 enc_hdl = le16_to_cpu(exp_pg0->enc_hdl);
	u64 sas_address = le64_to_cpu(exp_pg0->sas_address);
	u16 hdl = le16_to_cpu(exp_pg0->dev_hdl);
	u8 port_id = exp_pg0->physical_port;
	struct leapraid_card_port *card_port = leapraid_get_port_by_id(adapter,
								       port_id,
								       false);
	unsigned long flags;
	unsigned long enc_flags = 0;
	u64 enc_lid = 0;
	int i;

	if (enc_hdl) {
		spin_lock_irqsave(&adapter->dev_topo.enc_lock, enc_flags);
		enc_node = leapraid_enc_find_by_hdl(adapter, enc_hdl);
		if (enc_node)
			enc_lid = le64_to_cpu(enc_node->pg0.enc_lid);
		spin_unlock_irqrestore(&adapter->dev_topo.enc_lock, enc_flags);
	}

	spin_lock_irqsave(&adapter->dev_topo.topo_node_lock, flags);
	list_for_each_entry(topo_node_exp, &adapter->dev_topo.exp_list, list) {
		if (topo_node_exp->sas_address != sas_address ||
		    topo_node_exp->card_port != card_port)
			continue;

		topo_node_exp->resp = 1;
		topo_node_exp->enc_hdl = enc_hdl;
		topo_node_exp->enc_lid = enc_lid;
		if (topo_node_exp->hdl == hdl)
			goto unlock;

		dev_info(&adapter->pdev->dev,
			 "hdl changed: 0x%04x -> 0x%04x\n",
			 topo_node_exp->hdl, hdl);
		topo_node_exp->hdl = hdl;
		for (i = 0; i < topo_node_exp->phys_num; i++)
			topo_node_exp->card_phy[i].hdl = hdl;
		goto unlock;
	}
unlock:
	spin_unlock_irqrestore(&adapter->dev_topo.topo_node_lock, flags);
}

static void leapraid_search_resp_exp(struct leapraid_adapter *adapter)
{
	union cfg_param_1 cfgp1 = {0};
	union cfg_param_2 cfgp2 = {0};
	struct leapraid_exp_p0 exp_p0;
	u64 sas_address;
	u16 hdl;
	u8 port;

	if (list_empty(&adapter->dev_topo.exp_list))
		return;

	cfgp1.form = LEAPRAID_SAS_CFG_PGAD_GET_NEXT_LOOP;
	for (hdl = 0xFFFF, cfgp2.handle = hdl;
	     !leapraid_op_config_page(adapter, &exp_p0, cfgp1, cfgp2,
				      GET_SAS_EXPANDER_PG0);
	     cfgp2.handle = hdl) {
		hdl = le16_to_cpu(exp_p0.dev_hdl);
		sas_address = le64_to_cpu(exp_p0.sas_address);
		port = exp_p0.physical_port;

		dev_dbg(&adapter->pdev->dev,
			"exp detected: hdl=0x%04x, sas=0x%016llx, port=%u",
			hdl, (unsigned long long)sas_address,
			adapter->adapter_attr.enable_mp ? port :
				LEAPRAID_DISABLE_MP_PORT_ID);
		leapraid_mark_resp_exp(adapter, &exp_p0);
	}
}

void leapraid_wait_cmds_done(struct leapraid_adapter *adapter)
{
	struct leapraid_io_req_tracker *io_req_tracker;
	unsigned long flags;
	u16 i;

	adapter->reset_desc.pending_io_cnt = 0;
	if (!leapraid_pci_active(adapter)) {
		dev_err(&adapter->pdev->dev,
			"%s %s: PCI error, device reset or unplugged!\n",
			adapter->adapter_attr.name, __func__);
		return;
	}

	if (leapraid_get_adapter_state(adapter) != LEAPRAID_DB_OPERATIONAL)
		return;

	spin_lock_irqsave(&adapter->dynamic_task_desc.task_lock, flags);
	for (i = 1; i <= adapter->shost->can_queue; i++) {
		io_req_tracker = leapraid_get_io_tracker_from_taskid(adapter,
								     i);
		if (io_req_tracker && io_req_tracker->taskid != 0 &&
		    io_req_tracker->scmd)
			adapter->reset_desc.pending_io_cnt++;
	}
	spin_unlock_irqrestore(&adapter->dynamic_task_desc.task_lock, flags);

	if (!adapter->reset_desc.pending_io_cnt)
		return;

	wait_event_timeout(adapter->reset_desc.reset_wait_queue,
			   adapter->reset_desc.pending_io_cnt == 0,
			   LEAPRAID_IO_CMD_TIMEOUT * HZ);
}

int leapraid_hard_reset_handler(struct leapraid_adapter *adapter,
				enum reset_type type)
{
	unsigned long flags;
	bool wake = false;
	int rc;

	mutex_lock(&adapter->reset_desc.adapter_reset_mutex);

	if (adapter->access_ctrl.shost_recover_async) {
		rc = adapter->reset_desc.adapter_reset_results;
		dev_info(&adapter->pdev->dev,
			 "Skip nested hard reset, async evt running, rc=%d\n",
			 rc);
		mutex_unlock(&adapter->reset_desc.adapter_reset_mutex);
		return rc;
	}

	if (!leapraid_pci_active(adapter)) {
		if (leapraid_pci_removed(adapter)) {
			dev_info(&adapter->pdev->dev,
				 "pci_dev removed, pause poll, clean cmds\n");
			leapraid_mq_polling_pause(adapter);
			leapraid_clean_active_scsi_cmds(adapter);
			leapraid_mq_polling_resume(adapter);
		}
		dev_err(&adapter->pdev->dev, "PCIe unavailable!\n");
		mutex_unlock(&adapter->reset_desc.adapter_reset_mutex);
		return -ENXIO;
	}

	dev_info(&adapter->pdev->dev, "Starting hard reset\n");

	spin_lock_irqsave(&adapter->reset_desc.adapter_reset_lock, flags);
	adapter->access_ctrl.shost_recovering = 1;
	adapter->access_ctrl.shost_recover_async = 1;
	spin_unlock_irqrestore(&adapter->reset_desc.adapter_reset_lock, flags);

	leapraid_wait_cmds_done(adapter);
	leapraid_mask_int(adapter);
	leapraid_mq_polling_pause(adapter);
	rc = leapraid_make_adapter_ready(adapter, type);
	if (rc) {
		dev_err(&adapter->pdev->dev,
			"Failed to make adapter ready, rc=%d\n", rc);
		goto out_cleanup;
	}

	rc = leapraid_fw_log_init(adapter);
	if (rc) {
		dev_err(&adapter->pdev->dev, "Firmware log init failed\n");
		goto out_cleanup;
	}

	leapraid_clean_active_cmds(adapter);
	if (adapter->scan_dev_desc.driver_loading &&
	    adapter->scan_dev_desc.scan_dev_failed) {
		dev_err(&adapter->pdev->dev,
			"Previous device scan failed or driver loading\n");
		adapter->access_ctrl.host_removing = 1;
		rc = -EFAULT;
		goto out_cleanup;
	}

	rc = leapraid_make_adapter_available(adapter);
	if (!rc) {
		dev_info(&adapter->pdev->dev,
			 "Adapter is now available, rebuilding topology\n");
		if (adapter->adapter_attr.enable_mp) {
			leapraid_update_card_port_after_reset(adapter);
			leapraid_update_vphys_after_reset(adapter);
		}
		leapraid_mark_all_dev_deleted(adapter);
		leapraid_rebuild_enc_list_after_reset(adapter);
		leapraid_search_resp_sas_dev(adapter);
		leapraid_search_resp_raid_volume(adapter);
		leapraid_search_resp_exp(adapter);
		leapraid_hardreset_barrier(adapter);
	}
out_cleanup:
	if (rc)
		dev_err(&adapter->pdev->dev, "Hard reset failed\n");

	spin_lock_irqsave(&adapter->reset_desc.adapter_reset_lock, flags);
	adapter->reset_desc.adapter_reset_results = rc;
	adapter->access_ctrl.shost_recovering = 0;
	wake_up(&adapter->access_ctrl.recovery_waitq);
	if (rc) {
		adapter->access_ctrl.shost_recover_async = 0;
		wake = true;
	}
	spin_unlock_irqrestore(&adapter->reset_desc.adapter_reset_lock, flags);
	if (wake)
		wake_up(&adapter->access_ctrl.shost_recover_wq);

	adapter->reset_desc.reset_cnt++;
	mutex_unlock(&adapter->reset_desc.adapter_reset_mutex);

	if (rc)
		leapraid_clean_active_scsi_cmds(adapter);
	leapraid_mq_polling_resume(adapter);

	return rc;
}

static int leapraid_get_adapter_features(struct leapraid_adapter *adapter)
{
	struct leapraid_adapter_features_req leap_mpi_req;
	struct leapraid_adapter_features_rep leap_mpi_rep;
	u8 fw_major, fw_minor, fw_build, fw_release;
	u32 req_sz;
	u32 rep_sz;
	u32 db;
	int r;

	db = leapraid_readl(&adapter->iomem_base->db);
	if (db & LEAPRAID_DB_USED ||
	    (db & LEAPRAID_DB_MASK) == LEAPRAID_DB_FAULT) {
		dev_err(&adapter->pdev->dev,
			"%s: Doorbell used or fault\n", __func__);
		return -EFAULT;
	}

	if ((db & LEAPRAID_DB_MASK) != LEAPRAID_DB_READY &&
	    (db & LEAPRAID_DB_MASK) != LEAPRAID_DB_OPERATIONAL &&
	    !leapraid_wait_adapter_ready(adapter)) {
		dev_err(&adapter->pdev->dev,
			"%s: adapter not ready\n", __func__);
		return -EFAULT;
	}

	req_sz = sizeof(struct leapraid_adapter_features_req);
	rep_sz = sizeof(struct leapraid_adapter_features_rep);
	memset(&leap_mpi_req, 0, req_sz);
	memset(&leap_mpi_rep, 0, rep_sz);
	leap_mpi_req.func = LEAPRAID_FUNC_GET_ADAPTER_FEATURES;
	r = leapraid_handshake_func(adapter,
				    req_sz,
				    (u32 *)&leap_mpi_req,
				    rep_sz,
				    (u16 *)&leap_mpi_rep);
	if (r) {
		dev_err(&adapter->pdev->dev,
			"%s %s: Handshake failed, r=%d\n",
			adapter->adapter_attr.name, __func__, r);
		return r;
	}

	memset(&adapter->adapter_attr.features, 0,
	       sizeof(struct leapraid_adapter_features));
	adapter->adapter_attr.features.req_slot =
		le16_to_cpu(leap_mpi_rep.req_slot);
	adapter->adapter_attr.features.hp_slot =
		le16_to_cpu(leap_mpi_rep.hp_slot);
	adapter->adapter_attr.features.adapter_caps =
		le32_to_cpu(leap_mpi_rep.adapter_caps);
	adapter->adapter_attr.features.max_msix_vectors =
		leap_mpi_rep.max_msix_vectors;
	adapter->adapter_attr.features.max_volumes =
		leap_mpi_rep.max_volumes;
	if (!adapter->adapter_attr.features.max_volumes)
		adapter->adapter_attr.features.max_volumes =
			LEAPRAID_MAX_VOLUMES_DEFAULT;
	adapter->adapter_attr.features.max_dev_handle =
		le16_to_cpu(leap_mpi_rep.max_dev_hdl);
	if (!adapter->adapter_attr.features.max_dev_handle)
		adapter->adapter_attr.features.max_dev_handle =
			LEAPRAID_MAX_DEV_HANDLE_DEFAULT;
	adapter->adapter_attr.features.min_dev_handle =
		le16_to_cpu(leap_mpi_rep.min_dev_hdl);
	if (adapter->adapter_attr.features.adapter_caps &
	    LEAPRAID_ADAPTER_FEATURES_CAP_INTEGRATED_RAID)
		adapter->adapter_attr.raid_support = 1;
	adapter->adapter_attr.wideport_max_queue_depth =
		le16_to_cpu(leap_mpi_rep.sas_wide_max_qdepth) ?
		le16_to_cpu(leap_mpi_rep.sas_wide_max_qdepth) :
		LEAPRAID_SAS_QUEUE_DEPTH;
	adapter->adapter_attr.narrowport_max_queue_depth =
		le16_to_cpu(leap_mpi_rep.sas_narrow_max_qdepth) ?
		le16_to_cpu(leap_mpi_rep.sas_narrow_max_qdepth) :
		LEAPRAID_SAS_QUEUE_DEPTH;
	adapter->adapter_attr.sata_max_queue_depth =
		leap_mpi_rep.sata_max_qdepth ?
		leap_mpi_rep.sata_max_qdepth :
		LEAPRAID_SATA_QUEUE_DEPTH;
	adapter->adapter_attr.raid_volume_max_queue_depth =
		LEAPRAID_RAID_QUEUE_DEPTH;
	dev_info(&adapter->pdev->dev,
		 "max: wp qd=%d, np qd=%d, SATA qd=%d, raid qd=%d\n",
		 adapter->adapter_attr.wideport_max_queue_depth,
		 adapter->adapter_attr.narrowport_max_queue_depth,
		 adapter->adapter_attr.sata_max_queue_depth,
		 adapter->adapter_attr.raid_volume_max_queue_depth);
	if (WARN_ON(!(adapter->adapter_attr.features.adapter_caps &
		      LEAPRAID_ADAPTER_FEATURES_CAP_ATOMIC_REQ)))
		return -EFAULT;
	adapter->adapter_attr.features.fw_version =
		le32_to_cpu(leap_mpi_rep.fw_version);

	fw_major = (adapter->adapter_attr.features.fw_version >>
		    LEAPRAID_VER_MAJOR_SHIFT) & LEAPRAID_VER_MASK;
	fw_minor = (adapter->adapter_attr.features.fw_version >>
		    LEAPRAID_VER_MINOR_SHIFT) & LEAPRAID_VER_MASK;
	fw_build = (adapter->adapter_attr.features.fw_version >>
		    LEAPRAID_VER_BUILD_SHIFT)  & LEAPRAID_VER_MASK;
	fw_release =
		adapter->adapter_attr.features.fw_version & LEAPRAID_VER_MASK;

	dev_info(&adapter->pdev->dev,
		 "Firmware version: %u.%u.%u.%u (0x%08x)\n",
		 fw_major, fw_minor, fw_build, fw_release,
		 adapter->adapter_attr.features.fw_version);

	adapter->adapter_attr.features.msg_ver =
		le16_to_cpu(leap_mpi_rep.msg_ver);
	adapter->adapter_attr.features.product_id =
		le16_to_cpu(leap_mpi_rep.product_id);
	dev_info(&adapter->pdev->dev,
		 "message version: 0x%x, product id 0x%x\n",
		 adapter->adapter_attr.features.msg_ver,
		 adapter->adapter_attr.features.product_id);

	if (adapter->adapter_attr.features.msg_ver < 0x1000) {
		dev_err(&adapter->pdev->dev, "Device not supported\n");
		return -EFAULT;
	}
	adapter->shost->max_id = LEAPRAID_INVALID_INITIAL_VALUE;

	return 0;
}

static inline void leapraid_disable_pcie(struct leapraid_adapter *adapter)
{
	mutex_lock(&adapter->access_ctrl.pci_access_lock);
	if (adapter->iomem_base) {
		iounmap(adapter->iomem_base);
		adapter->iomem_base = NULL;
	}
	if (pci_is_enabled(adapter->pdev)) {
		pci_release_regions(adapter->pdev);
		pci_disable_device(adapter->pdev);
	}
	mutex_unlock(&adapter->access_ctrl.pci_access_lock);
}

static int leapraid_enable_pcie(struct leapraid_adapter *adapter)
{
	u64 dma_mask;
	int rc;

	rc = pci_enable_device(adapter->pdev);
	if (rc) {
		dev_err(&adapter->pdev->dev, "Failed to enable PCI device\n");
		return rc;
	}

	rc = pci_request_regions(adapter->pdev, LEAPRAID_DRIVER_NAME);
	if (rc) {
		dev_err(&adapter->pdev->dev,
			"Failed to obtain PCI resources\n");
		return rc;
	}

	if (sizeof(dma_addr_t) > 4) {
		dma_mask = DMA_BIT_MASK(DMA_64_BITS);
		adapter->adapter_attr.use_32_dma_mask = 0;
	} else {
		dma_mask = DMA_BIT_MASK(DMA_32_BITS);
		adapter->adapter_attr.use_32_dma_mask = 1;
	}

	rc = dma_set_mask_and_coherent(&adapter->pdev->dev, dma_mask);
	if (rc) {
		dev_err(&adapter->pdev->dev,
			"Failed to set %lld DMA mask\n", dma_mask);
		return rc;
	}
	adapter->iomem_base = ioremap(pci_resource_start(adapter->pdev, 0),
				      sizeof(struct leapraid_reg_base));
	if (!adapter->iomem_base) {
		dev_err(&adapter->pdev->dev,
			"Failed to map memory for controller registers\n");
		return -ENOMEM;
	}

	pci_set_master(adapter->pdev);

	return 0;
}

static void leapraid_cpus_on_irq(struct leapraid_adapter *adapter)
{
	struct leapraid_int_rq *int_rq;
	unsigned int i, base_group, this_group;
	unsigned int cpu, nr_cpus, total_msix, index;

	total_msix = adapter->notification_desc.iopoll_qdex;
	nr_cpus = num_online_cpus();

	if (!nr_cpus || !total_msix)
		return;
	base_group = nr_cpus / total_msix;

	cpu = cpumask_first(cpu_online_mask);
	for (index = 0; index < adapter->notification_desc.iopoll_qdex;
	     index++) {
		int_rq = &adapter->notification_desc.int_rqs[index];

		if (cpu >= adapter->notification_desc.msix_cpu_map_sz)
			break;

		this_group = base_group +
				(index < (nr_cpus % total_msix) ? 1 : 0);

		for (i = 0 ; i < this_group ; i++) {
			if (cpu >= adapter->notification_desc.msix_cpu_map_sz)
				break;

			adapter->notification_desc.msix_cpu_map[cpu] =
				int_rq->rq.msix_idx;
			cpu = cpumask_next(cpu, cpu_online_mask);
		}
	}
}

static void leapraid_map_msix_to_cpu(struct leapraid_adapter *adapter)
{
	struct leapraid_int_rq *int_rq;
	const cpumask_t *affinity_mask;
	int cpu;
	u32 i;

	if (!adapter->adapter_attr.rq_cnt)
		return;

	for (i = 0; i < adapter->notification_desc.iopoll_qdex; i++) {
		int_rq = &adapter->notification_desc.int_rqs[i];
		affinity_mask = pci_irq_get_affinity(adapter->pdev,
						     int_rq->rq.msix_idx);
		if (!affinity_mask) {
			dev_warn(&adapter->pdev->dev,
				 "%s: IRQ affinity NULL, msix_idx=%d\n",
				 __func__, int_rq->rq.msix_idx);
			goto out_apply_irq_affinity;
		}

		for_each_cpu_and(cpu, affinity_mask, cpu_online_mask) {
			if (cpu >= adapter->notification_desc.msix_cpu_map_sz)
				continue;

			adapter->notification_desc.msix_cpu_map[cpu] =
				int_rq->rq.msix_idx;
		}
	}
	return;
out_apply_irq_affinity:
	leapraid_cpus_on_irq(adapter);
}

static int leapraid_alloc_msix_cpu_map(struct leapraid_adapter *adapter)
{
	adapter->notification_desc.msix_cpu_map_sz = nr_cpu_ids;
	adapter->notification_desc.msix_cpu_map =
		kzalloc(adapter->notification_desc.msix_cpu_map_sz,
			GFP_KERNEL);
	if (!adapter->notification_desc.msix_cpu_map)
		return -ENOMEM;

	return 0;
}

static void leapraid_configure_reply_queue_affinity(
		struct leapraid_adapter *adapter)
{
	if (!adapter || !adapter->notification_desc.msix_enable)
		return;

	leapraid_map_msix_to_cpu(adapter);
}

static void leapraid_free_irq(struct leapraid_adapter *adapter)
{
	struct leapraid_int_rq *int_rq;
	unsigned int i;
	int irq;

	for (i = 0; adapter->notification_desc.int_rqs &&
	     i < adapter->notification_desc.int_rqs_allocated; i++) {
		int_rq = &adapter->notification_desc.int_rqs[i];
		if (!int_rq)
			continue;

		irq = pci_irq_vector(adapter->pdev, int_rq->rq.msix_idx);
		irq_set_affinity_hint(irq, NULL);
		free_irq(irq, &int_rq->rq);
	}
	adapter->notification_desc.int_rqs_allocated = 0;

	if (adapter->notification_desc.irq_vectors_allocated) {
		pci_free_irq_vectors(adapter->pdev);
		adapter->notification_desc.irq_vectors_allocated = 0;
	}

	adapter->notification_desc.msix_enable = 0;

	kfree(adapter->notification_desc.blk_mq_poll_rqs);
	adapter->notification_desc.blk_mq_poll_rqs = NULL;

	kfree(adapter->notification_desc.int_rqs);
	adapter->notification_desc.int_rqs = NULL;

	kfree(adapter->notification_desc.msix_cpu_map);
	adapter->notification_desc.msix_cpu_map = NULL;
	adapter->notification_desc.msix_cpu_map_sz = 0;
	adapter->notification_desc.iopoll_qdex = 0;
	adapter->notification_desc.iopoll_qcnt = 0;
}

static int leapraid_setup_irqs(struct leapraid_adapter *adapter)
{
	int irq_mode = adapter->notification_desc.irq_mode;
	unsigned int i;
	int rc = 0;

	if (irq_mode == LEAPRAID_INTERRUPT_MODE_MSIX) {
		rc = pci_alloc_irq_vectors_affinity(
			adapter->pdev,
			adapter->notification_desc.iopoll_qdex,
			adapter->notification_desc.iopoll_qdex,
			PCI_IRQ_MSIX | PCI_IRQ_AFFINITY, NULL);
		if (rc < 0) {
			dev_err(&adapter->pdev->dev,
				"%d MSI/MSIX vectors allocated failed!\n",
				adapter->notification_desc.iopoll_qdex);
			return rc;
		}

		adapter->notification_desc.irq_vectors_allocated = 1;
	}

	for (i = 0; i < adapter->notification_desc.iopoll_qdex; i++) {
		adapter->notification_desc.int_rqs[i].rq.adapter = adapter;
		adapter->notification_desc.int_rqs[i].rq.msix_idx = i;
		atomic_set(&adapter->notification_desc.int_rqs[i].rq.busy, 0);
		if (irq_mode == LEAPRAID_INTERRUPT_MODE_MSIX)
			snprintf(adapter->notification_desc.int_rqs[i].rq.name,
				 LEAPRAID_NAME_LENGTH, "%s%u-MSIx%u",
				 LEAPRAID_DRIVER_NAME,
				 adapter->adapter_attr.id, i);
		else if (irq_mode == LEAPRAID_INTERRUPT_MODE_MSI)
			snprintf(adapter->notification_desc.int_rqs[i].rq.name,
				 LEAPRAID_NAME_LENGTH, "%s%u-MSI%u",
				 LEAPRAID_DRIVER_NAME,
				 adapter->adapter_attr.id, i);

		rc = request_irq(pci_irq_vector(adapter->pdev, i),
				 leapraid_irq_handler,
				 IRQF_SHARED,
				 adapter->notification_desc.int_rqs[i].rq.name,
				 &adapter->notification_desc.int_rqs[i].rq);
		if (rc) {
			dev_err(&adapter->pdev->dev,
				"MSI/MSIx: request_irq %s failed!\n",
				adapter->notification_desc.int_rqs[i].rq.name);
			return rc;
		}
		adapter->notification_desc.int_rqs_allocated++;
	}

	return 0;
}

static int leapraid_setup_legacy_int(struct leapraid_adapter *adapter)
{
	int rc;

	adapter->notification_desc.int_rqs[0].rq.adapter = adapter;
	adapter->notification_desc.int_rqs[0].rq.msix_idx = 0;
	atomic_set(&adapter->notification_desc.int_rqs[0].rq.busy, 0);
	snprintf(adapter->notification_desc.int_rqs[0].rq.name,
		 LEAPRAID_NAME_LENGTH, "%s%d-LegacyInt",
		 LEAPRAID_DRIVER_NAME, adapter->adapter_attr.id);

	rc = pci_alloc_irq_vectors_affinity(
		adapter->pdev,
		adapter->notification_desc.iopoll_qdex,
		adapter->notification_desc.iopoll_qdex,
		PCI_IRQ_INTX | PCI_IRQ_AFFINITY,
		NULL);
	if (rc < 0) {
		dev_err(&adapter->pdev->dev,
			"Legacy irq allocated failed!\n");
		return rc;
	}

	adapter->notification_desc.irq_vectors_allocated = 1;
	adapter->notification_desc.irq_mode = LEAPRAID_INTERRUPT_MODE_LEGACY;
	rc = request_irq(pci_irq_vector(adapter->pdev, 0),
			 leapraid_irq_handler,
			 IRQF_SHARED,
			 adapter->notification_desc.int_rqs[0].rq.name,
			 &adapter->notification_desc.int_rqs[0].rq);
	if (rc) {
		irq_set_affinity_hint(pci_irq_vector(adapter->pdev, 0), NULL);
		pci_free_irq_vectors(adapter->pdev);
		adapter->notification_desc.irq_vectors_allocated = 0;
		dev_err(&adapter->pdev->dev,
			"Legacy Int: request_irq %s failed!\n",
			adapter->notification_desc.int_rqs[0].rq.name);
		return -EBUSY;
	}
	adapter->notification_desc.int_rqs_allocated = 1;
	return rc;
}

static int leapraid_set_legacy_int(struct leapraid_adapter *adapter)
{
	int rc;

	rc = leapraid_alloc_msix_cpu_map(adapter);
	if (rc)
		return rc;

	adapter->adapter_attr.rq_cnt = 1;
	adapter->notification_desc.iopoll_qdex =
		adapter->adapter_attr.rq_cnt;
	adapter->notification_desc.iopoll_qcnt = 0;
	dev_info(&adapter->pdev->dev,
		 "Legacy Intr: req queue cnt=%d intr=%d/poll=%d rep queues!\n",
		 adapter->adapter_attr.rq_cnt,
		 adapter->notification_desc.iopoll_qdex,
		 adapter->notification_desc.iopoll_qcnt);
	adapter->notification_desc.int_rqs =
		kzalloc_objs(struct leapraid_int_rq,
			     adapter->notification_desc.iopoll_qdex);
	if (!adapter->notification_desc.int_rqs)
		return -ENOMEM;

	return leapraid_setup_legacy_int(adapter);
}

static int leapraid_set_msix(struct leapraid_adapter *adapter)
{
	int iopoll_qcnt = 0;
	unsigned int i;
	int rc, msix_cnt;

	msix_cnt = pci_msix_vec_count(adapter->pdev);
	if (msix_cnt <= 0 ||
	    adapter->adapter_attr.features.max_msix_vectors == 0) {
		dev_info(&adapter->pdev->dev, "MSIX unsupported!\n");
		return -EOPNOTSUPP;
	}

	msix_cnt = min_t(int, msix_cnt,
			 adapter->adapter_attr.features.max_msix_vectors);

	if (reset_devices)
		adapter->adapter_attr.rq_cnt = 1;
	else
		adapter->adapter_attr.rq_cnt = min_t(int,
						     num_online_cpus(),
						     msix_cnt);

	if (max_msix_vectors > 0)
		adapter->adapter_attr.rq_cnt = min_t(
			int, max_msix_vectors, adapter->adapter_attr.rq_cnt);

	if (adapter->adapter_attr.rq_cnt <= 1)
		adapter->shost->host_tagset = 0;
	if (adapter->shost->host_tagset) {
		iopoll_qcnt = poll_queues;
		if (iopoll_qcnt >= adapter->adapter_attr.rq_cnt)
			iopoll_qcnt = 0;
	}
	if (iopoll_qcnt) {
		adapter->notification_desc.blk_mq_poll_rqs =
			kzalloc_objs(struct leapraid_blk_mq_poll_rq,
				     iopoll_qcnt);
		if (!adapter->notification_desc.blk_mq_poll_rqs)
			return -ENOMEM;
		adapter->adapter_attr.rq_cnt =
			min(adapter->adapter_attr.rq_cnt + iopoll_qcnt,
			    msix_cnt);
	}

	adapter->notification_desc.iopoll_qdex =
		adapter->adapter_attr.rq_cnt - iopoll_qcnt;

	adapter->notification_desc.iopoll_qcnt = iopoll_qcnt;
	dev_info(&adapter->pdev->dev,
		 "MSIx: req queue cnt=%d, intr=%d/poll=%d rep queues!\n",
		 adapter->adapter_attr.rq_cnt,
		 adapter->notification_desc.iopoll_qdex,
		 adapter->notification_desc.iopoll_qcnt);

	adapter->notification_desc.int_rqs =
		kzalloc_objs(struct leapraid_int_rq,
			     adapter->notification_desc.iopoll_qdex);
	if (!adapter->notification_desc.int_rqs)
		return -ENOMEM;

	for (i = 0; i < adapter->notification_desc.iopoll_qcnt; i++) {
		adapter->notification_desc.blk_mq_poll_rqs[i].rq.adapter =
			adapter;
		adapter->notification_desc.blk_mq_poll_rqs[i].rq.msix_idx =
			i + adapter->notification_desc.iopoll_qdex;
		atomic_set(
			&adapter->notification_desc.blk_mq_poll_rqs[i].rq.busy,
			0);
		snprintf(adapter->notification_desc.blk_mq_poll_rqs[i].rq.name,
			 LEAPRAID_NAME_LENGTH,
			 "%s%u-MQ-Poll%u", LEAPRAID_DRIVER_NAME,
			 adapter->adapter_attr.id, i);
		atomic_set(&adapter->notification_desc.blk_mq_poll_rqs[i].busy,
			   0);
		atomic_set(&adapter->notification_desc.blk_mq_poll_rqs[i].pause,
			   0);
	}

	rc = leapraid_alloc_msix_cpu_map(adapter);
	if (rc)
		return rc;

	adapter->notification_desc.irq_mode = LEAPRAID_INTERRUPT_MODE_MSIX;
	adapter->notification_desc.msix_enable = 1;
	rc = leapraid_setup_irqs(adapter);
	if (rc) {
		leapraid_free_irq(adapter);
		adapter->notification_desc.msix_enable = 0;
		return rc;
	}

	return 0;
}

static int leapraid_set_msi(struct leapraid_adapter *adapter)
{
	int iopoll_qcnt = 0;
	unsigned int i;
	int rc, msi_cnt;

	msi_cnt = pci_msi_vec_count(adapter->pdev);
	if (msi_cnt <= 0 ||
	    adapter->adapter_attr.features.max_msix_vectors == 0) {
		dev_info(&adapter->pdev->dev, "MSI unsupported!\n");
		return -EOPNOTSUPP;
	}

	msi_cnt = min_t(int, msi_cnt,
			adapter->adapter_attr.features.max_msix_vectors);

	if (reset_devices)
		adapter->adapter_attr.rq_cnt = 1;
	else
		adapter->adapter_attr.rq_cnt = min_t(int,
						     num_online_cpus(),
						     msi_cnt);

	if (max_msix_vectors > 0)
		adapter->adapter_attr.rq_cnt = min_t(
			int, max_msix_vectors, adapter->adapter_attr.rq_cnt);

	if (adapter->adapter_attr.rq_cnt <= 1)
		adapter->shost->host_tagset = 0;
	if (adapter->shost->host_tagset) {
		iopoll_qcnt = poll_queues;
		if (iopoll_qcnt >= adapter->adapter_attr.rq_cnt)
			iopoll_qcnt = 0;
	}

	if (iopoll_qcnt) {
		adapter->notification_desc.blk_mq_poll_rqs =
			kzalloc_objs(struct leapraid_blk_mq_poll_rq,
				     iopoll_qcnt);
		if (!adapter->notification_desc.blk_mq_poll_rqs)
			return -ENOMEM;

		adapter->adapter_attr.rq_cnt =
			 min(adapter->adapter_attr.rq_cnt + iopoll_qcnt,
			     msi_cnt);
	}

	adapter->notification_desc.iopoll_qdex =
		adapter->adapter_attr.rq_cnt - iopoll_qcnt;
	rc = pci_alloc_irq_vectors_affinity(
		adapter->pdev,
		1,
		adapter->notification_desc.iopoll_qdex,
		PCI_IRQ_MSI | PCI_IRQ_AFFINITY, NULL);
	if (rc < 0) {
		dev_err(&adapter->pdev->dev,
			"%d MSI vectors allocated failed!\n",
			adapter->notification_desc.iopoll_qdex);
		leapraid_free_irq(adapter);
		return rc;
	}
	adapter->notification_desc.irq_vectors_allocated = 1;
	if (rc != adapter->notification_desc.iopoll_qdex) {
		adapter->notification_desc.iopoll_qdex = rc;
		adapter->adapter_attr.rq_cnt =
			adapter->notification_desc.iopoll_qdex + iopoll_qcnt;
	}
	adapter->notification_desc.iopoll_qcnt = iopoll_qcnt;
	dev_info(&adapter->pdev->dev,
		 "MSI: req queue cnt=%d, intr=%d/poll=%d rep queues!\n",
		 adapter->adapter_attr.rq_cnt,
		 adapter->notification_desc.iopoll_qdex,
		 adapter->notification_desc.iopoll_qcnt);

	adapter->notification_desc.int_rqs =
		kzalloc_objs(struct leapraid_int_rq,
			     adapter->notification_desc.iopoll_qdex);
	if (!adapter->notification_desc.int_rqs)
		return -ENOMEM;

	for (i = 0; i < adapter->notification_desc.iopoll_qcnt; i++) {
		adapter->notification_desc.blk_mq_poll_rqs[i].rq.adapter =
			adapter;
		adapter->notification_desc.blk_mq_poll_rqs[i].rq.msix_idx =
			i + adapter->notification_desc.iopoll_qdex;
		atomic_set(
			&adapter->notification_desc.blk_mq_poll_rqs[i].rq.busy,
			0);
		snprintf(adapter->notification_desc.blk_mq_poll_rqs[i].rq.name,
			 LEAPRAID_NAME_LENGTH,
			 "%s%u-MQ-Poll%u", LEAPRAID_DRIVER_NAME,
			 adapter->adapter_attr.id, i);
		atomic_set(
			&adapter->notification_desc.blk_mq_poll_rqs[i].busy,
			0);
		atomic_set(
			&adapter->notification_desc.blk_mq_poll_rqs[i].pause,
			0);
	}

	rc = leapraid_alloc_msix_cpu_map(adapter);
	if (rc)
		return rc;

	adapter->notification_desc.irq_mode = LEAPRAID_INTERRUPT_MODE_MSI;
	adapter->notification_desc.msix_enable = 1;
	rc = leapraid_setup_irqs(adapter);
	if (rc) {
		leapraid_free_irq(adapter);
		adapter->notification_desc.msix_enable = 0;
		return rc;
	}

	return 0;
}

static int leapraid_set_notification_auto(struct leapraid_adapter *adapter)
{
	int rc;

	rc = leapraid_set_msix(adapter);
	if (!rc)
		return 0;

	leapraid_free_irq(adapter);
	dev_info(&adapter->pdev->dev,
		 "MSI-X setup failed (%d), trying MSI\n", rc);

	rc = leapraid_set_msi(adapter);
	if (!rc)
		return 0;

	leapraid_free_irq(adapter);
	dev_info(&adapter->pdev->dev,
		 "MSI setup failed (%d), back to legacy INTx\n", rc);

	rc = leapraid_set_legacy_int(adapter);
	if (rc)
		dev_err(&adapter->pdev->dev,
			"%s: Enable legacy irq failed!\n", __func__);

	return rc;
}

int leapraid_set_pcie_and_notification(struct leapraid_adapter *adapter)
{
	int rc;

	rc = leapraid_enable_pcie(adapter);
	if (rc)
		goto out_fail;

	leapraid_mask_int(adapter);

	rc = leapraid_make_adapter_ready(adapter, PART_RESET);
	if (rc) {
		dev_err(&adapter->pdev->dev, "Make adapter ready failure\n");
		goto out_fail;
	}

	rc = leapraid_get_adapter_features(adapter);
	if (rc) {
		dev_err(&adapter->pdev->dev, "Get adapter feature failure\n");
		goto out_fail;
	}

	rc = leapraid_set_notification_auto(adapter);
	if (rc)
		goto out_fail;

	pci_save_state(adapter->pdev);

	return 0;

out_fail:
	leapraid_free_irq(adapter);
	leapraid_disable_pcie(adapter);
	return rc;
}

void leapraid_disable_controller(struct leapraid_adapter *adapter)
{
	if (!adapter->iomem_base)
		return;

	leapraid_mask_int(adapter);

	adapter->access_ctrl.shost_recovering = 1;
	leapraid_make_adapter_ready(adapter, PART_RESET);
	adapter->access_ctrl.shost_recovering = 0;
	wake_up(&adapter->access_ctrl.recovery_waitq);

	leapraid_free_irq(adapter);
	leapraid_disable_pcie(adapter);
}

static int leapraid_adapter_unit_reset(struct leapraid_adapter *adapter)
{
	int rc = 0;

	writel(LEAPRAID_FUNC_ADAPTER_UNIT_RESET << LEAPRAID_DB_FUNC_SHIFT,
	       &adapter->iomem_base->db);
	if (leapraid_db_wait_ack_and_clear_int(adapter))
		rc = -EFAULT;

	if (!leapraid_wait_adapter_ready(adapter)) {
		dev_err(&adapter->pdev->dev, "unit reset failed\n");
		return -EFAULT;
	}

	return rc;
}

static int leapraid_make_adapter_ready(struct leapraid_adapter *adapter,
				       enum reset_type type)
{
	u32 db;
	int count;

	if (!leapraid_pci_active(adapter))
		return 0;

	count = 0;
	db = leapraid_readl(&adapter->iomem_base->db);
	if ((db & LEAPRAID_DB_MASK) == LEAPRAID_DB_RESET) {
		while ((db & LEAPRAID_DB_MASK) != LEAPRAID_DB_READY) {
			if (count++ == LEAPRAID_DB_RETRY_COUNT_MAX) {
				dev_err(&adapter->pdev->dev,
					"Wait adapter ready timeout\n");
				return -EFAULT;
			}
			ssleep(1);
			db = leapraid_readl(&adapter->iomem_base->db);
			dev_info(&adapter->pdev->dev,
				 "Wait adapter ready, count=%d, db=0x%x\n",
				 count, db);
		}
	}
	if ((db & LEAPRAID_DB_MASK) == LEAPRAID_DB_READY)
		return 0;

	if (db & LEAPRAID_DB_USED)
		goto full_reset;

	if ((db & LEAPRAID_DB_MASK) == LEAPRAID_DB_FAULT)
		goto full_reset;

	if (type == FULL_RESET)
		goto full_reset;

	if ((db & LEAPRAID_DB_MASK) == LEAPRAID_DB_OPERATIONAL &&
	    !leapraid_adapter_unit_reset(adapter))
		return 0;

full_reset:
	return leapraid_host_diag_reset(adapter);
}

static void leapraid_fw_log_exit(struct leapraid_adapter *adapter)
{
	if (!adapter->fw_log_desc.open_pcie_trace)
		return;

	if (adapter->fw_log_desc.fw_log_buffer) {
		wait_event(adapter->fw_log_desc.mmap_waitq,
			   !atomic_read(&adapter->fw_log_desc.mmap_refcnt));
		dma_free_coherent(&adapter->pdev->dev,
				  (LEAPRAID_SYS_LOG_BUF_SIZE +
				   LEAPRAID_SYS_LOG_BUF_RESERVE),
				  adapter->fw_log_desc.fw_log_buffer,
				  adapter->fw_log_desc.fw_log_buffer_dma);
		adapter->fw_log_desc.fw_log_buffer = NULL;
	}
}

static int leapraid_fw_log_init(struct leapraid_adapter *adapter)
{
	struct leapraid_adapter_log_req adapter_log_req;
	struct leapraid_adapter_log_rep adapter_log_rep;
	u16 adapter_status;
	u64 buf_addr;
	u32 rc;

	if (!adapter->fw_log_desc.open_pcie_trace)
		return 0;

	if (!adapter->fw_log_desc.fw_log_buffer) {
		adapter->fw_log_desc.fw_log_buffer =
			dma_alloc_coherent(
				&adapter->pdev->dev,
				(LEAPRAID_SYS_LOG_BUF_SIZE +
				 LEAPRAID_SYS_LOG_BUF_RESERVE),
				&adapter->fw_log_desc.fw_log_buffer_dma,
				GFP_KERNEL);
		if (!adapter->fw_log_desc.fw_log_buffer)
			return -ENOMEM;
	}

	memset(&adapter_log_req, 0, sizeof(struct leapraid_adapter_log_req));
	adapter_log_req.func = LEAPRAID_FUNC_LOGBUF_INIT;
	buf_addr = adapter->fw_log_desc.fw_log_buffer_dma;

	adapter_log_req.mbox.w[0] =
		cpu_to_le32((u32)(buf_addr & 0xFFFFFFFF));
	adapter_log_req.mbox.w[1] =
		cpu_to_le32((u32)((buf_addr >> 32) & 0xFFFFFFFF));
	adapter_log_req.mbox.w[2] =
		cpu_to_le32(LEAPRAID_SYS_LOG_BUF_SIZE);
	rc = leapraid_handshake_func(adapter,
				     sizeof(struct leapraid_adapter_log_req),
				     (u32 *)&adapter_log_req,
				     sizeof(struct leapraid_adapter_log_rep),
				     (u16 *)&adapter_log_rep);
	if (rc != 0) {
		dev_err(&adapter->pdev->dev, "%s: Handshake failed, rc=%d\n",
			__func__, rc);
		return rc;
	}

	adapter_status = le16_to_cpu(adapter_log_rep.adapter_status) &
			LEAPRAID_ADAPTER_STATUS_MASK;
	if (adapter_status != LEAPRAID_ADAPTER_STATUS_SUCCESS) {
		dev_err(&adapter->pdev->dev, "%s: failed!\n", __func__);
		rc = -EIO;
	}

	return rc;
}

static void leapraid_free_host_memory(struct leapraid_adapter *adapter)
{
	unsigned int i;

	if (adapter->mem_desc.task_desc) {
		dma_free_coherent(&adapter->pdev->dev,
				  adapter->adapter_attr.task_desc_dma_size,
				  adapter->mem_desc.task_desc,
				  adapter->mem_desc.task_desc_dma);
		adapter->mem_desc.task_desc = NULL;
	}

	if (adapter->mem_desc.sense_data) {
		dma_free_coherent(
			&adapter->pdev->dev,
			adapter->adapter_attr.io_qd * SCSI_SENSE_BUFFERSIZE,
			adapter->mem_desc.sense_data,
			adapter->mem_desc.sense_data_dma);
		adapter->mem_desc.sense_data = NULL;
	}

	if (adapter->mem_desc.rep_msg) {
		dma_free_coherent(
			&adapter->pdev->dev,
			adapter->adapter_attr.rep_msg_qd * LEAPRAID_REPLY_SIZE,
			adapter->mem_desc.rep_msg,
			adapter->mem_desc.rep_msg_dma);
		adapter->mem_desc.rep_msg = NULL;
	}

	if (adapter->mem_desc.rep_msg_addr) {
		dma_free_coherent(&adapter->pdev->dev,
				  adapter->adapter_attr.rep_msg_qd *
				  LEAPRAID_REP_MSG_ADDR_SIZE,
				  adapter->mem_desc.rep_msg_addr,
				  adapter->mem_desc.rep_msg_addr_dma);
		adapter->mem_desc.rep_msg_addr = NULL;
	}

	if (adapter->mem_desc.rep_desc_seg_maint) {
		for (i = 0; i < adapter->adapter_attr.rep_desc_q_seg_cnt;
		     i++) {
			if (adapter->mem_desc.rep_desc_seg_maint[i]
			    .rep_desc_seg) {
				dma_free_coherent(
					&adapter->pdev->dev,
					(adapter->adapter_attr.rep_desc_qd *
					 LEAPRAID_REP_DESC_ENTRY_SIZE) *
					LEAPRAID_REP_DESC_CHUNK_SIZE,
					adapter->mem_desc.rep_desc_seg_maint[i]
						.rep_desc_seg,
					adapter->mem_desc.rep_desc_seg_maint[i]
						.rep_desc_seg_dma);
				adapter->mem_desc.rep_desc_seg_maint[i]
					.rep_desc_seg = NULL;
			}
		}

		if (adapter->mem_desc.rep_desc_q_arr) {
			dma_free_coherent(
				&adapter->pdev->dev,
				adapter->adapter_attr.rq_cnt *
				LEAPRAID_REP_RQ_CNT_SIZE,
				adapter->mem_desc.rep_desc_q_arr,
				adapter->mem_desc.rep_desc_q_arr_dma);
			adapter->mem_desc.rep_desc_q_arr = NULL;
		}

		for (i = 0; i < adapter->adapter_attr.rep_desc_q_seg_cnt; i++) {
			struct leapraid_mem_desc *mem_desc =
				 &adapter->mem_desc;
			kfree(adapter->mem_desc.rep_desc_seg_maint[i]
				.rep_desc_maint);
			mem_desc->rep_desc_seg_maint[i].rep_desc_maint = NULL;
		}
		kfree(adapter->mem_desc.rep_desc_seg_maint);
		adapter->mem_desc.rep_desc_seg_maint = NULL;
	}

	kfree(adapter->mem_desc.taskid_to_uniq_tag);
	adapter->mem_desc.taskid_to_uniq_tag = NULL;

	dma_pool_destroy(adapter->mem_desc.sg_chain_pool);
	adapter->mem_desc.sg_chain_pool = NULL;
}

static inline bool leapraid_is_in_same_4g_seg(dma_addr_t start, u32 size)
{
	return upper_32_bits(start) == upper_32_bits(start + size - 1);
}

int leapraid_internal_init_cmd_priv(struct leapraid_adapter *adapter,
				    struct leapraid_io_req_tracker *io_tracker)
{
	io_tracker->chain =
		dma_pool_alloc(adapter->mem_desc.sg_chain_pool,
			       GFP_KERNEL,
			       &io_tracker->chain_dma);

	if (!io_tracker->chain)
		return -ENOMEM;

	return 0;
}

void leapraid_internal_exit_cmd_priv(struct leapraid_adapter *adapter,
				     struct leapraid_io_req_tracker *io_tracker)
{
	if (io_tracker && io_tracker->chain)
		dma_pool_free(adapter->mem_desc.sg_chain_pool,
			      io_tracker->chain,
			      io_tracker->chain_dma);
}

static int leapraid_request_host_memory(struct leapraid_adapter *adapter)
{
	struct leapraid_adapter_features *facts =
			&adapter->adapter_attr.features;
	u16 rep_desc_q_cnt_allocated;
	unsigned int i, j;
	int rc;

	/* Scatter-gather table size. */
	adapter->shost->sg_tablesize = LEAPRAID_SG_DEPTH;
	if (reset_devices)
		adapter->shost->sg_tablesize =
				LEAPRAID_KDUMP_MIN_PHYS_SEGMENTS;
	/* High-priority commands queue depth. */
	adapter->dynamic_task_desc.hp_cmd_qd = LEAPRAID_FIXED_HP_CMDS;
	/* Internal commands queue depth. */
	adapter->dynamic_task_desc.inter_cmd_qd = LEAPRAID_FIXED_INTER_CMDS;
	/* Adapter commands total queue depth. */
	if (reset_devices)
		adapter->adapter_attr.adapter_total_qd =
			LEAPRAID_DEFAULT_CMD_QD_OFFSET +
			adapter->dynamic_task_desc.inter_cmd_qd +
			adapter->dynamic_task_desc.hp_cmd_qd;
	else
		adapter->adapter_attr.adapter_total_qd = facts->req_slot +
				adapter->dynamic_task_desc.hp_cmd_qd;
	/* Reply message queue depth. */
	adapter->adapter_attr.rep_msg_qd =
		adapter->adapter_attr.adapter_total_qd +
		LEAPRAID_DEFAULT_CMD_QD_OFFSET;
	/* Reply descriptor queue depth. */
	adapter->adapter_attr.rep_desc_qd =
		round_up(adapter->adapter_attr.adapter_total_qd +
			 adapter->adapter_attr.rep_msg_qd +
			 LEAPRAID_TASKID_OFFSET_CTRL_CMD,
			 LEAPRAID_REPLY_QD_ALIGNMENT);
	/* SCSI command I/O depth. */
	adapter->adapter_attr.io_qd =
		adapter->adapter_attr.adapter_total_qd -
		adapter->dynamic_task_desc.hp_cmd_qd -
		adapter->dynamic_task_desc.inter_cmd_qd;
	/* SCSI host can queue. */
	adapter->shost->can_queue = adapter->adapter_attr.io_qd -
		LEAPRAID_TASKID_OFFSET_CTRL_CMD;
	adapter->driver_cmds.ctl_cmd.taskid = adapter->shost->can_queue +
		LEAPRAID_TASKID_OFFSET_CTRL_CMD;

	/* Allocate task descriptor. */
try_again:
	adapter->adapter_attr.task_desc_dma_size =
		(adapter->adapter_attr.adapter_total_qd +
			LEAPRAID_TASKID_OFFSET_CTRL_CMD) *
		LEAPRAID_REQUEST_SIZE;
	adapter->mem_desc.task_desc =
		dma_alloc_coherent(&adapter->pdev->dev,
				   adapter->adapter_attr.task_desc_dma_size,
				   &adapter->mem_desc.task_desc_dma,
				   GFP_KERNEL);
	if (!adapter->mem_desc.task_desc)
		return -ENOMEM;

	/* Allocate chain message pool. */
	adapter->mem_desc.sg_chain_pool_size =
		LEAPRAID_DEFAULT_CHAINS_PER_IO * LEAPRAID_CHAIN_SEG_SIZE;
	adapter->mem_desc.sg_chain_pool =
		dma_pool_create("leapraid chain pool",
				&adapter->pdev->dev,
				adapter->mem_desc.sg_chain_pool_size,
				LEAPRAID_DMA_ALIGN, 0);
	if (!adapter->mem_desc.sg_chain_pool)
		return -ENOMEM;

	/* Allocate I/O tracker to SCSI I/O. */
	adapter->mem_desc.taskid_to_uniq_tag =
		kcalloc(adapter->shost->can_queue, sizeof(u16), GFP_KERNEL);
	if (!adapter->mem_desc.taskid_to_uniq_tag)
		return -ENOMEM;

	adapter->dynamic_task_desc.hp_taskid =
		adapter->adapter_attr.io_qd +
		LEAPRAID_HP_TASKID_OFFSET_CTL_CMD;
	/* Allocate static high-priority task ID. */
	adapter->driver_cmds.ctl_cmd.hp_taskid =
		adapter->dynamic_task_desc.hp_taskid;
	adapter->driver_cmds.tm_cmd.hp_taskid =
		adapter->dynamic_task_desc.hp_taskid +
		LEAPRAID_HP_TASKID_OFFSET_TM_CMD;

	adapter->dynamic_task_desc.inter_taskid =
		adapter->dynamic_task_desc.hp_taskid +
		adapter->dynamic_task_desc.hp_cmd_qd;
	adapter->driver_cmds.scan_dev_cmd.inter_taskid =
		adapter->dynamic_task_desc.inter_taskid;
	adapter->driver_cmds.cfg_op_cmd.inter_taskid =
		adapter->dynamic_task_desc.inter_taskid +
		LEAPRAID_TASKID_OFFSET_CFG_OP_CMD;
	adapter->driver_cmds.transport_cmd.inter_taskid =
		adapter->dynamic_task_desc.inter_taskid +
		LEAPRAID_TASKID_OFFSET_TRANSPORT_CMD;
	adapter->driver_cmds.enc_cmd.inter_taskid =
		adapter->dynamic_task_desc.inter_taskid +
		LEAPRAID_TASKID_OFFSET_ENC_CMD;
	adapter->driver_cmds.notify_event_cmd.inter_taskid =
		adapter->dynamic_task_desc.inter_taskid +
		LEAPRAID_TASKID_OFFSET_NOTIFY_EVENT_CMD;
	dev_info(&adapter->pdev->dev, "queue depth:\n");
	dev_info(&adapter->pdev->dev, "	host->can_queue: %d\n",
		 adapter->shost->can_queue);
	dev_info(&adapter->pdev->dev, "	io_qd: %d\n",
		 adapter->adapter_attr.io_qd);
	dev_info(&adapter->pdev->dev, "	hpr_cmd_qd: %d\n",
		 adapter->dynamic_task_desc.hp_cmd_qd);
	dev_info(&adapter->pdev->dev, "	inter_cmd_qd: %d\n",
		 adapter->dynamic_task_desc.inter_cmd_qd);
	dev_info(&adapter->pdev->dev, "	adapter_total_qd: %d\n",
		 adapter->adapter_attr.adapter_total_qd);

	dev_info(&adapter->pdev->dev, "taskid range:\n");
	dev_info(&adapter->pdev->dev,
		 "	adapter->dynamic_task_desc.hp_taskid: %d\n",
		 adapter->dynamic_task_desc.hp_taskid);
	dev_info(&adapter->pdev->dev,
		 "	adapter->dynamic_task_desc.inter_taskid: %d\n",
		 adapter->dynamic_task_desc.inter_taskid);

	/*
	 * Allocate sense data DMA buffer.
	 * Constraint: Must reside within the same 4GB segment
	 * (driver-maintained).
	 */
	adapter->mem_desc.sense_data =
		dma_alloc_coherent(
			&adapter->pdev->dev,
			adapter->adapter_attr.io_qd * SCSI_SENSE_BUFFERSIZE,
			&adapter->mem_desc.sense_data_dma,
			GFP_KERNEL);
	if (!adapter->mem_desc.sense_data)
		return -ENOMEM;

	if (!leapraid_is_in_same_4g_seg(adapter->mem_desc.sense_data_dma,
					adapter->adapter_attr.io_qd *
					SCSI_SENSE_BUFFERSIZE)) {
		dev_warn(&adapter->pdev->dev,
			 "Try 32bit DMA: Sense not in same 4G\n");
		rc = -EAGAIN;
		goto out_fail;
	}

	/*
	 * Allocate reply frame buffer.
	 * Constraint: Must reside in the same 4GB segment as the sense DMA.
	 */
	adapter->mem_desc.rep_msg =
		dma_alloc_coherent(&adapter->pdev->dev,
				   adapter->adapter_attr.rep_msg_qd *
				   LEAPRAID_REPLY_SIZE,
				   &adapter->mem_desc.rep_msg_dma,
				   GFP_KERNEL);
	if (!adapter->mem_desc.rep_msg) {
		rc = -ENOMEM;
		goto out_fail;
	}

	if (!leapraid_is_in_same_4g_seg(adapter->mem_desc.rep_msg_dma,
					adapter->adapter_attr.rep_msg_qd *
					LEAPRAID_REPLY_SIZE)) {
		dev_warn(&adapter->pdev->dev,
			 "Use 32 bit DMA due to rep msg is not in same 4g!\n");
		rc = -EAGAIN;
		goto out_fail;
	}

	/* Address of reply frame. */
	adapter->mem_desc.rep_msg_addr =
		dma_alloc_coherent(&adapter->pdev->dev,
				   adapter->adapter_attr.rep_msg_qd *
				   LEAPRAID_REP_MSG_ADDR_SIZE,
				   &adapter->mem_desc.rep_msg_addr_dma,
				   GFP_KERNEL);
	if (!adapter->mem_desc.rep_msg_addr)
		return -ENOMEM;
	adapter->adapter_attr.rep_desc_q_seg_cnt =
		DIV_ROUND_UP(adapter->adapter_attr.rq_cnt,
			     LEAPRAID_REP_DESC_CHUNK_SIZE);
	adapter->mem_desc.rep_desc_seg_maint =
		kzalloc_objs(struct leapraid_rep_desc_seg_maint,
			     adapter->adapter_attr.rep_desc_q_seg_cnt);
	if (!adapter->mem_desc.rep_desc_seg_maint)
		return -ENOMEM;

	rep_desc_q_cnt_allocated = 0;
	for (i = 0; i < adapter->adapter_attr.rep_desc_q_seg_cnt; i++) {
		adapter->mem_desc.rep_desc_seg_maint[i].rep_desc_maint =
			kzalloc_objs(struct leapraid_rep_desc_maint,
				     LEAPRAID_REP_DESC_CHUNK_SIZE);
		if (!adapter->mem_desc.rep_desc_seg_maint[i].rep_desc_maint)
			return -ENOMEM;

		adapter->mem_desc.rep_desc_seg_maint[i].rep_desc_seg =
			dma_alloc_coherent(
				&adapter->pdev->dev,
				(adapter->adapter_attr.rep_desc_qd *
				LEAPRAID_REP_DESC_ENTRY_SIZE) *
				LEAPRAID_REP_DESC_CHUNK_SIZE,
				&adapter->mem_desc.rep_desc_seg_maint[i]
					.rep_desc_seg_dma,
				GFP_KERNEL);
		if (!adapter->mem_desc.rep_desc_seg_maint[i].rep_desc_seg)
			return -ENOMEM;

		for (j = 0; j < LEAPRAID_REP_DESC_CHUNK_SIZE; j++) {
			if (rep_desc_q_cnt_allocated >=
			    adapter->adapter_attr.rq_cnt)
				break;
			adapter->mem_desc
				.rep_desc_seg_maint[i]
				.rep_desc_maint[j]
				.rep_desc =
					(void *)((u8 *)(
					adapter->mem_desc
						.rep_desc_seg_maint[i]
						.rep_desc_seg) +
					j *
					(adapter->adapter_attr.rep_desc_qd *
						LEAPRAID_REP_DESC_ENTRY_SIZE));
			adapter->mem_desc
				.rep_desc_seg_maint[i]
				.rep_desc_maint[j]
				.rep_desc_dma =
					adapter->mem_desc
						.rep_desc_seg_maint[i]
						.rep_desc_seg_dma +
					j *
					(adapter->adapter_attr.rep_desc_qd *
						LEAPRAID_REP_DESC_ENTRY_SIZE);
			rep_desc_q_cnt_allocated++;
		}
	}

	if (!reset_devices) {
		adapter->mem_desc.rep_desc_q_arr =
			dma_alloc_coherent(
				&adapter->pdev->dev,
				adapter->adapter_attr.rq_cnt *
				LEAPRAID_REP_RQ_CNT_SIZE,
				&adapter->mem_desc.rep_desc_q_arr_dma,
				GFP_KERNEL);
		if (!adapter->mem_desc.rep_desc_q_arr)
			return -ENOMEM;
	}

	return 0;
out_fail:
	if (rc == -EAGAIN) {
		leapraid_free_host_memory(adapter);
		adapter->adapter_attr.use_32_dma_mask = 1;
		rc = dma_set_mask_and_coherent(&adapter->pdev->dev,
					       DMA_BIT_MASK(DMA_32_BITS));
		if (rc) {
			dev_err(&adapter->pdev->dev,
				"Failed to set 32 DMA mask\n");
			return rc;
		}
		goto try_again;
	}
	return rc;
}

static int leapraid_alloc_dev_topo_bitmaps(struct leapraid_adapter *adapter)
{
	u16 pd_hdls_sz;

	pd_hdls_sz =
		BITS_TO_LONGS(
			adapter->adapter_attr.features.max_dev_handle + 1) *
		sizeof(unsigned long);

	adapter->dev_topo.pd_hdls_sz = pd_hdls_sz;
	adapter->dev_topo.pd_hdls =
		kzalloc(adapter->dev_topo.pd_hdls_sz, GFP_KERNEL);
	if (!adapter->dev_topo.pd_hdls)
		return -ENOMEM;

	adapter->dev_topo.blocking_hdls =
		kzalloc(adapter->dev_topo.pd_hdls_sz, GFP_KERNEL);
	if (!adapter->dev_topo.blocking_hdls)
		return -ENOMEM;

	return 0;
}

static void leapraid_free_dev_topo_bitmaps(struct leapraid_adapter *adapter)
{
	kfree(adapter->dev_topo.pd_hdls);
	kfree(adapter->dev_topo.blocking_hdls);
}

static int leapraid_init_driver_cmds(struct leapraid_adapter *adapter)
{
	INIT_LIST_HEAD(&adapter->driver_cmds.special_cmd_list);

	adapter->driver_cmds.scan_dev_cmd.status = LEAPRAID_CMD_NOT_USED;
	adapter->driver_cmds.scan_dev_cmd.cb_idx = LEAPRAID_SCAN_DEV_CB_IDX;
	list_add_tail(&adapter->driver_cmds.scan_dev_cmd.list,
		      &adapter->driver_cmds.special_cmd_list);

	adapter->driver_cmds.cfg_op_cmd.status = LEAPRAID_CMD_NOT_USED;
	adapter->driver_cmds.cfg_op_cmd.cb_idx = LEAPRAID_CONFIG_CB_IDX;
	mutex_init(&adapter->driver_cmds.cfg_op_cmd.mutex);
	list_add_tail(&adapter->driver_cmds.cfg_op_cmd.list,
		      &adapter->driver_cmds.special_cmd_list);

	adapter->driver_cmds.transport_cmd.status = LEAPRAID_CMD_NOT_USED;
	adapter->driver_cmds.transport_cmd.cb_idx = LEAPRAID_TRANSPORT_CB_IDX;
	mutex_init(&adapter->driver_cmds.transport_cmd.mutex);
	list_add_tail(&adapter->driver_cmds.transport_cmd.list,
		      &adapter->driver_cmds.special_cmd_list);

	adapter->driver_cmds.enc_cmd.status = LEAPRAID_CMD_NOT_USED;
	adapter->driver_cmds.enc_cmd.cb_idx = LEAPRAID_ENC_CB_IDX;
	mutex_init(&adapter->driver_cmds.enc_cmd.mutex);
	list_add_tail(&adapter->driver_cmds.enc_cmd.list,
		      &adapter->driver_cmds.special_cmd_list);

	adapter->driver_cmds.notify_event_cmd.status = LEAPRAID_CMD_NOT_USED;
	adapter->driver_cmds.notify_event_cmd.cb_idx =
		LEAPRAID_NOTIFY_EVENT_CB_IDX;
	mutex_init(&adapter->driver_cmds.notify_event_cmd.mutex);
	list_add_tail(&adapter->driver_cmds.notify_event_cmd.list,
		      &adapter->driver_cmds.special_cmd_list);

	adapter->driver_cmds.ctl_cmd.status = LEAPRAID_CMD_NOT_USED;
	adapter->driver_cmds.ctl_cmd.cb_idx = LEAPRAID_CTL_CB_IDX;
	mutex_init(&adapter->driver_cmds.ctl_cmd.mutex);
	list_add_tail(&adapter->driver_cmds.ctl_cmd.list,
		      &adapter->driver_cmds.special_cmd_list);

	adapter->driver_cmds.tm_cmd.status = LEAPRAID_CMD_NOT_USED;
	adapter->driver_cmds.tm_cmd.cb_idx = LEAPRAID_TM_CB_IDX;
	mutex_init(&adapter->driver_cmds.tm_cmd.mutex);
	list_add_tail(&adapter->driver_cmds.tm_cmd.list,
		      &adapter->driver_cmds.special_cmd_list);

	return 0;
}

static void leapraid_unmask_evts(struct leapraid_adapter *adapter, u16 evt)
{
	if (evt >= LEAPRAID_MAX_EVENT_NUM)
		return;

	clear_bit(evt, (unsigned long *)adapter->fw_evt_s.leapraid_evt_masks);
}

static void leapraid_init_event_mask(struct leapraid_adapter *adapter)
{
	int i;

	for (i = 0; i < LEAPRAID_EVT_MASK_COUNT; i++)
		adapter->fw_evt_s.leapraid_evt_masks[i] =
			LEAPRAID_INVALID_INITIAL_VALUE;

	leapraid_unmask_evts(adapter, LEAPRAID_EVT_SAS_TOPO_CHANGE_LIST);
	leapraid_unmask_evts(adapter, LEAPRAID_EVT_SAS_ENCL_DEV_STATUS_CHANGE);
	leapraid_unmask_evts(adapter, LEAPRAID_EVT_SAS_DEV_STATUS_CHANGE);
	leapraid_unmask_evts(adapter, LEAPRAID_EVT_IR_CHANGE);
}

static void leapraid_prepare_adapter_init_req(
		struct leapraid_adapter *adapter,
		struct leapraid_adapter_init_req *init_req)
{
	ktime_t cur_time;
	int i, chunk;
	u32 reply_post_free_ary_sz;

	memset(init_req, 0, sizeof(struct leapraid_adapter_init_req));
	init_req->func = LEAPRAID_FUNC_ADAPTER_INIT;
	init_req->who_init = LEAPRAID_WHOINIT_LINUX_DRIVER;
	init_req->msg_ver = cpu_to_le16(LEAPRAID_MSG_VERSION);
	init_req->header_ver = cpu_to_le16(LEAPRAID_HEADER_VERSION);
	init_req->driver_ver =
		cpu_to_le32((LEAPRAID_MAJOR_VERSION <<
				LEAPRAID_VER_MAJOR_SHIFT) |
			(LEAPRAID_MINOR_VERSION << LEAPRAID_VER_MINOR_SHIFT) |
			(LEAPRAID_BUILD_VERSION << LEAPRAID_VER_BUILD_SHIFT) |
			LEAPRAID_RELEASE_VERSION);
	if (adapter->notification_desc.msix_enable)
		init_req->host_msix_vectors = adapter->adapter_attr.rq_cnt;

	init_req->req_frame_size =
		cpu_to_le16(LEAPRAID_REQUEST_SIZE / LEAPRAID_DWORDS_BYTE_SIZE);
	init_req->rep_desc_qd =
		cpu_to_le16(adapter->adapter_attr.rep_desc_qd);
	init_req->rep_msg_qd =
		cpu_to_le16(adapter->adapter_attr.rep_msg_qd);
	init_req->sense_buffer_add_high =
		cpu_to_le32((u64)adapter->mem_desc.sense_data_dma >> 32);
	init_req->rep_msg_dma_high =
		cpu_to_le32((u64)adapter->mem_desc.rep_msg_dma >> 32);
	init_req->task_desc_base_addr =
		cpu_to_le64((u64)adapter->mem_desc.task_desc_dma);
	init_req->rep_msg_addr_dma =
		cpu_to_le64((u64)adapter->mem_desc.rep_msg_addr_dma);
	if (!reset_devices) {
		reply_post_free_ary_sz =
			adapter->adapter_attr.rq_cnt * LEAPRAID_REP_RQ_CNT_SIZE;
		memset(adapter->mem_desc.rep_desc_q_arr, 0,
		       reply_post_free_ary_sz);

		chunk = LEAPRAID_REP_DESC_CHUNK_SIZE;
		for (i = 0; i < adapter->adapter_attr.rq_cnt; i++)
			adapter->mem_desc.rep_desc_q_arr[i].rep_desc_base_addr =
				cpu_to_le64 ((u64)adapter->mem_desc
					     .rep_desc_seg_maint[i / chunk]
					     .rep_desc_maint[i % chunk]
					     .rep_desc_dma);

		init_req->msg_flg =
			LEAPRAID_ADAPTER_INIT_MSGFLG_RDPQ_ARRAY_MODE;
		init_req->rep_desc_q_arr_addr =
			cpu_to_le64((u64)adapter->mem_desc.rep_desc_q_arr_dma);
	} else {
		init_req->rep_desc_q_arr_addr =
			cpu_to_le64((u64)adapter->mem_desc
					.rep_desc_seg_maint[0]
					.rep_desc_maint[0]
					.rep_desc_dma);
	}
	cur_time = ktime_get_real();
	init_req->time_stamp = cpu_to_le64(ktime_to_ms(cur_time));
}

static int leapraid_send_adapter_init(struct leapraid_adapter *adapter)
{
	struct leapraid_adapter_init_req init_req;
	struct leapraid_adapter_init_rep init_rep;
	u16 adapter_status;
	int rc;

	leapraid_prepare_adapter_init_req(adapter, &init_req);

	rc = leapraid_handshake_func(adapter,
				     sizeof(struct leapraid_adapter_init_req),
				     (u32 *)&init_req,
				     sizeof(struct leapraid_adapter_init_rep),
				     (u16 *)&init_rep);
	if (rc != 0) {
		dev_err(&adapter->pdev->dev, "%s: Handshake failed, rc=%d\n",
			__func__, rc);
		return rc;
	}

	adapter_status =
		le16_to_cpu(init_rep.adapter_status) &
		LEAPRAID_ADAPTER_STATUS_MASK;
	if (adapter_status != LEAPRAID_ADAPTER_STATUS_SUCCESS) {
		dev_err(&adapter->pdev->dev, "%s: failed\n", __func__);
		rc = -EIO;
	}

	return rc;
}

static int leapraid_cfg_pages(struct leapraid_adapter *adapter)
{
	union cfg_param_1 cfgp1 = {0};
	union cfg_param_2 cfgp2 = {0};
	union {
		struct leapraid_manufacturing_p0 manufacturing_page0;
		struct leapraid_bios_page3 bios_page3;
		struct leapraid_bios_page2 bios_page2;
	} cfg_page;
	int rc;

	rc = leapraid_op_config_page(adapter, &cfg_page.bios_page3, cfgp1,
				     cfgp2, GET_BIOS_PG3);
	if (rc)
		return rc;

	adapter->adapter_attr.bios_version =
		le32_to_cpu(cfg_page.bios_page3.bios_version);

	rc = leapraid_op_config_page(adapter, &cfg_page.bios_page2, cfgp1,
				     cfgp2, GET_BIOS_PG2);
	if (rc)
		return rc;

	adapter->boot_devs.requested_boot_dev.form =
		cfg_page.bios_page2.requested_boot_dev_form;
	memcpy(adapter->boot_devs.requested_boot_dev.pg_dev,
	       &cfg_page.bios_page2.requested_boot_dev,
	       LEAPRAID_BOOT_DEV_SIZE);
	adapter->boot_devs.requested_alt_boot_dev.form =
		cfg_page.bios_page2.requested_alt_boot_dev_form;
	memcpy(adapter->boot_devs.requested_alt_boot_dev.pg_dev,
	       &cfg_page.bios_page2.requested_alt_boot_dev,
	       LEAPRAID_BOOT_DEV_SIZE);
	adapter->boot_devs.current_boot_dev.form =
		cfg_page.bios_page2.current_boot_dev_form;
	memcpy(adapter->boot_devs.current_boot_dev.pg_dev,
	       &cfg_page.bios_page2.current_boot_dev,
	       LEAPRAID_BOOT_DEV_SIZE);

	rc = leapraid_op_config_page(adapter, &cfg_page.manufacturing_page0,
				     cfgp1, cfgp2, GET_MANUFACTURING_PG0);
	if (rc)
		return rc;

	snprintf(adapter->adapter_attr.board_name,
		 sizeof(adapter->adapter_attr.board_name),
		 "%.*s",
		 (int)sizeof(cfg_page.manufacturing_page0.board_name),
		 cfg_page.manufacturing_page0.board_name);
	return rc;
}

static int leapraid_evt_notify(struct leapraid_adapter *adapter)
{
	struct leapraid_evt_notify_req *evt_notify_req;
	struct leapraid_evt_notify_rep *evt_notify_rep;
	u16 adapter_status;
	int rc = 0;
	int i;

	mutex_lock(&adapter->driver_cmds.notify_event_cmd.mutex);
	adapter->driver_cmds.notify_event_cmd.status = LEAPRAID_CMD_PENDING;
	evt_notify_req =
		leapraid_get_task_desc(
			adapter,
			adapter->driver_cmds.notify_event_cmd.inter_taskid);
	memset(evt_notify_req, 0, sizeof(struct leapraid_evt_notify_req));
	evt_notify_req->func = LEAPRAID_FUNC_EVENT_NOTIFY;
	for (i = 0; i < LEAPRAID_EVT_MASK_COUNT; i++)
		evt_notify_req->evt_masks[i] =
			cpu_to_le32(adapter->fw_evt_s.leapraid_evt_masks[i]);
	init_completion(&adapter->driver_cmds.notify_event_cmd.done);
	leapraid_fire_task(adapter,
			   adapter->driver_cmds.notify_event_cmd.inter_taskid);
	wait_for_completion_timeout(
		&adapter->driver_cmds.notify_event_cmd.done,
		LEAPRAID_NOTIFY_EVENT_CMD_TIMEOUT * HZ);

	if (!(adapter->driver_cmds.notify_event_cmd.status &
	      LEAPRAID_CMD_DONE)) {
		rc = -EFAULT;
		goto out_cleanup;
	}

	if (!(adapter->driver_cmds.notify_event_cmd.status &
	      LEAPRAID_CMD_REPLY_VALID)) {
		rc = -EFAULT;
		goto out_cleanup;
	}

	evt_notify_rep = (void *)&adapter->driver_cmds.notify_event_cmd.reply;
	adapter_status = le16_to_cpu(evt_notify_rep->adapter_status) &
			 LEAPRAID_ADAPTER_STATUS_MASK;
	if (adapter_status != LEAPRAID_ADAPTER_STATUS_SUCCESS)
		rc = -EFAULT;

out_cleanup:
	adapter->driver_cmds.notify_event_cmd.status = LEAPRAID_CMD_NOT_USED;
	mutex_unlock(&adapter->driver_cmds.notify_event_cmd.mutex);

	return rc;
}

int leapraid_scan_dev(struct leapraid_adapter *adapter, bool async_scan_dev)
{
	struct leapraid_scan_dev_req *scan_dev_req;
	struct leapraid_scan_dev_rep *scan_dev_rep;
	u16 adapter_status;
	int rc = 0;

	dev_info(&adapter->pdev->dev,
		 "Send device scan, async_scan_dev=%d!\n", async_scan_dev);

	adapter->driver_cmds.scan_dev_cmd.status = LEAPRAID_CMD_PENDING;
	adapter->driver_cmds.scan_dev_cmd.async_scan_dev = async_scan_dev;
	scan_dev_req =
		leapraid_get_task_desc(
			adapter,
			adapter->driver_cmds.scan_dev_cmd.inter_taskid);
	memset(scan_dev_req, 0, sizeof(struct leapraid_scan_dev_req));
	scan_dev_req->func = LEAPRAID_FUNC_SCAN_DEV;

	if (async_scan_dev) {
		adapter->scan_dev_desc.first_scan_dev_fired = 1;
		leapraid_fire_task(
			adapter,
			adapter->driver_cmds.scan_dev_cmd.inter_taskid);
		return 0;
	}

	init_completion(&adapter->driver_cmds.scan_dev_cmd.done);
	leapraid_fire_task(adapter,
			   adapter->driver_cmds.scan_dev_cmd.inter_taskid);
	wait_for_completion_timeout(&adapter->driver_cmds.scan_dev_cmd.done,
				    LEAPRAID_SCAN_DEV_CMD_TIMEOUT * HZ);
	if (!(adapter->driver_cmds.scan_dev_cmd.status & LEAPRAID_CMD_DONE)) {
		dev_err(&adapter->pdev->dev, "Device scan timeout!\n");
		if (adapter->driver_cmds.scan_dev_cmd.status &
		    LEAPRAID_CMD_RESET)
			rc = -EFAULT;
		else
			rc = -ETIME;
		goto out_cleanup;
	}

	scan_dev_rep = (void *)&adapter->driver_cmds.scan_dev_cmd.reply;
	adapter_status =
		le16_to_cpu(scan_dev_rep->adapter_status) &
		LEAPRAID_ADAPTER_STATUS_MASK;
	if (adapter_status != LEAPRAID_ADAPTER_STATUS_SUCCESS) {
		dev_err(&adapter->pdev->dev, "Device scan failure!\n");
		rc = -EFAULT;
		goto out_cleanup;
	}

out_cleanup:
	adapter->driver_cmds.scan_dev_cmd.status = LEAPRAID_CMD_NOT_USED;
	return rc;
}

static void leapraid_init_task_tracker(struct leapraid_adapter *adapter)
{
	unsigned long flags;

	spin_lock_irqsave(&adapter->dynamic_task_desc.task_lock, flags);

	spin_unlock_irqrestore(&adapter->dynamic_task_desc.task_lock, flags);
}

static void leapraid_init_rep_msg_addr(struct leapraid_adapter *adapter)
{
	u32 reply_address;
	unsigned int i;

	for (i = 0, reply_address = (u32)adapter->mem_desc.rep_msg_dma;
	     i < adapter->adapter_attr.rep_msg_qd;
	     i++, reply_address += LEAPRAID_REPLY_SIZE)
		adapter->mem_desc.rep_msg_addr[i] = cpu_to_le32(reply_address);
}

static void init_rep_desc(
		struct leapraid_rq *rq,
		int index,
		union leapraid_rep_desc_union *reply_post_free_contig)
{
	struct leapraid_adapter *adapter = rq->adapter;
	unsigned int i;

	if (!reset_devices)
		rq->rep_desc =
			adapter->mem_desc
				.rep_desc_seg_maint[index /
					LEAPRAID_REP_DESC_CHUNK_SIZE]
				.rep_desc_maint[index %
					LEAPRAID_REP_DESC_CHUNK_SIZE]
				.rep_desc;
	else
		rq->rep_desc = reply_post_free_contig;

	rq->rep_post_host_idx = 0;
	for (i = 0; i < adapter->adapter_attr.rep_desc_qd; i++)
		rq->rep_desc[i].words = cpu_to_le64(ULLONG_MAX);
}

static void leapraid_init_rep_desc(struct leapraid_adapter *adapter)
{
	union leapraid_rep_desc_union *reply_post_free_contig;
	struct leapraid_int_rq *int_rq;
	struct leapraid_blk_mq_poll_rq *blk_mq_poll_rq;
	unsigned int i;
	int index;

	index = 0;
	reply_post_free_contig = adapter->mem_desc
					.rep_desc_seg_maint[0]
					.rep_desc_maint[0]
					.rep_desc;

	for (i = 0; i < adapter->notification_desc.iopoll_qdex; i++) {
		int_rq = &adapter->notification_desc.int_rqs[i];
		init_rep_desc(&int_rq->rq, index, reply_post_free_contig);
		if (!reset_devices)
			index++;
		else
			reply_post_free_contig +=
				adapter->adapter_attr.rep_desc_qd;
	}

	for (i = 0; i < adapter->notification_desc.iopoll_qcnt; i++) {
		blk_mq_poll_rq = &adapter->notification_desc.blk_mq_poll_rqs[i];
		init_rep_desc(&blk_mq_poll_rq->rq,
			      index, reply_post_free_contig);
		if (!reset_devices)
			index++;
		else
			reply_post_free_contig +=
				adapter->adapter_attr.rep_desc_qd;
	}
}

static void leapraid_init_bar_idx_regs(struct leapraid_adapter *adapter)
{
	struct leapraid_int_rq *int_rq;
	struct leapraid_blk_mq_poll_rq *blk_mq_poll_rq;
	unsigned int i, j;

	adapter->rep_msg_host_idx = adapter->adapter_attr.rep_msg_qd - 1;
	writel(adapter->rep_msg_host_idx,
	       &adapter->iomem_base->rep_msg_host_idx);

	for (i = 0; i < adapter->notification_desc.iopoll_qdex; i++) {
		int_rq = &adapter->notification_desc.int_rqs[i];
		for (j = 0; j < REP_POST_HOST_IDX_REG_CNT; j++)
			writel((int_rq->rq.msix_idx & 7) <<
				LEAPRAID_RPHI_MSIX_IDX_SHIFT,
				&adapter->iomem_base->rep_post_reg_idx[j].idx);
	}

	for (i = 0; i < adapter->notification_desc.iopoll_qcnt; i++) {
		blk_mq_poll_rq =
			&adapter->notification_desc.blk_mq_poll_rqs[i];
		for (j = 0; j < REP_POST_HOST_IDX_REG_CNT; j++)
			writel((blk_mq_poll_rq->rq.msix_idx & 7) <<
				LEAPRAID_RPHI_MSIX_IDX_SHIFT,
				&adapter->iomem_base->rep_post_reg_idx[j].idx);
	}
}

static int leapraid_make_adapter_available(struct leapraid_adapter *adapter)
{
	int rc;

	leapraid_init_task_tracker(adapter);
	leapraid_init_rep_msg_addr(adapter);

	if (adapter->scan_dev_desc.driver_loading)
		leapraid_configure_reply_queue_affinity(adapter);

	leapraid_init_rep_desc(adapter);
	rc = leapraid_send_adapter_init(adapter);
	if (rc)
		return rc;

	leapraid_init_bar_idx_regs(adapter);
	leapraid_unmask_int(adapter);
	rc = leapraid_cfg_pages(adapter);
	if (rc)
		return rc;

	rc = leapraid_evt_notify(adapter);
	if (rc)
		return rc;

	if (!adapter->access_ctrl.shost_recovering) {
		adapter->scan_dev_desc.wait_scan_dev_done = 1;
		return 0;
	}

	return leapraid_scan_dev(adapter, false);
}

int leapraid_ctrl_init(struct leapraid_adapter *adapter)
{
	u32 cap;
	int rc;

	rc = leapraid_init_driver_cmds(adapter);
	if (rc) {
		dev_err(&adapter->pdev->dev, "Init driver cmds failure\n");
		goto out_free;
	}

	rc = leapraid_set_pcie_and_notification(adapter);
	if (rc)
		goto out_free;

	pci_set_drvdata(adapter->pdev, adapter->shost);

	pcie_capability_read_dword(adapter->pdev, PCI_EXP_DEVCAP, &cap);

	if (cap & PCI_EXP_DEVCAP_EXT_TAG)
		pcie_capability_set_word(adapter->pdev, PCI_EXP_DEVCTL,
					 PCI_EXP_DEVCTL_EXT_TAG);

	rc = leapraid_fw_log_init(adapter);
	if (rc) {
		dev_err(&adapter->pdev->dev, "FW log init failure\n");
		goto out_free;
	}

	rc = leapraid_request_host_memory(adapter);
	if (rc) {
		dev_err(&adapter->pdev->dev, "Request host memory failure\n");
		goto out_free;
	}

	init_waitqueue_head(&adapter->reset_desc.reset_wait_queue);
	init_waitqueue_head(&adapter->access_ctrl.shost_recover_wq);

	rc = leapraid_alloc_dev_topo_bitmaps(adapter);
	if (rc) {
		dev_err(&adapter->pdev->dev, "Alloc topo bitmaps failure\n");
		goto out_free;
	}

	leapraid_init_event_mask(adapter);

	rc = leapraid_make_adapter_available(adapter);
	if (rc) {
		dev_err(&adapter->pdev->dev,
			"Make adapter available failure\n");
		goto out_free;
	}

	leapraid_overheat_init(adapter);
	if (!adapter->overheat_desc.fault_overheat_wq) {
		rc = -ENOMEM;
		goto out_free;
	}

	return 0;

out_free:
	adapter->access_ctrl.host_removing = 1;
	leapraid_fw_log_exit(adapter);
	leapraid_disable_controller(adapter);
	leapraid_free_host_memory(adapter);
	leapraid_free_dev_topo_bitmaps(adapter);
	pci_set_drvdata(adapter->pdev, NULL);
	return rc;
}

void leapraid_remove_ctrl(struct leapraid_adapter *adapter)
{
	leapraid_overheat_cleanup(adapter);
	leapraid_check_scheduled_fault_stop(adapter);
	leapraid_fw_log_stop(adapter);
	leapraid_fw_log_exit(adapter);
	leapraid_disable_controller(adapter);
	leapraid_free_host_memory(adapter);
	leapraid_free_dev_topo_bitmaps(adapter);
	leapraid_free_enc_list(adapter);
	pci_set_drvdata(adapter->pdev, NULL);
}