| /* | |
| * This file is part of the Chelsio FCoE driver for Linux. | |
| * | |
| * Copyright (c) 2008-2012 Chelsio Communications, Inc. All rights reserved. | |
| * | |
| * This software is available to you under a choice of one of two | |
| * licenses. You may choose to be licensed under the terms of the GNU | |
| * General Public License (GPL) Version 2, available from the file | |
| * COPYING in the main directory of this source tree, or the | |
| * OpenIB.org BSD license below: | |
| * | |
| * Redistribution and use in source and binary forms, with or | |
| * without modification, are permitted provided that the following | |
| * conditions are met: | |
| * | |
| * - Redistributions of source code must retain the above | |
| * copyright notice, this list of conditions and the following | |
| * disclaimer. | |
| * | |
| * - Redistributions in binary form must reproduce the above | |
| * copyright notice, this list of conditions and the following | |
| * disclaimer in the documentation and/or other materials | |
| * provided with the distribution. | |
| * | |
| * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
| * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
| * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
| * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
| * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
| * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
| * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
| * SOFTWARE. | |
| */ | |
| #include <linux/kernel.h> | |
| #include <linux/string.h> | |
| #include <linux/compiler.h> | |
| #include <linux/slab.h> | |
| #include <asm/page.h> | |
| #include <linux/cache.h> | |
| #include "t4_values.h" | |
| #include "csio_hw.h" | |
| #include "csio_wr.h" | |
| #include "csio_mb.h" | |
| #include "csio_defs.h" | |
| int csio_intr_coalesce_cnt; /* value:SGE_INGRESS_RX_THRESHOLD[0] */ | |
| static int csio_sge_thresh_reg; /* SGE_INGRESS_RX_THRESHOLD[0] */ | |
| int csio_intr_coalesce_time = 10; /* value:SGE_TIMER_VALUE_1 */ | |
| static int csio_sge_timer_reg = 1; | |
| #define CSIO_SET_FLBUF_SIZE(_hw, _reg, _val) \ | |
| csio_wr_reg32((_hw), (_val), SGE_FL_BUFFER_SIZE##_reg##_A) | |
| static void | |
| csio_get_flbuf_size(struct csio_hw *hw, struct csio_sge *sge, uint32_t reg) | |
| { | |
| sge->sge_fl_buf_size[reg] = csio_rd_reg32(hw, SGE_FL_BUFFER_SIZE0_A + | |
| reg * sizeof(uint32_t)); | |
| } | |
| /* Free list buffer size */ | |
| static inline uint32_t | |
| csio_wr_fl_bufsz(struct csio_sge *sge, struct csio_dma_buf *buf) | |
| { | |
| return sge->sge_fl_buf_size[buf->paddr & 0xF]; | |
| } | |
| /* Size of the egress queue status page */ | |
| static inline uint32_t | |
| csio_wr_qstat_pgsz(struct csio_hw *hw) | |
| { | |
| return (hw->wrm.sge.sge_control & EGRSTATUSPAGESIZE_F) ? 128 : 64; | |
| } | |
| /* Ring freelist doorbell */ | |
| static inline void | |
| csio_wr_ring_fldb(struct csio_hw *hw, struct csio_q *flq) | |
| { | |
| /* | |
| * Ring the doorbell only when we have atleast CSIO_QCREDIT_SZ | |
| * number of bytes in the freelist queue. This translates to atleast | |
| * 8 freelist buffer pointers (since each pointer is 8 bytes). | |
| */ | |
| if (flq->inc_idx >= 8) { | |
| csio_wr_reg32(hw, DBPRIO_F | QID_V(flq->un.fl.flid) | | |
| PIDX_T5_V(flq->inc_idx / 8) | DBTYPE_F, | |
| MYPF_REG(SGE_PF_KDOORBELL_A)); | |
| flq->inc_idx &= 7; | |
| } | |
| } | |
| /* Write a 0 cidx increment value to enable SGE interrupts for this queue */ | |
| static void | |
| csio_wr_sge_intr_enable(struct csio_hw *hw, uint16_t iqid) | |
| { | |
| csio_wr_reg32(hw, CIDXINC_V(0) | | |
| INGRESSQID_V(iqid) | | |
| TIMERREG_V(X_TIMERREG_RESTART_COUNTER), | |
| MYPF_REG(SGE_PF_GTS_A)); | |
| } | |
| /* | |
| * csio_wr_fill_fl - Populate the FL buffers of a FL queue. | |
| * @hw: HW module. | |
| * @flq: Freelist queue. | |
| * | |
| * Fill up freelist buffer entries with buffers of size specified | |
| * in the size register. | |
| * | |
| */ | |
| static int | |
| csio_wr_fill_fl(struct csio_hw *hw, struct csio_q *flq) | |
| { | |
| struct csio_wrm *wrm = csio_hw_to_wrm(hw); | |
| struct csio_sge *sge = &wrm->sge; | |
| __be64 *d = (__be64 *)(flq->vstart); | |
| struct csio_dma_buf *buf = &flq->un.fl.bufs[0]; | |
| uint64_t paddr; | |
| int sreg = flq->un.fl.sreg; | |
| int n = flq->credits; | |
| while (n--) { | |
| buf->len = sge->sge_fl_buf_size[sreg]; | |
| buf->vaddr = dma_alloc_coherent(&hw->pdev->dev, buf->len, | |
| &buf->paddr, GFP_KERNEL); | |
| if (!buf->vaddr) { | |
| csio_err(hw, "Could only fill %d buffers!\n", n + 1); | |
| return -ENOMEM; | |
| } | |
| paddr = buf->paddr | (sreg & 0xF); | |
| *d++ = cpu_to_be64(paddr); | |
| buf++; | |
| } | |
| return 0; | |
| } | |
| /* | |
| * csio_wr_update_fl - | |
| * @hw: HW module. | |
| * @flq: Freelist queue. | |
| * | |
| * | |
| */ | |
| static inline void | |
| csio_wr_update_fl(struct csio_hw *hw, struct csio_q *flq, uint16_t n) | |
| { | |
| flq->inc_idx += n; | |
| flq->pidx += n; | |
| if (unlikely(flq->pidx >= flq->credits)) | |
| flq->pidx -= (uint16_t)flq->credits; | |
| CSIO_INC_STATS(flq, n_flq_refill); | |
| } | |
| /* | |
| * csio_wr_alloc_q - Allocate a WR queue and initialize it. | |
| * @hw: HW module | |
| * @qsize: Size of the queue in bytes | |
| * @wrsize: Since of WR in this queue, if fixed. | |
| * @type: Type of queue (Ingress/Egress/Freelist) | |
| * @owner: Module that owns this queue. | |
| * @nflb: Number of freelist buffers for FL. | |
| * @sreg: What is the FL buffer size register? | |
| * @iq_int_handler: Ingress queue handler in INTx mode. | |
| * | |
| * This function allocates and sets up a queue for the caller | |
| * of size qsize, aligned at the required boundary. This is subject to | |
| * be free entries being available in the queue array. If one is found, | |
| * it is initialized with the allocated queue, marked as being used (owner), | |
| * and a handle returned to the caller in form of the queue's index | |
| * into the q_arr array. | |
| * If user has indicated a freelist (by specifying nflb > 0), create | |
| * another queue (with its own index into q_arr) for the freelist. Allocate | |
| * memory for DMA buffer metadata (vaddr, len etc). Save off the freelist | |
| * idx in the ingress queue's flq.idx. This is how a Freelist is associated | |
| * with its owning ingress queue. | |
| */ | |
| int | |
| csio_wr_alloc_q(struct csio_hw *hw, uint32_t qsize, uint32_t wrsize, | |
| uint16_t type, void *owner, uint32_t nflb, int sreg, | |
| iq_handler_t iq_intx_handler) | |
| { | |
| struct csio_wrm *wrm = csio_hw_to_wrm(hw); | |
| struct csio_q *q, *flq; | |
| int free_idx = wrm->free_qidx; | |
| int ret_idx = free_idx; | |
| uint32_t qsz; | |
| int flq_idx; | |
| if (free_idx >= wrm->num_q) { | |
| csio_err(hw, "No more free queues.\n"); | |
| return -1; | |
| } | |
| switch (type) { | |
| case CSIO_EGRESS: | |
| qsz = ALIGN(qsize, CSIO_QCREDIT_SZ) + csio_wr_qstat_pgsz(hw); | |
| break; | |
| case CSIO_INGRESS: | |
| switch (wrsize) { | |
| case 16: | |
| case 32: | |
| case 64: | |
| case 128: | |
| break; | |
| default: | |
| csio_err(hw, "Invalid Ingress queue WR size:%d\n", | |
| wrsize); | |
| return -1; | |
| } | |
| /* | |
| * Number of elements must be a multiple of 16 | |
| * So this includes status page size | |
| */ | |
| qsz = ALIGN(qsize/wrsize, 16) * wrsize; | |
| break; | |
| case CSIO_FREELIST: | |
| qsz = ALIGN(qsize/wrsize, 8) * wrsize + csio_wr_qstat_pgsz(hw); | |
| break; | |
| default: | |
| csio_err(hw, "Invalid queue type: 0x%x\n", type); | |
| return -1; | |
| } | |
| q = wrm->q_arr[free_idx]; | |
| q->vstart = dma_alloc_coherent(&hw->pdev->dev, qsz, &q->pstart, | |
| GFP_KERNEL); | |
| if (!q->vstart) { | |
| csio_err(hw, | |
| "Failed to allocate DMA memory for " | |
| "queue at id: %d size: %d\n", free_idx, qsize); | |
| return -1; | |
| } | |
| q->type = type; | |
| q->owner = owner; | |
| q->pidx = q->cidx = q->inc_idx = 0; | |
| q->size = qsz; | |
| q->wr_sz = wrsize; /* If using fixed size WRs */ | |
| wrm->free_qidx++; | |
| if (type == CSIO_INGRESS) { | |
| /* Since queue area is set to zero */ | |
| q->un.iq.genbit = 1; | |
| /* | |
| * Ingress queue status page size is always the size of | |
| * the ingress queue entry. | |
| */ | |
| q->credits = (qsz - q->wr_sz) / q->wr_sz; | |
| q->vwrap = (void *)((uintptr_t)(q->vstart) + qsz | |
| - q->wr_sz); | |
| /* Allocate memory for FL if requested */ | |
| if (nflb > 0) { | |
| flq_idx = csio_wr_alloc_q(hw, nflb * sizeof(__be64), | |
| sizeof(__be64), CSIO_FREELIST, | |
| owner, 0, sreg, NULL); | |
| if (flq_idx == -1) { | |
| csio_err(hw, | |
| "Failed to allocate FL queue" | |
| " for IQ idx:%d\n", free_idx); | |
| return -1; | |
| } | |
| /* Associate the new FL with the Ingress quue */ | |
| q->un.iq.flq_idx = flq_idx; | |
| flq = wrm->q_arr[q->un.iq.flq_idx]; | |
| flq->un.fl.bufs = kcalloc(flq->credits, | |
| sizeof(struct csio_dma_buf), | |
| GFP_KERNEL); | |
| if (!flq->un.fl.bufs) { | |
| csio_err(hw, | |
| "Failed to allocate FL queue bufs" | |
| " for IQ idx:%d\n", free_idx); | |
| return -1; | |
| } | |
| flq->un.fl.packen = 0; | |
| flq->un.fl.offset = 0; | |
| flq->un.fl.sreg = sreg; | |
| /* Fill up the free list buffers */ | |
| if (csio_wr_fill_fl(hw, flq)) | |
| return -1; | |
| /* | |
| * Make sure in a FLQ, atleast 1 credit (8 FL buffers) | |
| * remains unpopulated,otherwise HW thinks | |
| * FLQ is empty. | |
| */ | |
| flq->pidx = flq->inc_idx = flq->credits - 8; | |
| } else { | |
| q->un.iq.flq_idx = -1; | |
| } | |
| /* Associate the IQ INTx handler. */ | |
| q->un.iq.iq_intx_handler = iq_intx_handler; | |
| csio_q_iqid(hw, ret_idx) = CSIO_MAX_QID; | |
| } else if (type == CSIO_EGRESS) { | |
| q->credits = (qsz - csio_wr_qstat_pgsz(hw)) / CSIO_QCREDIT_SZ; | |
| q->vwrap = (void *)((uintptr_t)(q->vstart) + qsz | |
| - csio_wr_qstat_pgsz(hw)); | |
| csio_q_eqid(hw, ret_idx) = CSIO_MAX_QID; | |
| } else { /* Freelist */ | |
| q->credits = (qsz - csio_wr_qstat_pgsz(hw)) / sizeof(__be64); | |
| q->vwrap = (void *)((uintptr_t)(q->vstart) + qsz | |
| - csio_wr_qstat_pgsz(hw)); | |
| csio_q_flid(hw, ret_idx) = CSIO_MAX_QID; | |
| } | |
| return ret_idx; | |
| } | |
| /* | |
| * csio_wr_iq_create_rsp - Response handler for IQ creation. | |
| * @hw: The HW module. | |
| * @mbp: Mailbox. | |
| * @iq_idx: Ingress queue that got created. | |
| * | |
| * Handle FW_IQ_CMD mailbox completion. Save off the assigned IQ/FL ids. | |
| */ | |
| static int | |
| csio_wr_iq_create_rsp(struct csio_hw *hw, struct csio_mb *mbp, int iq_idx) | |
| { | |
| struct csio_iq_params iqp; | |
| enum fw_retval retval; | |
| uint32_t iq_id; | |
| int flq_idx; | |
| memset(&iqp, 0, sizeof(struct csio_iq_params)); | |
| csio_mb_iq_alloc_write_rsp(hw, mbp, &retval, &iqp); | |
| if (retval != FW_SUCCESS) { | |
| csio_err(hw, "IQ cmd returned 0x%x!\n", retval); | |
| mempool_free(mbp, hw->mb_mempool); | |
| return -EINVAL; | |
| } | |
| csio_q_iqid(hw, iq_idx) = iqp.iqid; | |
| csio_q_physiqid(hw, iq_idx) = iqp.physiqid; | |
| csio_q_pidx(hw, iq_idx) = csio_q_cidx(hw, iq_idx) = 0; | |
| csio_q_inc_idx(hw, iq_idx) = 0; | |
| /* Actual iq-id. */ | |
| iq_id = iqp.iqid - hw->wrm.fw_iq_start; | |
| /* Set the iq-id to iq map table. */ | |
| if (iq_id >= CSIO_MAX_IQ) { | |
| csio_err(hw, | |
| "Exceeding MAX_IQ(%d) supported!" | |
| " iqid:%d rel_iqid:%d FW iq_start:%d\n", | |
| CSIO_MAX_IQ, iq_id, iqp.iqid, hw->wrm.fw_iq_start); | |
| mempool_free(mbp, hw->mb_mempool); | |
| return -EINVAL; | |
| } | |
| csio_q_set_intr_map(hw, iq_idx, iq_id); | |
| /* | |
| * During FW_IQ_CMD, FW sets interrupt_sent bit to 1 in the SGE | |
| * ingress context of this queue. This will block interrupts to | |
| * this queue until the next GTS write. Therefore, we do a | |
| * 0-cidx increment GTS write for this queue just to clear the | |
| * interrupt_sent bit. This will re-enable interrupts to this | |
| * queue. | |
| */ | |
| csio_wr_sge_intr_enable(hw, iqp.physiqid); | |
| flq_idx = csio_q_iq_flq_idx(hw, iq_idx); | |
| if (flq_idx != -1) { | |
| struct csio_q *flq = hw->wrm.q_arr[flq_idx]; | |
| csio_q_flid(hw, flq_idx) = iqp.fl0id; | |
| csio_q_cidx(hw, flq_idx) = 0; | |
| csio_q_pidx(hw, flq_idx) = csio_q_credits(hw, flq_idx) - 8; | |
| csio_q_inc_idx(hw, flq_idx) = csio_q_credits(hw, flq_idx) - 8; | |
| /* Now update SGE about the buffers allocated during init */ | |
| csio_wr_ring_fldb(hw, flq); | |
| } | |
| mempool_free(mbp, hw->mb_mempool); | |
| return 0; | |
| } | |
| /* | |
| * csio_wr_iq_create - Configure an Ingress queue with FW. | |
| * @hw: The HW module. | |
| * @priv: Private data object. | |
| * @iq_idx: Ingress queue index in the WR module. | |
| * @vec: MSIX vector. | |
| * @portid: PCIE Channel to be associated with this queue. | |
| * @async: Is this a FW asynchronous message handling queue? | |
| * @cbfn: Completion callback. | |
| * | |
| * This API configures an ingress queue with FW by issuing a FW_IQ_CMD mailbox | |
| * with alloc/write bits set. | |
| */ | |
| int | |
| csio_wr_iq_create(struct csio_hw *hw, void *priv, int iq_idx, | |
| uint32_t vec, uint8_t portid, bool async, | |
| void (*cbfn) (struct csio_hw *, struct csio_mb *)) | |
| { | |
| struct csio_mb *mbp; | |
| struct csio_iq_params iqp; | |
| int flq_idx; | |
| memset(&iqp, 0, sizeof(struct csio_iq_params)); | |
| csio_q_portid(hw, iq_idx) = portid; | |
| mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC); | |
| if (!mbp) { | |
| csio_err(hw, "IQ command out of memory!\n"); | |
| return -ENOMEM; | |
| } | |
| switch (hw->intr_mode) { | |
| case CSIO_IM_INTX: | |
| case CSIO_IM_MSI: | |
| /* For interrupt forwarding queue only */ | |
| if (hw->intr_iq_idx == iq_idx) | |
| iqp.iqandst = X_INTERRUPTDESTINATION_PCIE; | |
| else | |
| iqp.iqandst = X_INTERRUPTDESTINATION_IQ; | |
| iqp.iqandstindex = | |
| csio_q_physiqid(hw, hw->intr_iq_idx); | |
| break; | |
| case CSIO_IM_MSIX: | |
| iqp.iqandst = X_INTERRUPTDESTINATION_PCIE; | |
| iqp.iqandstindex = (uint16_t)vec; | |
| break; | |
| case CSIO_IM_NONE: | |
| mempool_free(mbp, hw->mb_mempool); | |
| return -EINVAL; | |
| } | |
| /* Pass in the ingress queue cmd parameters */ | |
| iqp.pfn = hw->pfn; | |
| iqp.vfn = 0; | |
| iqp.iq_start = 1; | |
| iqp.viid = 0; | |
| iqp.type = FW_IQ_TYPE_FL_INT_CAP; | |
| iqp.iqasynch = async; | |
| if (csio_intr_coalesce_cnt) | |
| iqp.iqanus = X_UPDATESCHEDULING_COUNTER_OPTTIMER; | |
| else | |
| iqp.iqanus = X_UPDATESCHEDULING_TIMER; | |
| iqp.iqanud = X_UPDATEDELIVERY_INTERRUPT; | |
| iqp.iqpciech = portid; | |
| iqp.iqintcntthresh = (uint8_t)csio_sge_thresh_reg; | |
| switch (csio_q_wr_sz(hw, iq_idx)) { | |
| case 16: | |
| iqp.iqesize = 0; break; | |
| case 32: | |
| iqp.iqesize = 1; break; | |
| case 64: | |
| iqp.iqesize = 2; break; | |
| case 128: | |
| iqp.iqesize = 3; break; | |
| } | |
| iqp.iqsize = csio_q_size(hw, iq_idx) / | |
| csio_q_wr_sz(hw, iq_idx); | |
| iqp.iqaddr = csio_q_pstart(hw, iq_idx); | |
| flq_idx = csio_q_iq_flq_idx(hw, iq_idx); | |
| if (flq_idx != -1) { | |
| enum chip_type chip = CHELSIO_CHIP_VERSION(hw->chip_id); | |
| struct csio_q *flq = hw->wrm.q_arr[flq_idx]; | |
| iqp.fl0paden = 1; | |
| iqp.fl0packen = flq->un.fl.packen ? 1 : 0; | |
| iqp.fl0fbmin = X_FETCHBURSTMIN_64B; | |
| iqp.fl0fbmax = ((chip == CHELSIO_T5) ? | |
| X_FETCHBURSTMAX_512B : X_FETCHBURSTMAX_256B); | |
| iqp.fl0size = csio_q_size(hw, flq_idx) / CSIO_QCREDIT_SZ; | |
| iqp.fl0addr = csio_q_pstart(hw, flq_idx); | |
| } | |
| csio_mb_iq_alloc_write(hw, mbp, priv, CSIO_MB_DEFAULT_TMO, &iqp, cbfn); | |
| if (csio_mb_issue(hw, mbp)) { | |
| csio_err(hw, "Issue of IQ cmd failed!\n"); | |
| mempool_free(mbp, hw->mb_mempool); | |
| return -EINVAL; | |
| } | |
| if (cbfn != NULL) | |
| return 0; | |
| return csio_wr_iq_create_rsp(hw, mbp, iq_idx); | |
| } | |
| /* | |
| * csio_wr_eq_create_rsp - Response handler for EQ creation. | |
| * @hw: The HW module. | |
| * @mbp: Mailbox. | |
| * @eq_idx: Egress queue that got created. | |
| * | |
| * Handle FW_EQ_OFLD_CMD mailbox completion. Save off the assigned EQ ids. | |
| */ | |
| static int | |
| csio_wr_eq_cfg_rsp(struct csio_hw *hw, struct csio_mb *mbp, int eq_idx) | |
| { | |
| struct csio_eq_params eqp; | |
| enum fw_retval retval; | |
| memset(&eqp, 0, sizeof(struct csio_eq_params)); | |
| csio_mb_eq_ofld_alloc_write_rsp(hw, mbp, &retval, &eqp); | |
| if (retval != FW_SUCCESS) { | |
| csio_err(hw, "EQ OFLD cmd returned 0x%x!\n", retval); | |
| mempool_free(mbp, hw->mb_mempool); | |
| return -EINVAL; | |
| } | |
| csio_q_eqid(hw, eq_idx) = (uint16_t)eqp.eqid; | |
| csio_q_physeqid(hw, eq_idx) = (uint16_t)eqp.physeqid; | |
| csio_q_pidx(hw, eq_idx) = csio_q_cidx(hw, eq_idx) = 0; | |
| csio_q_inc_idx(hw, eq_idx) = 0; | |
| mempool_free(mbp, hw->mb_mempool); | |
| return 0; | |
| } | |
| /* | |
| * csio_wr_eq_create - Configure an Egress queue with FW. | |
| * @hw: HW module. | |
| * @priv: Private data. | |
| * @eq_idx: Egress queue index in the WR module. | |
| * @iq_idx: Associated ingress queue index. | |
| * @cbfn: Completion callback. | |
| * | |
| * This API configures a offload egress queue with FW by issuing a | |
| * FW_EQ_OFLD_CMD (with alloc + write ) mailbox. | |
| */ | |
| int | |
| csio_wr_eq_create(struct csio_hw *hw, void *priv, int eq_idx, | |
| int iq_idx, uint8_t portid, | |
| void (*cbfn) (struct csio_hw *, struct csio_mb *)) | |
| { | |
| struct csio_mb *mbp; | |
| struct csio_eq_params eqp; | |
| memset(&eqp, 0, sizeof(struct csio_eq_params)); | |
| mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC); | |
| if (!mbp) { | |
| csio_err(hw, "EQ command out of memory!\n"); | |
| return -ENOMEM; | |
| } | |
| eqp.pfn = hw->pfn; | |
| eqp.vfn = 0; | |
| eqp.eqstart = 1; | |
| eqp.hostfcmode = X_HOSTFCMODE_STATUS_PAGE; | |
| eqp.iqid = csio_q_iqid(hw, iq_idx); | |
| eqp.fbmin = X_FETCHBURSTMIN_64B; | |
| eqp.fbmax = X_FETCHBURSTMAX_512B; | |
| eqp.cidxfthresh = 0; | |
| eqp.pciechn = portid; | |
| eqp.eqsize = csio_q_size(hw, eq_idx) / CSIO_QCREDIT_SZ; | |
| eqp.eqaddr = csio_q_pstart(hw, eq_idx); | |
| csio_mb_eq_ofld_alloc_write(hw, mbp, priv, CSIO_MB_DEFAULT_TMO, | |
| &eqp, cbfn); | |
| if (csio_mb_issue(hw, mbp)) { | |
| csio_err(hw, "Issue of EQ OFLD cmd failed!\n"); | |
| mempool_free(mbp, hw->mb_mempool); | |
| return -EINVAL; | |
| } | |
| if (cbfn != NULL) | |
| return 0; | |
| return csio_wr_eq_cfg_rsp(hw, mbp, eq_idx); | |
| } | |
| /* | |
| * csio_wr_iq_destroy_rsp - Response handler for IQ removal. | |
| * @hw: The HW module. | |
| * @mbp: Mailbox. | |
| * @iq_idx: Ingress queue that was freed. | |
| * | |
| * Handle FW_IQ_CMD (free) mailbox completion. | |
| */ | |
| static int | |
| csio_wr_iq_destroy_rsp(struct csio_hw *hw, struct csio_mb *mbp, int iq_idx) | |
| { | |
| enum fw_retval retval = csio_mb_fw_retval(mbp); | |
| int rv = 0; | |
| if (retval != FW_SUCCESS) | |
| rv = -EINVAL; | |
| mempool_free(mbp, hw->mb_mempool); | |
| return rv; | |
| } | |
| /* | |
| * csio_wr_iq_destroy - Free an ingress queue. | |
| * @hw: The HW module. | |
| * @priv: Private data object. | |
| * @iq_idx: Ingress queue index to destroy | |
| * @cbfn: Completion callback. | |
| * | |
| * This API frees an ingress queue by issuing the FW_IQ_CMD | |
| * with the free bit set. | |
| */ | |
| static int | |
| csio_wr_iq_destroy(struct csio_hw *hw, void *priv, int iq_idx, | |
| void (*cbfn)(struct csio_hw *, struct csio_mb *)) | |
| { | |
| int rv = 0; | |
| struct csio_mb *mbp; | |
| struct csio_iq_params iqp; | |
| int flq_idx; | |
| memset(&iqp, 0, sizeof(struct csio_iq_params)); | |
| mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC); | |
| if (!mbp) | |
| return -ENOMEM; | |
| iqp.pfn = hw->pfn; | |
| iqp.vfn = 0; | |
| iqp.iqid = csio_q_iqid(hw, iq_idx); | |
| iqp.type = FW_IQ_TYPE_FL_INT_CAP; | |
| flq_idx = csio_q_iq_flq_idx(hw, iq_idx); | |
| if (flq_idx != -1) | |
| iqp.fl0id = csio_q_flid(hw, flq_idx); | |
| else | |
| iqp.fl0id = 0xFFFF; | |
| iqp.fl1id = 0xFFFF; | |
| csio_mb_iq_free(hw, mbp, priv, CSIO_MB_DEFAULT_TMO, &iqp, cbfn); | |
| rv = csio_mb_issue(hw, mbp); | |
| if (rv != 0) { | |
| mempool_free(mbp, hw->mb_mempool); | |
| return rv; | |
| } | |
| if (cbfn != NULL) | |
| return 0; | |
| return csio_wr_iq_destroy_rsp(hw, mbp, iq_idx); | |
| } | |
| /* | |
| * csio_wr_eq_destroy_rsp - Response handler for OFLD EQ creation. | |
| * @hw: The HW module. | |
| * @mbp: Mailbox. | |
| * @eq_idx: Egress queue that was freed. | |
| * | |
| * Handle FW_OFLD_EQ_CMD (free) mailbox completion. | |
| */ | |
| static int | |
| csio_wr_eq_destroy_rsp(struct csio_hw *hw, struct csio_mb *mbp, int eq_idx) | |
| { | |
| enum fw_retval retval = csio_mb_fw_retval(mbp); | |
| int rv = 0; | |
| if (retval != FW_SUCCESS) | |
| rv = -EINVAL; | |
| mempool_free(mbp, hw->mb_mempool); | |
| return rv; | |
| } | |
| /* | |
| * csio_wr_eq_destroy - Free an Egress queue. | |
| * @hw: The HW module. | |
| * @priv: Private data object. | |
| * @eq_idx: Egress queue index to destroy | |
| * @cbfn: Completion callback. | |
| * | |
| * This API frees an Egress queue by issuing the FW_EQ_OFLD_CMD | |
| * with the free bit set. | |
| */ | |
| static int | |
| csio_wr_eq_destroy(struct csio_hw *hw, void *priv, int eq_idx, | |
| void (*cbfn) (struct csio_hw *, struct csio_mb *)) | |
| { | |
| int rv = 0; | |
| struct csio_mb *mbp; | |
| struct csio_eq_params eqp; | |
| memset(&eqp, 0, sizeof(struct csio_eq_params)); | |
| mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC); | |
| if (!mbp) | |
| return -ENOMEM; | |
| eqp.pfn = hw->pfn; | |
| eqp.vfn = 0; | |
| eqp.eqid = csio_q_eqid(hw, eq_idx); | |
| csio_mb_eq_ofld_free(hw, mbp, priv, CSIO_MB_DEFAULT_TMO, &eqp, cbfn); | |
| rv = csio_mb_issue(hw, mbp); | |
| if (rv != 0) { | |
| mempool_free(mbp, hw->mb_mempool); | |
| return rv; | |
| } | |
| if (cbfn != NULL) | |
| return 0; | |
| return csio_wr_eq_destroy_rsp(hw, mbp, eq_idx); | |
| } | |
| /* | |
| * csio_wr_cleanup_eq_stpg - Cleanup Egress queue status page | |
| * @hw: HW module | |
| * @qidx: Egress queue index | |
| * | |
| * Cleanup the Egress queue status page. | |
| */ | |
| static void | |
| csio_wr_cleanup_eq_stpg(struct csio_hw *hw, int qidx) | |
| { | |
| struct csio_q *q = csio_hw_to_wrm(hw)->q_arr[qidx]; | |
| struct csio_qstatus_page *stp = (struct csio_qstatus_page *)q->vwrap; | |
| memset(stp, 0, sizeof(*stp)); | |
| } | |
| /* | |
| * csio_wr_cleanup_iq_ftr - Cleanup Footer entries in IQ | |
| * @hw: HW module | |
| * @qidx: Ingress queue index | |
| * | |
| * Cleanup the footer entries in the given ingress queue, | |
| * set to 1 the internal copy of genbit. | |
| */ | |
| static void | |
| csio_wr_cleanup_iq_ftr(struct csio_hw *hw, int qidx) | |
| { | |
| struct csio_wrm *wrm = csio_hw_to_wrm(hw); | |
| struct csio_q *q = wrm->q_arr[qidx]; | |
| void *wr; | |
| struct csio_iqwr_footer *ftr; | |
| uint32_t i = 0; | |
| /* set to 1 since we are just about zero out genbit */ | |
| q->un.iq.genbit = 1; | |
| for (i = 0; i < q->credits; i++) { | |
| /* Get the WR */ | |
| wr = (void *)((uintptr_t)q->vstart + | |
| (i * q->wr_sz)); | |
| /* Get the footer */ | |
| ftr = (struct csio_iqwr_footer *)((uintptr_t)wr + | |
| (q->wr_sz - sizeof(*ftr))); | |
| /* Zero out footer */ | |
| memset(ftr, 0, sizeof(*ftr)); | |
| } | |
| } | |
| int | |
| csio_wr_destroy_queues(struct csio_hw *hw, bool cmd) | |
| { | |
| int i, flq_idx; | |
| struct csio_q *q; | |
| struct csio_wrm *wrm = csio_hw_to_wrm(hw); | |
| int rv; | |
| for (i = 0; i < wrm->free_qidx; i++) { | |
| q = wrm->q_arr[i]; | |
| switch (q->type) { | |
| case CSIO_EGRESS: | |
| if (csio_q_eqid(hw, i) != CSIO_MAX_QID) { | |
| csio_wr_cleanup_eq_stpg(hw, i); | |
| if (!cmd) { | |
| csio_q_eqid(hw, i) = CSIO_MAX_QID; | |
| continue; | |
| } | |
| rv = csio_wr_eq_destroy(hw, NULL, i, NULL); | |
| if ((rv == -EBUSY) || (rv == -ETIMEDOUT)) | |
| cmd = false; | |
| csio_q_eqid(hw, i) = CSIO_MAX_QID; | |
| } | |
| fallthrough; | |
| case CSIO_INGRESS: | |
| if (csio_q_iqid(hw, i) != CSIO_MAX_QID) { | |
| csio_wr_cleanup_iq_ftr(hw, i); | |
| if (!cmd) { | |
| csio_q_iqid(hw, i) = CSIO_MAX_QID; | |
| flq_idx = csio_q_iq_flq_idx(hw, i); | |
| if (flq_idx != -1) | |
| csio_q_flid(hw, flq_idx) = | |
| CSIO_MAX_QID; | |
| continue; | |
| } | |
| rv = csio_wr_iq_destroy(hw, NULL, i, NULL); | |
| if ((rv == -EBUSY) || (rv == -ETIMEDOUT)) | |
| cmd = false; | |
| csio_q_iqid(hw, i) = CSIO_MAX_QID; | |
| flq_idx = csio_q_iq_flq_idx(hw, i); | |
| if (flq_idx != -1) | |
| csio_q_flid(hw, flq_idx) = CSIO_MAX_QID; | |
| } | |
| break; | |
| default: | |
| break; | |
| } | |
| } | |
| hw->flags &= ~CSIO_HWF_Q_FW_ALLOCED; | |
| return 0; | |
| } | |
| /* | |
| * csio_wr_get - Get requested size of WR entry/entries from queue. | |
| * @hw: HW module. | |
| * @qidx: Index of queue. | |
| * @size: Cumulative size of Work request(s). | |
| * @wrp: Work request pair. | |
| * | |
| * If requested credits are available, return the start address of the | |
| * work request in the work request pair. Set pidx accordingly and | |
| * return. | |
| * | |
| * NOTE about WR pair: | |
| * ================== | |
| * A WR can start towards the end of a queue, and then continue at the | |
| * beginning, since the queue is considered to be circular. This will | |
| * require a pair of address/size to be passed back to the caller - | |
| * hence Work request pair format. | |
| */ | |
| int | |
| csio_wr_get(struct csio_hw *hw, int qidx, uint32_t size, | |
| struct csio_wr_pair *wrp) | |
| { | |
| struct csio_wrm *wrm = csio_hw_to_wrm(hw); | |
| struct csio_q *q = wrm->q_arr[qidx]; | |
| void *cwr = (void *)((uintptr_t)(q->vstart) + | |
| (q->pidx * CSIO_QCREDIT_SZ)); | |
| struct csio_qstatus_page *stp = (struct csio_qstatus_page *)q->vwrap; | |
| uint16_t cidx = q->cidx = ntohs(stp->cidx); | |
| uint16_t pidx = q->pidx; | |
| uint32_t req_sz = ALIGN(size, CSIO_QCREDIT_SZ); | |
| int req_credits = req_sz / CSIO_QCREDIT_SZ; | |
| int credits; | |
| CSIO_DB_ASSERT(q->owner != NULL); | |
| CSIO_DB_ASSERT((qidx >= 0) && (qidx < wrm->free_qidx)); | |
| CSIO_DB_ASSERT(cidx <= q->credits); | |
| /* Calculate credits */ | |
| if (pidx > cidx) { | |
| credits = q->credits - (pidx - cidx) - 1; | |
| } else if (cidx > pidx) { | |
| credits = cidx - pidx - 1; | |
| } else { | |
| /* cidx == pidx, empty queue */ | |
| credits = q->credits; | |
| CSIO_INC_STATS(q, n_qempty); | |
| } | |
| /* | |
| * Check if we have enough credits. | |
| * credits = 1 implies queue is full. | |
| */ | |
| if (!credits || (req_credits > credits)) { | |
| CSIO_INC_STATS(q, n_qfull); | |
| return -EBUSY; | |
| } | |
| /* | |
| * If we are here, we have enough credits to satisfy the | |
| * request. Check if we are near the end of q, and if WR spills over. | |
| * If it does, use the first addr/size to cover the queue until | |
| * the end. Fit the remainder portion of the request at the top | |
| * of queue and return it in the second addr/len. Set pidx | |
| * accordingly. | |
| */ | |
| if (unlikely(((uintptr_t)cwr + req_sz) > (uintptr_t)(q->vwrap))) { | |
| wrp->addr1 = cwr; | |
| wrp->size1 = (uint32_t)((uintptr_t)q->vwrap - (uintptr_t)cwr); | |
| wrp->addr2 = q->vstart; | |
| wrp->size2 = req_sz - wrp->size1; | |
| q->pidx = (uint16_t)(ALIGN(wrp->size2, CSIO_QCREDIT_SZ) / | |
| CSIO_QCREDIT_SZ); | |
| CSIO_INC_STATS(q, n_qwrap); | |
| CSIO_INC_STATS(q, n_eq_wr_split); | |
| } else { | |
| wrp->addr1 = cwr; | |
| wrp->size1 = req_sz; | |
| wrp->addr2 = NULL; | |
| wrp->size2 = 0; | |
| q->pidx += (uint16_t)req_credits; | |
| /* We are the end of queue, roll back pidx to top of queue */ | |
| if (unlikely(q->pidx == q->credits)) { | |
| q->pidx = 0; | |
| CSIO_INC_STATS(q, n_qwrap); | |
| } | |
| } | |
| q->inc_idx = (uint16_t)req_credits; | |
| CSIO_INC_STATS(q, n_tot_reqs); | |
| return 0; | |
| } | |
| /* | |
| * csio_wr_copy_to_wrp - Copies given data into WR. | |
| * @data_buf - Data buffer | |
| * @wrp - Work request pair. | |
| * @wr_off - Work request offset. | |
| * @data_len - Data length. | |
| * | |
| * Copies the given data in Work Request. Work request pair(wrp) specifies | |
| * address information of Work request. | |
| * Returns: none | |
| */ | |
| void | |
| csio_wr_copy_to_wrp(void *data_buf, struct csio_wr_pair *wrp, | |
| uint32_t wr_off, uint32_t data_len) | |
| { | |
| uint32_t nbytes; | |
| /* Number of space available in buffer addr1 of WRP */ | |
| nbytes = ((wrp->size1 - wr_off) >= data_len) ? | |
| data_len : (wrp->size1 - wr_off); | |
| memcpy((uint8_t *) wrp->addr1 + wr_off, data_buf, nbytes); | |
| data_len -= nbytes; | |
| /* Write the remaining data from the begining of circular buffer */ | |
| if (data_len) { | |
| CSIO_DB_ASSERT(data_len <= wrp->size2); | |
| CSIO_DB_ASSERT(wrp->addr2 != NULL); | |
| memcpy(wrp->addr2, (uint8_t *) data_buf + nbytes, data_len); | |
| } | |
| } | |
| /* | |
| * csio_wr_issue - Notify chip of Work request. | |
| * @hw: HW module. | |
| * @qidx: Index of queue. | |
| * @prio: 0: Low priority, 1: High priority | |
| * | |
| * Rings the SGE Doorbell by writing the current producer index of the passed | |
| * in queue into the register. | |
| * | |
| */ | |
| int | |
| csio_wr_issue(struct csio_hw *hw, int qidx, bool prio) | |
| { | |
| struct csio_wrm *wrm = csio_hw_to_wrm(hw); | |
| struct csio_q *q = wrm->q_arr[qidx]; | |
| CSIO_DB_ASSERT((qidx >= 0) && (qidx < wrm->free_qidx)); | |
| wmb(); | |
| /* Ring SGE Doorbell writing q->pidx into it */ | |
| csio_wr_reg32(hw, DBPRIO_V(prio) | QID_V(q->un.eq.physeqid) | | |
| PIDX_T5_V(q->inc_idx) | DBTYPE_F, | |
| MYPF_REG(SGE_PF_KDOORBELL_A)); | |
| q->inc_idx = 0; | |
| return 0; | |
| } | |
| static inline uint32_t | |
| csio_wr_avail_qcredits(struct csio_q *q) | |
| { | |
| if (q->pidx > q->cidx) | |
| return q->pidx - q->cidx; | |
| else if (q->cidx > q->pidx) | |
| return q->credits - (q->cidx - q->pidx); | |
| else | |
| return 0; /* cidx == pidx, empty queue */ | |
| } | |
| /* | |
| * csio_wr_inval_flq_buf - Invalidate a free list buffer entry. | |
| * @hw: HW module. | |
| * @flq: The freelist queue. | |
| * | |
| * Invalidate the driver's version of a freelist buffer entry, | |
| * without freeing the associated the DMA memory. The entry | |
| * to be invalidated is picked up from the current Free list | |
| * queue cidx. | |
| * | |
| */ | |
| static inline void | |
| csio_wr_inval_flq_buf(struct csio_hw *hw, struct csio_q *flq) | |
| { | |
| flq->cidx++; | |
| if (flq->cidx == flq->credits) { | |
| flq->cidx = 0; | |
| CSIO_INC_STATS(flq, n_qwrap); | |
| } | |
| } | |
| /* | |
| * csio_wr_process_fl - Process a freelist completion. | |
| * @hw: HW module. | |
| * @q: The ingress queue attached to the Freelist. | |
| * @wr: The freelist completion WR in the ingress queue. | |
| * @len_to_qid: The lower 32-bits of the first flit of the RSP footer | |
| * @iq_handler: Caller's handler for this completion. | |
| * @priv: Private pointer of caller | |
| * | |
| */ | |
| static inline void | |
| csio_wr_process_fl(struct csio_hw *hw, struct csio_q *q, | |
| void *wr, uint32_t len_to_qid, | |
| void (*iq_handler)(struct csio_hw *, void *, | |
| uint32_t, struct csio_fl_dma_buf *, | |
| void *), | |
| void *priv) | |
| { | |
| struct csio_wrm *wrm = csio_hw_to_wrm(hw); | |
| struct csio_sge *sge = &wrm->sge; | |
| struct csio_fl_dma_buf flb; | |
| struct csio_dma_buf *buf, *fbuf; | |
| uint32_t bufsz, len, lastlen = 0; | |
| int n; | |
| struct csio_q *flq = hw->wrm.q_arr[q->un.iq.flq_idx]; | |
| CSIO_DB_ASSERT(flq != NULL); | |
| len = len_to_qid; | |
| if (len & IQWRF_NEWBUF) { | |
| if (flq->un.fl.offset > 0) { | |
| csio_wr_inval_flq_buf(hw, flq); | |
| flq->un.fl.offset = 0; | |
| } | |
| len = IQWRF_LEN_GET(len); | |
| } | |
| CSIO_DB_ASSERT(len != 0); | |
| flb.totlen = len; | |
| /* Consume all freelist buffers used for len bytes */ | |
| for (n = 0, fbuf = flb.flbufs; ; n++, fbuf++) { | |
| buf = &flq->un.fl.bufs[flq->cidx]; | |
| bufsz = csio_wr_fl_bufsz(sge, buf); | |
| fbuf->paddr = buf->paddr; | |
| fbuf->vaddr = buf->vaddr; | |
| flb.offset = flq->un.fl.offset; | |
| lastlen = min(bufsz, len); | |
| fbuf->len = lastlen; | |
| len -= lastlen; | |
| if (!len) | |
| break; | |
| csio_wr_inval_flq_buf(hw, flq); | |
| } | |
| flb.defer_free = flq->un.fl.packen ? 0 : 1; | |
| iq_handler(hw, wr, q->wr_sz - sizeof(struct csio_iqwr_footer), | |
| &flb, priv); | |
| if (flq->un.fl.packen) | |
| flq->un.fl.offset += ALIGN(lastlen, sge->csio_fl_align); | |
| else | |
| csio_wr_inval_flq_buf(hw, flq); | |
| } | |
| /* | |
| * csio_is_new_iqwr - Is this a new Ingress queue entry ? | |
| * @q: Ingress quueue. | |
| * @ftr: Ingress queue WR SGE footer. | |
| * | |
| * The entry is new if our generation bit matches the corresponding | |
| * bit in the footer of the current WR. | |
| */ | |
| static inline bool | |
| csio_is_new_iqwr(struct csio_q *q, struct csio_iqwr_footer *ftr) | |
| { | |
| return (q->un.iq.genbit == (ftr->u.type_gen >> IQWRF_GEN_SHIFT)); | |
| } | |
| /* | |
| * csio_wr_process_iq - Process elements in Ingress queue. | |
| * @hw: HW pointer | |
| * @qidx: Index of queue | |
| * @iq_handler: Handler for this queue | |
| * @priv: Caller's private pointer | |
| * | |
| * This routine walks through every entry of the ingress queue, calling | |
| * the provided iq_handler with the entry, until the generation bit | |
| * flips. | |
| */ | |
| int | |
| csio_wr_process_iq(struct csio_hw *hw, struct csio_q *q, | |
| void (*iq_handler)(struct csio_hw *, void *, | |
| uint32_t, struct csio_fl_dma_buf *, | |
| void *), | |
| void *priv) | |
| { | |
| struct csio_wrm *wrm = csio_hw_to_wrm(hw); | |
| void *wr = (void *)((uintptr_t)q->vstart + (q->cidx * q->wr_sz)); | |
| struct csio_iqwr_footer *ftr; | |
| uint32_t wr_type, fw_qid, qid; | |
| struct csio_q *q_completed; | |
| struct csio_q *flq = csio_iq_has_fl(q) ? | |
| wrm->q_arr[q->un.iq.flq_idx] : NULL; | |
| int rv = 0; | |
| /* Get the footer */ | |
| ftr = (struct csio_iqwr_footer *)((uintptr_t)wr + | |
| (q->wr_sz - sizeof(*ftr))); | |
| /* | |
| * When q wrapped around last time, driver should have inverted | |
| * ic.genbit as well. | |
| */ | |
| while (csio_is_new_iqwr(q, ftr)) { | |
| CSIO_DB_ASSERT(((uintptr_t)wr + q->wr_sz) <= | |
| (uintptr_t)q->vwrap); | |
| rmb(); | |
| wr_type = IQWRF_TYPE_GET(ftr->u.type_gen); | |
| switch (wr_type) { | |
| case X_RSPD_TYPE_CPL: | |
| /* Subtract footer from WR len */ | |
| iq_handler(hw, wr, q->wr_sz - sizeof(*ftr), NULL, priv); | |
| break; | |
| case X_RSPD_TYPE_FLBUF: | |
| csio_wr_process_fl(hw, q, wr, | |
| ntohl(ftr->pldbuflen_qid), | |
| iq_handler, priv); | |
| break; | |
| case X_RSPD_TYPE_INTR: | |
| fw_qid = ntohl(ftr->pldbuflen_qid); | |
| qid = fw_qid - wrm->fw_iq_start; | |
| q_completed = hw->wrm.intr_map[qid]; | |
| if (unlikely(qid == | |
| csio_q_physiqid(hw, hw->intr_iq_idx))) { | |
| /* | |
| * We are already in the Forward Interrupt | |
| * Interrupt Queue Service! Do-not service | |
| * again! | |
| * | |
| */ | |
| } else { | |
| CSIO_DB_ASSERT(q_completed); | |
| CSIO_DB_ASSERT( | |
| q_completed->un.iq.iq_intx_handler); | |
| /* Call the queue handler. */ | |
| q_completed->un.iq.iq_intx_handler(hw, NULL, | |
| 0, NULL, (void *)q_completed); | |
| } | |
| break; | |
| default: | |
| csio_warn(hw, "Unknown resp type 0x%x received\n", | |
| wr_type); | |
| CSIO_INC_STATS(q, n_rsp_unknown); | |
| break; | |
| } | |
| /* | |
| * Ingress *always* has fixed size WR entries. Therefore, | |
| * there should always be complete WRs towards the end of | |
| * queue. | |
| */ | |
| if (((uintptr_t)wr + q->wr_sz) == (uintptr_t)q->vwrap) { | |
| /* Roll over to start of queue */ | |
| q->cidx = 0; | |
| wr = q->vstart; | |
| /* Toggle genbit */ | |
| q->un.iq.genbit ^= 0x1; | |
| CSIO_INC_STATS(q, n_qwrap); | |
| } else { | |
| q->cidx++; | |
| wr = (void *)((uintptr_t)(q->vstart) + | |
| (q->cidx * q->wr_sz)); | |
| } | |
| ftr = (struct csio_iqwr_footer *)((uintptr_t)wr + | |
| (q->wr_sz - sizeof(*ftr))); | |
| q->inc_idx++; | |
| } /* while (q->un.iq.genbit == hdr->genbit) */ | |
| /* | |
| * We need to re-arm SGE interrupts in case we got a stray interrupt, | |
| * especially in msix mode. With INTx, this may be a common occurence. | |
| */ | |
| if (unlikely(!q->inc_idx)) { | |
| CSIO_INC_STATS(q, n_stray_comp); | |
| rv = -EINVAL; | |
| goto restart; | |
| } | |
| /* Replenish free list buffers if pending falls below low water mark */ | |
| if (flq) { | |
| uint32_t avail = csio_wr_avail_qcredits(flq); | |
| if (avail <= 16) { | |
| /* Make sure in FLQ, atleast 1 credit (8 FL buffers) | |
| * remains unpopulated otherwise HW thinks | |
| * FLQ is empty. | |
| */ | |
| csio_wr_update_fl(hw, flq, (flq->credits - 8) - avail); | |
| csio_wr_ring_fldb(hw, flq); | |
| } | |
| } | |
| restart: | |
| /* Now inform SGE about our incremental index value */ | |
| csio_wr_reg32(hw, CIDXINC_V(q->inc_idx) | | |
| INGRESSQID_V(q->un.iq.physiqid) | | |
| TIMERREG_V(csio_sge_timer_reg), | |
| MYPF_REG(SGE_PF_GTS_A)); | |
| q->stats.n_tot_rsps += q->inc_idx; | |
| q->inc_idx = 0; | |
| return rv; | |
| } | |
| int | |
| csio_wr_process_iq_idx(struct csio_hw *hw, int qidx, | |
| void (*iq_handler)(struct csio_hw *, void *, | |
| uint32_t, struct csio_fl_dma_buf *, | |
| void *), | |
| void *priv) | |
| { | |
| struct csio_wrm *wrm = csio_hw_to_wrm(hw); | |
| struct csio_q *iq = wrm->q_arr[qidx]; | |
| return csio_wr_process_iq(hw, iq, iq_handler, priv); | |
| } | |
| static int | |
| csio_closest_timer(struct csio_sge *s, int time) | |
| { | |
| int i, delta, match = 0, min_delta = INT_MAX; | |
| for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) { | |
| delta = time - s->timer_val[i]; | |
| if (delta < 0) | |
| delta = -delta; | |
| if (delta < min_delta) { | |
| min_delta = delta; | |
| match = i; | |
| } | |
| } | |
| return match; | |
| } | |
| static int | |
| csio_closest_thresh(struct csio_sge *s, int cnt) | |
| { | |
| int i, delta, match = 0, min_delta = INT_MAX; | |
| for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) { | |
| delta = cnt - s->counter_val[i]; | |
| if (delta < 0) | |
| delta = -delta; | |
| if (delta < min_delta) { | |
| min_delta = delta; | |
| match = i; | |
| } | |
| } | |
| return match; | |
| } | |
| static void | |
| csio_wr_fixup_host_params(struct csio_hw *hw) | |
| { | |
| struct csio_wrm *wrm = csio_hw_to_wrm(hw); | |
| struct csio_sge *sge = &wrm->sge; | |
| uint32_t clsz = L1_CACHE_BYTES; | |
| uint32_t s_hps = PAGE_SHIFT - 10; | |
| uint32_t stat_len = clsz > 64 ? 128 : 64; | |
| u32 fl_align = clsz < 32 ? 32 : clsz; | |
| u32 pack_align; | |
| u32 ingpad, ingpack; | |
| csio_wr_reg32(hw, HOSTPAGESIZEPF0_V(s_hps) | HOSTPAGESIZEPF1_V(s_hps) | | |
| HOSTPAGESIZEPF2_V(s_hps) | HOSTPAGESIZEPF3_V(s_hps) | | |
| HOSTPAGESIZEPF4_V(s_hps) | HOSTPAGESIZEPF5_V(s_hps) | | |
| HOSTPAGESIZEPF6_V(s_hps) | HOSTPAGESIZEPF7_V(s_hps), | |
| SGE_HOST_PAGE_SIZE_A); | |
| /* T5 introduced the separation of the Free List Padding and | |
| * Packing Boundaries. Thus, we can select a smaller Padding | |
| * Boundary to avoid uselessly chewing up PCIe Link and Memory | |
| * Bandwidth, and use a Packing Boundary which is large enough | |
| * to avoid false sharing between CPUs, etc. | |
| * | |
| * For the PCI Link, the smaller the Padding Boundary the | |
| * better. For the Memory Controller, a smaller Padding | |
| * Boundary is better until we cross under the Memory Line | |
| * Size (the minimum unit of transfer to/from Memory). If we | |
| * have a Padding Boundary which is smaller than the Memory | |
| * Line Size, that'll involve a Read-Modify-Write cycle on the | |
| * Memory Controller which is never good. | |
| */ | |
| /* We want the Packing Boundary to be based on the Cache Line | |
| * Size in order to help avoid False Sharing performance | |
| * issues between CPUs, etc. We also want the Packing | |
| * Boundary to incorporate the PCI-E Maximum Payload Size. We | |
| * get best performance when the Packing Boundary is a | |
| * multiple of the Maximum Payload Size. | |
| */ | |
| pack_align = fl_align; | |
| if (pci_is_pcie(hw->pdev)) { | |
| u32 mps, mps_log; | |
| u16 devctl; | |
| /* The PCIe Device Control Maximum Payload Size field | |
| * [bits 7:5] encodes sizes as powers of 2 starting at | |
| * 128 bytes. | |
| */ | |
| pcie_capability_read_word(hw->pdev, PCI_EXP_DEVCTL, &devctl); | |
| mps_log = ((devctl & PCI_EXP_DEVCTL_PAYLOAD) >> 5) + 7; | |
| mps = 1 << mps_log; | |
| if (mps > pack_align) | |
| pack_align = mps; | |
| } | |
| /* T5/T6 have a special interpretation of the "0" | |
| * value for the Packing Boundary. This corresponds to 16 | |
| * bytes instead of the expected 32 bytes. | |
| */ | |
| if (pack_align <= 16) { | |
| ingpack = INGPACKBOUNDARY_16B_X; | |
| fl_align = 16; | |
| } else if (pack_align == 32) { | |
| ingpack = INGPACKBOUNDARY_64B_X; | |
| fl_align = 64; | |
| } else { | |
| u32 pack_align_log = fls(pack_align) - 1; | |
| ingpack = pack_align_log - INGPACKBOUNDARY_SHIFT_X; | |
| fl_align = pack_align; | |
| } | |
| /* Use the smallest Ingress Padding which isn't smaller than | |
| * the Memory Controller Read/Write Size. We'll take that as | |
| * being 8 bytes since we don't know of any system with a | |
| * wider Memory Controller Bus Width. | |
| */ | |
| if (csio_is_t5(hw->pdev->device & CSIO_HW_CHIP_MASK)) | |
| ingpad = INGPADBOUNDARY_32B_X; | |
| else | |
| ingpad = T6_INGPADBOUNDARY_8B_X; | |
| csio_set_reg_field(hw, SGE_CONTROL_A, | |
| INGPADBOUNDARY_V(INGPADBOUNDARY_M) | | |
| EGRSTATUSPAGESIZE_F, | |
| INGPADBOUNDARY_V(ingpad) | | |
| EGRSTATUSPAGESIZE_V(stat_len != 64)); | |
| csio_set_reg_field(hw, SGE_CONTROL2_A, | |
| INGPACKBOUNDARY_V(INGPACKBOUNDARY_M), | |
| INGPACKBOUNDARY_V(ingpack)); | |
| /* FL BUFFER SIZE#0 is Page size i,e already aligned to cache line */ | |
| csio_wr_reg32(hw, PAGE_SIZE, SGE_FL_BUFFER_SIZE0_A); | |
| /* | |
| * If using hard params, the following will get set correctly | |
| * in csio_wr_set_sge(). | |
| */ | |
| if (hw->flags & CSIO_HWF_USING_SOFT_PARAMS) { | |
| csio_wr_reg32(hw, | |
| (csio_rd_reg32(hw, SGE_FL_BUFFER_SIZE2_A) + | |
| fl_align - 1) & ~(fl_align - 1), | |
| SGE_FL_BUFFER_SIZE2_A); | |
| csio_wr_reg32(hw, | |
| (csio_rd_reg32(hw, SGE_FL_BUFFER_SIZE3_A) + | |
| fl_align - 1) & ~(fl_align - 1), | |
| SGE_FL_BUFFER_SIZE3_A); | |
| } | |
| sge->csio_fl_align = fl_align; | |
| csio_wr_reg32(hw, HPZ0_V(PAGE_SHIFT - 12), ULP_RX_TDDP_PSZ_A); | |
| /* default value of rx_dma_offset of the NIC driver */ | |
| csio_set_reg_field(hw, SGE_CONTROL_A, | |
| PKTSHIFT_V(PKTSHIFT_M), | |
| PKTSHIFT_V(CSIO_SGE_RX_DMA_OFFSET)); | |
| csio_hw_tp_wr_bits_indirect(hw, TP_INGRESS_CONFIG_A, | |
| CSUM_HAS_PSEUDO_HDR_F, 0); | |
| } | |
| static void | |
| csio_init_intr_coalesce_parms(struct csio_hw *hw) | |
| { | |
| struct csio_wrm *wrm = csio_hw_to_wrm(hw); | |
| struct csio_sge *sge = &wrm->sge; | |
| csio_sge_thresh_reg = csio_closest_thresh(sge, csio_intr_coalesce_cnt); | |
| if (csio_intr_coalesce_cnt) { | |
| csio_sge_thresh_reg = 0; | |
| csio_sge_timer_reg = X_TIMERREG_RESTART_COUNTER; | |
| return; | |
| } | |
| csio_sge_timer_reg = csio_closest_timer(sge, csio_intr_coalesce_time); | |
| } | |
| /* | |
| * csio_wr_get_sge - Get SGE register values. | |
| * @hw: HW module. | |
| * | |
| * Used by non-master functions and by master-functions relying on config file. | |
| */ | |
| static void | |
| csio_wr_get_sge(struct csio_hw *hw) | |
| { | |
| struct csio_wrm *wrm = csio_hw_to_wrm(hw); | |
| struct csio_sge *sge = &wrm->sge; | |
| uint32_t ingpad; | |
| int i; | |
| u32 timer_value_0_and_1, timer_value_2_and_3, timer_value_4_and_5; | |
| u32 ingress_rx_threshold; | |
| sge->sge_control = csio_rd_reg32(hw, SGE_CONTROL_A); | |
| ingpad = INGPADBOUNDARY_G(sge->sge_control); | |
| switch (ingpad) { | |
| case X_INGPCIEBOUNDARY_32B: | |
| sge->csio_fl_align = 32; break; | |
| case X_INGPCIEBOUNDARY_64B: | |
| sge->csio_fl_align = 64; break; | |
| case X_INGPCIEBOUNDARY_128B: | |
| sge->csio_fl_align = 128; break; | |
| case X_INGPCIEBOUNDARY_256B: | |
| sge->csio_fl_align = 256; break; | |
| case X_INGPCIEBOUNDARY_512B: | |
| sge->csio_fl_align = 512; break; | |
| case X_INGPCIEBOUNDARY_1024B: | |
| sge->csio_fl_align = 1024; break; | |
| case X_INGPCIEBOUNDARY_2048B: | |
| sge->csio_fl_align = 2048; break; | |
| case X_INGPCIEBOUNDARY_4096B: | |
| sge->csio_fl_align = 4096; break; | |
| } | |
| for (i = 0; i < CSIO_SGE_FL_SIZE_REGS; i++) | |
| csio_get_flbuf_size(hw, sge, i); | |
| timer_value_0_and_1 = csio_rd_reg32(hw, SGE_TIMER_VALUE_0_AND_1_A); | |
| timer_value_2_and_3 = csio_rd_reg32(hw, SGE_TIMER_VALUE_2_AND_3_A); | |
| timer_value_4_and_5 = csio_rd_reg32(hw, SGE_TIMER_VALUE_4_AND_5_A); | |
| sge->timer_val[0] = (uint16_t)csio_core_ticks_to_us(hw, | |
| TIMERVALUE0_G(timer_value_0_and_1)); | |
| sge->timer_val[1] = (uint16_t)csio_core_ticks_to_us(hw, | |
| TIMERVALUE1_G(timer_value_0_and_1)); | |
| sge->timer_val[2] = (uint16_t)csio_core_ticks_to_us(hw, | |
| TIMERVALUE2_G(timer_value_2_and_3)); | |
| sge->timer_val[3] = (uint16_t)csio_core_ticks_to_us(hw, | |
| TIMERVALUE3_G(timer_value_2_and_3)); | |
| sge->timer_val[4] = (uint16_t)csio_core_ticks_to_us(hw, | |
| TIMERVALUE4_G(timer_value_4_and_5)); | |
| sge->timer_val[5] = (uint16_t)csio_core_ticks_to_us(hw, | |
| TIMERVALUE5_G(timer_value_4_and_5)); | |
| ingress_rx_threshold = csio_rd_reg32(hw, SGE_INGRESS_RX_THRESHOLD_A); | |
| sge->counter_val[0] = THRESHOLD_0_G(ingress_rx_threshold); | |
| sge->counter_val[1] = THRESHOLD_1_G(ingress_rx_threshold); | |
| sge->counter_val[2] = THRESHOLD_2_G(ingress_rx_threshold); | |
| sge->counter_val[3] = THRESHOLD_3_G(ingress_rx_threshold); | |
| csio_init_intr_coalesce_parms(hw); | |
| } | |
| /* | |
| * csio_wr_set_sge - Initialize SGE registers | |
| * @hw: HW module. | |
| * | |
| * Used by Master function to initialize SGE registers in the absence | |
| * of a config file. | |
| */ | |
| static void | |
| csio_wr_set_sge(struct csio_hw *hw) | |
| { | |
| struct csio_wrm *wrm = csio_hw_to_wrm(hw); | |
| struct csio_sge *sge = &wrm->sge; | |
| int i; | |
| /* | |
| * Set up our basic SGE mode to deliver CPL messages to our Ingress | |
| * Queue and Packet Date to the Free List. | |
| */ | |
| csio_set_reg_field(hw, SGE_CONTROL_A, RXPKTCPLMODE_F, RXPKTCPLMODE_F); | |
| sge->sge_control = csio_rd_reg32(hw, SGE_CONTROL_A); | |
| /* sge->csio_fl_align is set up by csio_wr_fixup_host_params(). */ | |
| /* | |
| * Set up to drop DOORBELL writes when the DOORBELL FIFO overflows | |
| * and generate an interrupt when this occurs so we can recover. | |
| */ | |
| csio_set_reg_field(hw, SGE_DBFIFO_STATUS_A, | |
| LP_INT_THRESH_T5_V(LP_INT_THRESH_T5_M), | |
| LP_INT_THRESH_T5_V(CSIO_SGE_DBFIFO_INT_THRESH)); | |
| csio_set_reg_field(hw, SGE_DBFIFO_STATUS2_A, | |
| HP_INT_THRESH_T5_V(LP_INT_THRESH_T5_M), | |
| HP_INT_THRESH_T5_V(CSIO_SGE_DBFIFO_INT_THRESH)); | |
| csio_set_reg_field(hw, SGE_DOORBELL_CONTROL_A, ENABLE_DROP_F, | |
| ENABLE_DROP_F); | |
| /* SGE_FL_BUFFER_SIZE0 is set up by csio_wr_fixup_host_params(). */ | |
| CSIO_SET_FLBUF_SIZE(hw, 1, CSIO_SGE_FLBUF_SIZE1); | |
| csio_wr_reg32(hw, (CSIO_SGE_FLBUF_SIZE2 + sge->csio_fl_align - 1) | |
| & ~(sge->csio_fl_align - 1), SGE_FL_BUFFER_SIZE2_A); | |
| csio_wr_reg32(hw, (CSIO_SGE_FLBUF_SIZE3 + sge->csio_fl_align - 1) | |
| & ~(sge->csio_fl_align - 1), SGE_FL_BUFFER_SIZE3_A); | |
| CSIO_SET_FLBUF_SIZE(hw, 4, CSIO_SGE_FLBUF_SIZE4); | |
| CSIO_SET_FLBUF_SIZE(hw, 5, CSIO_SGE_FLBUF_SIZE5); | |
| CSIO_SET_FLBUF_SIZE(hw, 6, CSIO_SGE_FLBUF_SIZE6); | |
| CSIO_SET_FLBUF_SIZE(hw, 7, CSIO_SGE_FLBUF_SIZE7); | |
| CSIO_SET_FLBUF_SIZE(hw, 8, CSIO_SGE_FLBUF_SIZE8); | |
| for (i = 0; i < CSIO_SGE_FL_SIZE_REGS; i++) | |
| csio_get_flbuf_size(hw, sge, i); | |
| /* Initialize interrupt coalescing attributes */ | |
| sge->timer_val[0] = CSIO_SGE_TIMER_VAL_0; | |
| sge->timer_val[1] = CSIO_SGE_TIMER_VAL_1; | |
| sge->timer_val[2] = CSIO_SGE_TIMER_VAL_2; | |
| sge->timer_val[3] = CSIO_SGE_TIMER_VAL_3; | |
| sge->timer_val[4] = CSIO_SGE_TIMER_VAL_4; | |
| sge->timer_val[5] = CSIO_SGE_TIMER_VAL_5; | |
| sge->counter_val[0] = CSIO_SGE_INT_CNT_VAL_0; | |
| sge->counter_val[1] = CSIO_SGE_INT_CNT_VAL_1; | |
| sge->counter_val[2] = CSIO_SGE_INT_CNT_VAL_2; | |
| sge->counter_val[3] = CSIO_SGE_INT_CNT_VAL_3; | |
| csio_wr_reg32(hw, THRESHOLD_0_V(sge->counter_val[0]) | | |
| THRESHOLD_1_V(sge->counter_val[1]) | | |
| THRESHOLD_2_V(sge->counter_val[2]) | | |
| THRESHOLD_3_V(sge->counter_val[3]), | |
| SGE_INGRESS_RX_THRESHOLD_A); | |
| csio_wr_reg32(hw, | |
| TIMERVALUE0_V(csio_us_to_core_ticks(hw, sge->timer_val[0])) | | |
| TIMERVALUE1_V(csio_us_to_core_ticks(hw, sge->timer_val[1])), | |
| SGE_TIMER_VALUE_0_AND_1_A); | |
| csio_wr_reg32(hw, | |
| TIMERVALUE2_V(csio_us_to_core_ticks(hw, sge->timer_val[2])) | | |
| TIMERVALUE3_V(csio_us_to_core_ticks(hw, sge->timer_val[3])), | |
| SGE_TIMER_VALUE_2_AND_3_A); | |
| csio_wr_reg32(hw, | |
| TIMERVALUE4_V(csio_us_to_core_ticks(hw, sge->timer_val[4])) | | |
| TIMERVALUE5_V(csio_us_to_core_ticks(hw, sge->timer_val[5])), | |
| SGE_TIMER_VALUE_4_AND_5_A); | |
| csio_init_intr_coalesce_parms(hw); | |
| } | |
| void | |
| csio_wr_sge_init(struct csio_hw *hw) | |
| { | |
| /* | |
| * If we are master and chip is not initialized: | |
| * - If we plan to use the config file, we need to fixup some | |
| * host specific registers, and read the rest of the SGE | |
| * configuration. | |
| * - If we dont plan to use the config file, we need to initialize | |
| * SGE entirely, including fixing the host specific registers. | |
| * If we are master and chip is initialized, just read and work off of | |
| * the already initialized SGE values. | |
| * If we arent the master, we are only allowed to read and work off of | |
| * the already initialized SGE values. | |
| * | |
| * Therefore, before calling this function, we assume that the master- | |
| * ship of the card, state and whether to use config file or not, have | |
| * already been decided. | |
| */ | |
| if (csio_is_hw_master(hw)) { | |
| if (hw->fw_state != CSIO_DEV_STATE_INIT) | |
| csio_wr_fixup_host_params(hw); | |
| if (hw->flags & CSIO_HWF_USING_SOFT_PARAMS) | |
| csio_wr_get_sge(hw); | |
| else | |
| csio_wr_set_sge(hw); | |
| } else | |
| csio_wr_get_sge(hw); | |
| } | |
| /* | |
| * csio_wrm_init - Initialize Work request module. | |
| * @wrm: WR module | |
| * @hw: HW pointer | |
| * | |
| * Allocates memory for an array of queue pointers starting at q_arr. | |
| */ | |
| int | |
| csio_wrm_init(struct csio_wrm *wrm, struct csio_hw *hw) | |
| { | |
| int i; | |
| if (!wrm->num_q) { | |
| csio_err(hw, "Num queues is not set\n"); | |
| return -EINVAL; | |
| } | |
| wrm->q_arr = kcalloc(wrm->num_q, sizeof(struct csio_q *), GFP_KERNEL); | |
| if (!wrm->q_arr) | |
| goto err; | |
| for (i = 0; i < wrm->num_q; i++) { | |
| wrm->q_arr[i] = kzalloc(sizeof(struct csio_q), GFP_KERNEL); | |
| if (!wrm->q_arr[i]) { | |
| while (--i >= 0) | |
| kfree(wrm->q_arr[i]); | |
| goto err_free_arr; | |
| } | |
| } | |
| wrm->free_qidx = 0; | |
| return 0; | |
| err_free_arr: | |
| kfree(wrm->q_arr); | |
| err: | |
| return -ENOMEM; | |
| } | |
| /* | |
| * csio_wrm_exit - Initialize Work request module. | |
| * @wrm: WR module | |
| * @hw: HW module | |
| * | |
| * Uninitialize WR module. Free q_arr and pointers in it. | |
| * We have the additional job of freeing the DMA memory associated | |
| * with the queues. | |
| */ | |
| void | |
| csio_wrm_exit(struct csio_wrm *wrm, struct csio_hw *hw) | |
| { | |
| int i; | |
| uint32_t j; | |
| struct csio_q *q; | |
| struct csio_dma_buf *buf; | |
| for (i = 0; i < wrm->num_q; i++) { | |
| q = wrm->q_arr[i]; | |
| if (wrm->free_qidx && (i < wrm->free_qidx)) { | |
| if (q->type == CSIO_FREELIST) { | |
| if (!q->un.fl.bufs) | |
| continue; | |
| for (j = 0; j < q->credits; j++) { | |
| buf = &q->un.fl.bufs[j]; | |
| if (!buf->vaddr) | |
| continue; | |
| dma_free_coherent(&hw->pdev->dev, | |
| buf->len, buf->vaddr, | |
| buf->paddr); | |
| } | |
| kfree(q->un.fl.bufs); | |
| } | |
| dma_free_coherent(&hw->pdev->dev, q->size, | |
| q->vstart, q->pstart); | |
| } | |
| kfree(q); | |
| } | |
| hw->flags &= ~CSIO_HWF_Q_MEM_ALLOCED; | |
| kfree(wrm->q_arr); | |
| } |