blob: 57866c1e9c9821d7c94c819c2db7686fd0d1133e [file] [log] [blame]
/*
* Copyright (C) 2016 Felix Fietkau <nbd@nbd.name>
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#include <linux/dma-mapping.h>
#include "mt76.h"
#include "dma.h"
#define DMA_DUMMY_TXWI ((void *) ~0)
static int
mt76_dma_alloc_queue(struct mt76_dev *dev, struct mt76_queue *q)
{
int size;
int i;
spin_lock_init(&q->lock);
INIT_LIST_HEAD(&q->swq);
size = q->ndesc * sizeof(struct mt76_desc);
q->desc = dmam_alloc_coherent(dev->dev, size, &q->desc_dma, GFP_KERNEL);
if (!q->desc)
return -ENOMEM;
size = q->ndesc * sizeof(*q->entry);
q->entry = devm_kzalloc(dev->dev, size, GFP_KERNEL);
if (!q->entry)
return -ENOMEM;
/* clear descriptors */
for (i = 0; i < q->ndesc; i++)
q->desc[i].ctrl = cpu_to_le32(MT_DMA_CTL_DMA_DONE);
iowrite32(q->desc_dma, &q->regs->desc_base);
iowrite32(0, &q->regs->cpu_idx);
iowrite32(0, &q->regs->dma_idx);
iowrite32(q->ndesc, &q->regs->ring_size);
return 0;
}
static int
mt76_dma_add_buf(struct mt76_dev *dev, struct mt76_queue *q,
struct mt76_queue_buf *buf, int nbufs, u32 info,
struct sk_buff *skb, void *txwi)
{
struct mt76_desc *desc;
u32 ctrl;
int i, idx = -1;
if (txwi)
q->entry[q->head].txwi = DMA_DUMMY_TXWI;
for (i = 0; i < nbufs; i += 2, buf += 2) {
u32 buf0 = buf[0].addr, buf1 = 0;
ctrl = FIELD_PREP(MT_DMA_CTL_SD_LEN0, buf[0].len);
if (i < nbufs - 1) {
buf1 = buf[1].addr;
ctrl |= FIELD_PREP(MT_DMA_CTL_SD_LEN1, buf[1].len);
}
if (i == nbufs - 1)
ctrl |= MT_DMA_CTL_LAST_SEC0;
else if (i == nbufs - 2)
ctrl |= MT_DMA_CTL_LAST_SEC1;
idx = q->head;
q->head = (q->head + 1) % q->ndesc;
desc = &q->desc[idx];
WRITE_ONCE(desc->buf0, cpu_to_le32(buf0));
WRITE_ONCE(desc->buf1, cpu_to_le32(buf1));
WRITE_ONCE(desc->info, cpu_to_le32(info));
WRITE_ONCE(desc->ctrl, cpu_to_le32(ctrl));
q->queued++;
}
q->entry[idx].txwi = txwi;
q->entry[idx].skb = skb;
return idx;
}
static void
mt76_dma_tx_cleanup_idx(struct mt76_dev *dev, struct mt76_queue *q, int idx,
struct mt76_queue_entry *prev_e)
{
struct mt76_queue_entry *e = &q->entry[idx];
__le32 __ctrl = READ_ONCE(q->desc[idx].ctrl);
u32 ctrl = le32_to_cpu(__ctrl);
if (!e->txwi || !e->skb) {
__le32 addr = READ_ONCE(q->desc[idx].buf0);
u32 len = FIELD_GET(MT_DMA_CTL_SD_LEN0, ctrl);
dma_unmap_single(dev->dev, le32_to_cpu(addr), len,
DMA_TO_DEVICE);
}
if (!(ctrl & MT_DMA_CTL_LAST_SEC0)) {
__le32 addr = READ_ONCE(q->desc[idx].buf1);
u32 len = FIELD_GET(MT_DMA_CTL_SD_LEN1, ctrl);
dma_unmap_single(dev->dev, le32_to_cpu(addr), len,
DMA_TO_DEVICE);
}
if (e->txwi == DMA_DUMMY_TXWI)
e->txwi = NULL;
*prev_e = *e;
memset(e, 0, sizeof(*e));
}
static void
mt76_dma_sync_idx(struct mt76_dev *dev, struct mt76_queue *q)
{
q->head = ioread32(&q->regs->dma_idx);
q->tail = q->head;
iowrite32(q->head, &q->regs->cpu_idx);
}
static void
mt76_dma_tx_cleanup(struct mt76_dev *dev, enum mt76_txq_id qid, bool flush)
{
struct mt76_queue *q = &dev->q_tx[qid];
struct mt76_queue_entry entry;
bool wake = false;
int last;
if (!q->ndesc)
return;
spin_lock_bh(&q->lock);
if (flush)
last = -1;
else
last = ioread32(&q->regs->dma_idx);
while (q->queued && q->tail != last) {
mt76_dma_tx_cleanup_idx(dev, q, q->tail, &entry);
if (entry.schedule)
q->swq_queued--;
if (entry.skb)
dev->drv->tx_complete_skb(dev, q, &entry, flush);
if (entry.txwi) {
mt76_put_txwi(dev, entry.txwi);
wake = true;
}
q->tail = (q->tail + 1) % q->ndesc;
q->queued--;
if (!flush && q->tail == last)
last = ioread32(&q->regs->dma_idx);
}
if (!flush)
mt76_txq_schedule(dev, q);
else
mt76_dma_sync_idx(dev, q);
wake = wake && qid < IEEE80211_NUM_ACS && q->queued < q->ndesc - 8;
if (!q->queued)
wake_up(&dev->tx_wait);
spin_unlock_bh(&q->lock);
if (wake)
ieee80211_wake_queue(dev->hw, qid);
}
static void *
mt76_dma_get_buf(struct mt76_dev *dev, struct mt76_queue *q, int idx,
int *len, u32 *info, bool *more)
{
struct mt76_queue_entry *e = &q->entry[idx];
struct mt76_desc *desc = &q->desc[idx];
dma_addr_t buf_addr;
void *buf = e->buf;
int buf_len = SKB_WITH_OVERHEAD(q->buf_size);
buf_addr = le32_to_cpu(READ_ONCE(desc->buf0));
if (len) {
u32 ctl = le32_to_cpu(READ_ONCE(desc->ctrl));
*len = FIELD_GET(MT_DMA_CTL_SD_LEN0, ctl);
*more = !(ctl & MT_DMA_CTL_LAST_SEC0);
}
if (info)
*info = le32_to_cpu(desc->info);
dma_unmap_single(dev->dev, buf_addr, buf_len, DMA_FROM_DEVICE);
e->buf = NULL;
return buf;
}
static void *
mt76_dma_dequeue(struct mt76_dev *dev, struct mt76_queue *q, bool flush,
int *len, u32 *info, bool *more)
{
int idx = q->tail;
*more = false;
if (!q->queued)
return NULL;
if (!flush && !(q->desc[idx].ctrl & cpu_to_le32(MT_DMA_CTL_DMA_DONE)))
return NULL;
q->tail = (q->tail + 1) % q->ndesc;
q->queued--;
return mt76_dma_get_buf(dev, q, idx, len, info, more);
}
static void
mt76_dma_kick_queue(struct mt76_dev *dev, struct mt76_queue *q)
{
iowrite32(q->head, &q->regs->cpu_idx);
}
int mt76_dma_tx_queue_skb(struct mt76_dev *dev, struct mt76_queue *q,
struct sk_buff *skb, struct mt76_wcid *wcid,
struct ieee80211_sta *sta)
{
struct mt76_queue_entry e;
struct mt76_txwi_cache *t;
struct mt76_queue_buf buf[32];
struct sk_buff *iter;
dma_addr_t addr;
int len;
u32 tx_info = 0;
int n, ret;
t = mt76_get_txwi(dev);
if (!t) {
ieee80211_free_txskb(dev->hw, skb);
return -ENOMEM;
}
dma_sync_single_for_cpu(dev->dev, t->dma_addr, sizeof(t->txwi),
DMA_TO_DEVICE);
ret = dev->drv->tx_prepare_skb(dev, &t->txwi, skb, q, wcid, sta,
&tx_info);
dma_sync_single_for_device(dev->dev, t->dma_addr, sizeof(t->txwi),
DMA_TO_DEVICE);
if (ret < 0)
goto free;
len = skb->len - skb->data_len;
addr = dma_map_single(dev->dev, skb->data, len, DMA_TO_DEVICE);
if (dma_mapping_error(dev->dev, addr)) {
ret = -ENOMEM;
goto free;
}
n = 0;
buf[n].addr = t->dma_addr;
buf[n++].len = dev->drv->txwi_size;
buf[n].addr = addr;
buf[n++].len = len;
skb_walk_frags(skb, iter) {
if (n == ARRAY_SIZE(buf))
goto unmap;
addr = dma_map_single(dev->dev, iter->data, iter->len,
DMA_TO_DEVICE);
if (dma_mapping_error(dev->dev, addr))
goto unmap;
buf[n].addr = addr;
buf[n++].len = iter->len;
}
if (q->queued + (n + 1) / 2 >= q->ndesc - 1)
goto unmap;
return dev->queue_ops->add_buf(dev, q, buf, n, tx_info, skb, t);
unmap:
ret = -ENOMEM;
for (n--; n > 0; n--)
dma_unmap_single(dev->dev, buf[n].addr, buf[n].len,
DMA_TO_DEVICE);
free:
e.skb = skb;
e.txwi = t;
dev->drv->tx_complete_skb(dev, q, &e, true);
mt76_put_txwi(dev, t);
return ret;
}
EXPORT_SYMBOL_GPL(mt76_dma_tx_queue_skb);
static int
mt76_dma_rx_fill(struct mt76_dev *dev, struct mt76_queue *q, bool napi)
{
dma_addr_t addr;
void *buf;
int frames = 0;
int len = SKB_WITH_OVERHEAD(q->buf_size);
int offset = q->buf_offset;
int idx;
void *(*alloc)(unsigned int fragsz);
if (napi)
alloc = napi_alloc_frag;
else
alloc = netdev_alloc_frag;
spin_lock_bh(&q->lock);
while (q->queued < q->ndesc - 1) {
struct mt76_queue_buf qbuf;
buf = alloc(q->buf_size);
if (!buf)
break;
addr = dma_map_single(dev->dev, buf, len, DMA_FROM_DEVICE);
if (dma_mapping_error(dev->dev, addr)) {
skb_free_frag(buf);
break;
}
qbuf.addr = addr + offset;
qbuf.len = len - offset;
idx = mt76_dma_add_buf(dev, q, &qbuf, 1, 0, buf, NULL);
frames++;
}
if (frames)
mt76_dma_kick_queue(dev, q);
spin_unlock_bh(&q->lock);
return frames;
}
static void
mt76_dma_rx_cleanup(struct mt76_dev *dev, struct mt76_queue *q)
{
void *buf;
bool more;
spin_lock_bh(&q->lock);
do {
buf = mt76_dma_dequeue(dev, q, true, NULL, NULL, &more);
if (!buf)
break;
skb_free_frag(buf);
} while (1);
spin_unlock_bh(&q->lock);
}
static void
mt76_dma_rx_reset(struct mt76_dev *dev, enum mt76_rxq_id qid)
{
struct mt76_queue *q = &dev->q_rx[qid];
int i;
for (i = 0; i < q->ndesc; i++)
q->desc[i].ctrl &= ~cpu_to_le32(MT_DMA_CTL_DMA_DONE);
mt76_dma_rx_cleanup(dev, q);
mt76_dma_sync_idx(dev, q);
mt76_dma_rx_fill(dev, q, false);
}
static void
mt76_add_fragment(struct mt76_dev *dev, struct mt76_queue *q, void *data,
int len, bool more)
{
struct sk_buff *skb = q->rx_head;
struct skb_shared_info *shinfo = skb_shinfo(skb);
int nr_frags = shinfo->nr_frags;
if (nr_frags < ARRAY_SIZE(shinfo->frags)) {
struct page *page = virt_to_head_page(data);
int offset = data - page_address(page) + q->buf_offset;
skb_add_rx_frag(skb, nr_frags, page, offset, len, q->buf_size);
} else {
skb_free_frag(data);
}
if (more)
return;
q->rx_head = NULL;
if (nr_frags < ARRAY_SIZE(shinfo->frags))
dev->drv->rx_skb(dev, q - dev->q_rx, skb);
else
dev_kfree_skb(skb);
}
static int
mt76_dma_rx_process(struct mt76_dev *dev, struct mt76_queue *q, int budget)
{
struct sk_buff *skb;
unsigned char *data;
int len;
int done = 0;
bool more;
while (done < budget) {
u32 info;
data = mt76_dma_dequeue(dev, q, false, &len, &info, &more);
if (!data)
break;
if (q->rx_head) {
mt76_add_fragment(dev, q, data, len, more);
continue;
}
skb = build_skb(data, q->buf_size);
if (!skb) {
skb_free_frag(data);
continue;
}
skb_reserve(skb, q->buf_offset);
if (skb->tail + len > skb->end) {
dev_kfree_skb(skb);
continue;
}
if (q == &dev->q_rx[MT_RXQ_MCU]) {
u32 *rxfce = (u32 *) skb->cb;
*rxfce = info;
}
__skb_put(skb, len);
done++;
if (more) {
q->rx_head = skb;
continue;
}
dev->drv->rx_skb(dev, q - dev->q_rx, skb);
}
mt76_dma_rx_fill(dev, q, true);
return done;
}
static int
mt76_dma_rx_poll(struct napi_struct *napi, int budget)
{
struct mt76_dev *dev;
int qid, done = 0, cur;
dev = container_of(napi->dev, struct mt76_dev, napi_dev);
qid = napi - dev->napi;
rcu_read_lock();
do {
cur = mt76_dma_rx_process(dev, &dev->q_rx[qid], budget - done);
mt76_rx_poll_complete(dev, qid, napi);
done += cur;
} while (cur && done < budget);
rcu_read_unlock();
if (done < budget) {
napi_complete(napi);
dev->drv->rx_poll_complete(dev, qid);
}
return done;
}
static int
mt76_dma_init(struct mt76_dev *dev)
{
int i;
init_dummy_netdev(&dev->napi_dev);
for (i = 0; i < ARRAY_SIZE(dev->q_rx); i++) {
netif_napi_add(&dev->napi_dev, &dev->napi[i], mt76_dma_rx_poll,
64);
mt76_dma_rx_fill(dev, &dev->q_rx[i], false);
skb_queue_head_init(&dev->rx_skb[i]);
napi_enable(&dev->napi[i]);
}
return 0;
}
static const struct mt76_queue_ops mt76_dma_ops = {
.init = mt76_dma_init,
.alloc = mt76_dma_alloc_queue,
.add_buf = mt76_dma_add_buf,
.tx_queue_skb = mt76_dma_tx_queue_skb,
.tx_cleanup = mt76_dma_tx_cleanup,
.rx_reset = mt76_dma_rx_reset,
.kick = mt76_dma_kick_queue,
};
int mt76_dma_attach(struct mt76_dev *dev)
{
dev->queue_ops = &mt76_dma_ops;
return 0;
}
EXPORT_SYMBOL_GPL(mt76_dma_attach);
void mt76_dma_cleanup(struct mt76_dev *dev)
{
int i;
for (i = 0; i < ARRAY_SIZE(dev->q_tx); i++)
mt76_dma_tx_cleanup(dev, i, true);
for (i = 0; i < ARRAY_SIZE(dev->q_rx); i++) {
netif_napi_del(&dev->napi[i]);
mt76_dma_rx_cleanup(dev, &dev->q_rx[i]);
}
}
EXPORT_SYMBOL_GPL(mt76_dma_cleanup);