blob: 03402203cacdbcdf4062ed5f4dbd3c7b9759bacd [file] [log] [blame]
// SPDX-License-Identifier: GPL-2.0+
// Copyright 2017 IBM Corp.
#include <linux/sched/mm.h>
#include <linux/mutex.h>
#include <linux/mm.h>
#include <linux/mm_types.h>
#include <linux/mmu_context.h>
#include <linux/mmu_notifier.h>
#include <linux/irqdomain.h>
#include <asm/copro.h>
#include <asm/pnv-ocxl.h>
#include <asm/xive.h>
#include <misc/ocxl.h>
#include "ocxl_internal.h"
#include "trace.h"
#define SPA_PASID_BITS 15
#define SPA_PASID_MAX ((1 << SPA_PASID_BITS) - 1)
#define SPA_PE_MASK SPA_PASID_MAX
#define SPA_SPA_SIZE_LOG 22 /* Each SPA is 4 Mb */
#define SPA_CFG_SF (1ull << (63-0))
#define SPA_CFG_TA (1ull << (63-1))
#define SPA_CFG_HV (1ull << (63-3))
#define SPA_CFG_UV (1ull << (63-4))
#define SPA_CFG_XLAT_hpt (0ull << (63-6)) /* Hashed page table (HPT) mode */
#define SPA_CFG_XLAT_roh (2ull << (63-6)) /* Radix on HPT mode */
#define SPA_CFG_XLAT_ror (3ull << (63-6)) /* Radix on Radix mode */
#define SPA_CFG_PR (1ull << (63-49))
#define SPA_CFG_TC (1ull << (63-54))
#define SPA_CFG_DR (1ull << (63-59))
#define SPA_XSL_TF (1ull << (63-3)) /* Translation fault */
#define SPA_XSL_S (1ull << (63-38)) /* Store operation */
#define SPA_PE_VALID 0x80000000
struct ocxl_link;
struct pe_data {
struct mm_struct *mm;
/* callback to trigger when a translation fault occurs */
void (*xsl_err_cb)(void *data, u64 addr, u64 dsisr);
/* opaque pointer to be passed to the above callback */
void *xsl_err_data;
struct rcu_head rcu;
struct ocxl_link *link;
struct mmu_notifier mmu_notifier;
};
struct spa {
struct ocxl_process_element *spa_mem;
int spa_order;
struct mutex spa_lock;
struct radix_tree_root pe_tree; /* Maps PE handles to pe_data */
char *irq_name;
int virq;
void __iomem *reg_dsisr;
void __iomem *reg_dar;
void __iomem *reg_tfc;
void __iomem *reg_pe_handle;
/*
* The following field are used by the memory fault
* interrupt handler. We can only have one interrupt at a
* time. The NPU won't raise another interrupt until the
* previous one has been ack'd by writing to the TFC register
*/
struct xsl_fault {
struct work_struct fault_work;
u64 pe;
u64 dsisr;
u64 dar;
struct pe_data pe_data;
} xsl_fault;
};
/*
* A opencapi link can be used be by several PCI functions. We have
* one link per device slot.
*
* A linked list of opencapi links should suffice, as there's a
* limited number of opencapi slots on a system and lookup is only
* done when the device is probed
*/
struct ocxl_link {
struct list_head list;
struct kref ref;
int domain;
int bus;
int dev;
void __iomem *arva; /* ATSD register virtual address */
spinlock_t atsd_lock; /* to serialize shootdowns */
atomic_t irq_available;
struct spa *spa;
void *platform_data;
};
static LIST_HEAD(links_list);
static DEFINE_MUTEX(links_list_lock);
enum xsl_response {
CONTINUE,
ADDRESS_ERROR,
RESTART,
};
static void read_irq(struct spa *spa, u64 *dsisr, u64 *dar, u64 *pe)
{
u64 reg;
*dsisr = in_be64(spa->reg_dsisr);
*dar = in_be64(spa->reg_dar);
reg = in_be64(spa->reg_pe_handle);
*pe = reg & SPA_PE_MASK;
}
static void ack_irq(struct spa *spa, enum xsl_response r)
{
u64 reg = 0;
/* continue is not supported */
if (r == RESTART)
reg = PPC_BIT(31);
else if (r == ADDRESS_ERROR)
reg = PPC_BIT(30);
else
WARN(1, "Invalid irq response %d\n", r);
if (reg) {
trace_ocxl_fault_ack(spa->spa_mem, spa->xsl_fault.pe,
spa->xsl_fault.dsisr, spa->xsl_fault.dar, reg);
out_be64(spa->reg_tfc, reg);
}
}
static void xsl_fault_handler_bh(struct work_struct *fault_work)
{
vm_fault_t flt = 0;
unsigned long access, flags, inv_flags = 0;
enum xsl_response r;
struct xsl_fault *fault = container_of(fault_work, struct xsl_fault,
fault_work);
struct spa *spa = container_of(fault, struct spa, xsl_fault);
int rc;
/*
* We must release a reference on mm_users whenever exiting this
* function (taken in the memory fault interrupt handler)
*/
rc = copro_handle_mm_fault(fault->pe_data.mm, fault->dar, fault->dsisr,
&flt);
if (rc) {
pr_debug("copro_handle_mm_fault failed: %d\n", rc);
if (fault->pe_data.xsl_err_cb) {
fault->pe_data.xsl_err_cb(
fault->pe_data.xsl_err_data,
fault->dar, fault->dsisr);
}
r = ADDRESS_ERROR;
goto ack;
}
if (!radix_enabled()) {
/*
* update_mmu_cache() will not have loaded the hash
* since current->trap is not a 0x400 or 0x300, so
* just call hash_page_mm() here.
*/
access = _PAGE_PRESENT | _PAGE_READ;
if (fault->dsisr & SPA_XSL_S)
access |= _PAGE_WRITE;
if (get_region_id(fault->dar) != USER_REGION_ID)
access |= _PAGE_PRIVILEGED;
local_irq_save(flags);
hash_page_mm(fault->pe_data.mm, fault->dar, access, 0x300,
inv_flags);
local_irq_restore(flags);
}
r = RESTART;
ack:
mmput(fault->pe_data.mm);
ack_irq(spa, r);
}
static irqreturn_t xsl_fault_handler(int irq, void *data)
{
struct ocxl_link *link = data;
struct spa *spa = link->spa;
u64 dsisr, dar, pe_handle;
struct pe_data *pe_data;
struct ocxl_process_element *pe;
int pid;
bool schedule = false;
read_irq(spa, &dsisr, &dar, &pe_handle);
trace_ocxl_fault(spa->spa_mem, pe_handle, dsisr, dar, -1);
WARN_ON(pe_handle > SPA_PE_MASK);
pe = spa->spa_mem + pe_handle;
pid = be32_to_cpu(pe->pid);
/* We could be reading all null values here if the PE is being
* removed while an interrupt kicks in. It's not supposed to
* happen if the driver notified the AFU to terminate the
* PASID, and the AFU waited for pending operations before
* acknowledging. But even if it happens, we won't find a
* memory context below and fail silently, so it should be ok.
*/
if (!(dsisr & SPA_XSL_TF)) {
WARN(1, "Invalid xsl interrupt fault register %#llx\n", dsisr);
ack_irq(spa, ADDRESS_ERROR);
return IRQ_HANDLED;
}
rcu_read_lock();
pe_data = radix_tree_lookup(&spa->pe_tree, pe_handle);
if (!pe_data) {
/*
* Could only happen if the driver didn't notify the
* AFU about PASID termination before removing the PE,
* or the AFU didn't wait for all memory access to
* have completed.
*
* Either way, we fail early, but we shouldn't log an
* error message, as it is a valid (if unexpected)
* scenario
*/
rcu_read_unlock();
pr_debug("Unknown mm context for xsl interrupt\n");
ack_irq(spa, ADDRESS_ERROR);
return IRQ_HANDLED;
}
if (!pe_data->mm) {
/*
* translation fault from a kernel context - an OpenCAPI
* device tried to access a bad kernel address
*/
rcu_read_unlock();
pr_warn("Unresolved OpenCAPI xsl fault in kernel context\n");
ack_irq(spa, ADDRESS_ERROR);
return IRQ_HANDLED;
}
WARN_ON(pe_data->mm->context.id != pid);
if (mmget_not_zero(pe_data->mm)) {
spa->xsl_fault.pe = pe_handle;
spa->xsl_fault.dar = dar;
spa->xsl_fault.dsisr = dsisr;
spa->xsl_fault.pe_data = *pe_data;
schedule = true;
/* mm_users count released by bottom half */
}
rcu_read_unlock();
if (schedule)
schedule_work(&spa->xsl_fault.fault_work);
else
ack_irq(spa, ADDRESS_ERROR);
return IRQ_HANDLED;
}
static void unmap_irq_registers(struct spa *spa)
{
pnv_ocxl_unmap_xsl_regs(spa->reg_dsisr, spa->reg_dar, spa->reg_tfc,
spa->reg_pe_handle);
}
static int map_irq_registers(struct pci_dev *dev, struct spa *spa)
{
return pnv_ocxl_map_xsl_regs(dev, &spa->reg_dsisr, &spa->reg_dar,
&spa->reg_tfc, &spa->reg_pe_handle);
}
static int setup_xsl_irq(struct pci_dev *dev, struct ocxl_link *link)
{
struct spa *spa = link->spa;
int rc;
int hwirq;
rc = pnv_ocxl_get_xsl_irq(dev, &hwirq);
if (rc)
return rc;
rc = map_irq_registers(dev, spa);
if (rc)
return rc;
spa->irq_name = kasprintf(GFP_KERNEL, "ocxl-xsl-%x-%x-%x",
link->domain, link->bus, link->dev);
if (!spa->irq_name) {
dev_err(&dev->dev, "Can't allocate name for xsl interrupt\n");
rc = -ENOMEM;
goto err_xsl;
}
/*
* At some point, we'll need to look into allowing a higher
* number of interrupts. Could we have an IRQ domain per link?
*/
spa->virq = irq_create_mapping(NULL, hwirq);
if (!spa->virq) {
dev_err(&dev->dev,
"irq_create_mapping failed for translation interrupt\n");
rc = -EINVAL;
goto err_name;
}
dev_dbg(&dev->dev, "hwirq %d mapped to virq %d\n", hwirq, spa->virq);
rc = request_irq(spa->virq, xsl_fault_handler, 0, spa->irq_name,
link);
if (rc) {
dev_err(&dev->dev,
"request_irq failed for translation interrupt: %d\n",
rc);
rc = -EINVAL;
goto err_mapping;
}
return 0;
err_mapping:
irq_dispose_mapping(spa->virq);
err_name:
kfree(spa->irq_name);
err_xsl:
unmap_irq_registers(spa);
return rc;
}
static void release_xsl_irq(struct ocxl_link *link)
{
struct spa *spa = link->spa;
if (spa->virq) {
free_irq(spa->virq, link);
irq_dispose_mapping(spa->virq);
}
kfree(spa->irq_name);
unmap_irq_registers(spa);
}
static int alloc_spa(struct pci_dev *dev, struct ocxl_link *link)
{
struct spa *spa;
spa = kzalloc(sizeof(struct spa), GFP_KERNEL);
if (!spa)
return -ENOMEM;
mutex_init(&spa->spa_lock);
INIT_RADIX_TREE(&spa->pe_tree, GFP_KERNEL);
INIT_WORK(&spa->xsl_fault.fault_work, xsl_fault_handler_bh);
spa->spa_order = SPA_SPA_SIZE_LOG - PAGE_SHIFT;
spa->spa_mem = (struct ocxl_process_element *)
__get_free_pages(GFP_KERNEL | __GFP_ZERO, spa->spa_order);
if (!spa->spa_mem) {
dev_err(&dev->dev, "Can't allocate Shared Process Area\n");
kfree(spa);
return -ENOMEM;
}
pr_debug("Allocated SPA for %x:%x:%x at %p\n", link->domain, link->bus,
link->dev, spa->spa_mem);
link->spa = spa;
return 0;
}
static void free_spa(struct ocxl_link *link)
{
struct spa *spa = link->spa;
pr_debug("Freeing SPA for %x:%x:%x\n", link->domain, link->bus,
link->dev);
if (spa && spa->spa_mem) {
free_pages((unsigned long) spa->spa_mem, spa->spa_order);
kfree(spa);
link->spa = NULL;
}
}
static int alloc_link(struct pci_dev *dev, int PE_mask, struct ocxl_link **out_link)
{
struct ocxl_link *link;
int rc;
link = kzalloc(sizeof(struct ocxl_link), GFP_KERNEL);
if (!link)
return -ENOMEM;
kref_init(&link->ref);
link->domain = pci_domain_nr(dev->bus);
link->bus = dev->bus->number;
link->dev = PCI_SLOT(dev->devfn);
atomic_set(&link->irq_available, MAX_IRQ_PER_LINK);
spin_lock_init(&link->atsd_lock);
rc = alloc_spa(dev, link);
if (rc)
goto err_free;
rc = setup_xsl_irq(dev, link);
if (rc)
goto err_spa;
/* platform specific hook */
rc = pnv_ocxl_spa_setup(dev, link->spa->spa_mem, PE_mask,
&link->platform_data);
if (rc)
goto err_xsl_irq;
/* if link->arva is not defeined, MMIO registers are not used to
* generate TLB invalidate. PowerBus snooping is enabled.
* Otherwise, PowerBus snooping is disabled. TLB Invalidates are
* initiated using MMIO registers.
*/
pnv_ocxl_map_lpar(dev, mfspr(SPRN_LPID), 0, &link->arva);
*out_link = link;
return 0;
err_xsl_irq:
release_xsl_irq(link);
err_spa:
free_spa(link);
err_free:
kfree(link);
return rc;
}
static void free_link(struct ocxl_link *link)
{
release_xsl_irq(link);
free_spa(link);
kfree(link);
}
int ocxl_link_setup(struct pci_dev *dev, int PE_mask, void **link_handle)
{
int rc = 0;
struct ocxl_link *link;
mutex_lock(&links_list_lock);
list_for_each_entry(link, &links_list, list) {
/* The functions of a device all share the same link */
if (link->domain == pci_domain_nr(dev->bus) &&
link->bus == dev->bus->number &&
link->dev == PCI_SLOT(dev->devfn)) {
kref_get(&link->ref);
*link_handle = link;
goto unlock;
}
}
rc = alloc_link(dev, PE_mask, &link);
if (rc)
goto unlock;
list_add(&link->list, &links_list);
*link_handle = link;
unlock:
mutex_unlock(&links_list_lock);
return rc;
}
EXPORT_SYMBOL_GPL(ocxl_link_setup);
static void release_xsl(struct kref *ref)
{
struct ocxl_link *link = container_of(ref, struct ocxl_link, ref);
if (link->arva) {
pnv_ocxl_unmap_lpar(link->arva);
link->arva = NULL;
}
list_del(&link->list);
/* call platform code before releasing data */
pnv_ocxl_spa_release(link->platform_data);
free_link(link);
}
void ocxl_link_release(struct pci_dev *dev, void *link_handle)
{
struct ocxl_link *link = link_handle;
mutex_lock(&links_list_lock);
kref_put(&link->ref, release_xsl);
mutex_unlock(&links_list_lock);
}
EXPORT_SYMBOL_GPL(ocxl_link_release);
static void arch_invalidate_secondary_tlbs(struct mmu_notifier *mn,
struct mm_struct *mm,
unsigned long start, unsigned long end)
{
struct pe_data *pe_data = container_of(mn, struct pe_data, mmu_notifier);
struct ocxl_link *link = pe_data->link;
unsigned long addr, pid, page_size = PAGE_SIZE;
pid = mm->context.id;
trace_ocxl_mmu_notifier_range(start, end, pid);
spin_lock(&link->atsd_lock);
for (addr = start; addr < end; addr += page_size)
pnv_ocxl_tlb_invalidate(link->arva, pid, addr, page_size);
spin_unlock(&link->atsd_lock);
}
static const struct mmu_notifier_ops ocxl_mmu_notifier_ops = {
.arch_invalidate_secondary_tlbs = arch_invalidate_secondary_tlbs,
};
static u64 calculate_cfg_state(bool kernel)
{
u64 state;
state = SPA_CFG_DR;
if (mfspr(SPRN_LPCR) & LPCR_TC)
state |= SPA_CFG_TC;
if (radix_enabled())
state |= SPA_CFG_XLAT_ror;
else
state |= SPA_CFG_XLAT_hpt;
state |= SPA_CFG_HV;
if (kernel) {
if (mfmsr() & MSR_SF)
state |= SPA_CFG_SF;
} else {
state |= SPA_CFG_PR;
if (!test_tsk_thread_flag(current, TIF_32BIT))
state |= SPA_CFG_SF;
}
return state;
}
int ocxl_link_add_pe(void *link_handle, int pasid, u32 pidr, u32 tidr,
u64 amr, u16 bdf, struct mm_struct *mm,
void (*xsl_err_cb)(void *data, u64 addr, u64 dsisr),
void *xsl_err_data)
{
struct ocxl_link *link = link_handle;
struct spa *spa = link->spa;
struct ocxl_process_element *pe;
int pe_handle, rc = 0;
struct pe_data *pe_data;
BUILD_BUG_ON(sizeof(struct ocxl_process_element) != 128);
if (pasid > SPA_PASID_MAX)
return -EINVAL;
mutex_lock(&spa->spa_lock);
pe_handle = pasid & SPA_PE_MASK;
pe = spa->spa_mem + pe_handle;
if (pe->software_state) {
rc = -EBUSY;
goto unlock;
}
pe_data = kmalloc(sizeof(*pe_data), GFP_KERNEL);
if (!pe_data) {
rc = -ENOMEM;
goto unlock;
}
pe_data->mm = mm;
pe_data->xsl_err_cb = xsl_err_cb;
pe_data->xsl_err_data = xsl_err_data;
pe_data->link = link;
pe_data->mmu_notifier.ops = &ocxl_mmu_notifier_ops;
memset(pe, 0, sizeof(struct ocxl_process_element));
pe->config_state = cpu_to_be64(calculate_cfg_state(pidr == 0));
pe->pasid = cpu_to_be32(pasid << (31 - 19));
pe->bdf = cpu_to_be16(bdf);
pe->lpid = cpu_to_be32(mfspr(SPRN_LPID));
pe->pid = cpu_to_be32(pidr);
pe->tid = cpu_to_be32(tidr);
pe->amr = cpu_to_be64(amr);
pe->software_state = cpu_to_be32(SPA_PE_VALID);
/*
* For user contexts, register a copro so that TLBIs are seen
* by the nest MMU. If we have a kernel context, TLBIs are
* already global.
*/
if (mm) {
mm_context_add_copro(mm);
if (link->arva) {
/* Use MMIO registers for the TLB Invalidate
* operations.
*/
trace_ocxl_init_mmu_notifier(pasid, mm->context.id);
mmu_notifier_register(&pe_data->mmu_notifier, mm);
}
}
/*
* Barrier is to make sure PE is visible in the SPA before it
* is used by the device. It also helps with the global TLBI
* invalidation
*/
mb();
radix_tree_insert(&spa->pe_tree, pe_handle, pe_data);
/*
* The mm must stay valid for as long as the device uses it. We
* lower the count when the context is removed from the SPA.
*
* We grab mm_count (and not mm_users), as we don't want to
* end up in a circular dependency if a process mmaps its
* mmio, therefore incrementing the file ref count when
* calling mmap(), and forgets to unmap before exiting. In
* that scenario, when the kernel handles the death of the
* process, the file is not cleaned because unmap was not
* called, and the mm wouldn't be freed because we would still
* have a reference on mm_users. Incrementing mm_count solves
* the problem.
*/
if (mm)
mmgrab(mm);
trace_ocxl_context_add(current->pid, spa->spa_mem, pasid, pidr, tidr);
unlock:
mutex_unlock(&spa->spa_lock);
return rc;
}
EXPORT_SYMBOL_GPL(ocxl_link_add_pe);
int ocxl_link_update_pe(void *link_handle, int pasid, __u16 tid)
{
struct ocxl_link *link = link_handle;
struct spa *spa = link->spa;
struct ocxl_process_element *pe;
int pe_handle, rc;
if (pasid > SPA_PASID_MAX)
return -EINVAL;
pe_handle = pasid & SPA_PE_MASK;
pe = spa->spa_mem + pe_handle;
mutex_lock(&spa->spa_lock);
pe->tid = cpu_to_be32(tid);
/*
* The barrier makes sure the PE is updated
* before we clear the NPU context cache below, so that the
* old PE cannot be reloaded erroneously.
*/
mb();
/*
* hook to platform code
* On powerpc, the entry needs to be cleared from the context
* cache of the NPU.
*/
rc = pnv_ocxl_spa_remove_pe_from_cache(link->platform_data, pe_handle);
WARN_ON(rc);
mutex_unlock(&spa->spa_lock);
return rc;
}
int ocxl_link_remove_pe(void *link_handle, int pasid)
{
struct ocxl_link *link = link_handle;
struct spa *spa = link->spa;
struct ocxl_process_element *pe;
struct pe_data *pe_data;
int pe_handle, rc;
if (pasid > SPA_PASID_MAX)
return -EINVAL;
/*
* About synchronization with our memory fault handler:
*
* Before removing the PE, the driver is supposed to have
* notified the AFU, which should have cleaned up and make
* sure the PASID is no longer in use, including pending
* interrupts. However, there's no way to be sure...
*
* We clear the PE and remove the context from our radix
* tree. From that point on, any new interrupt for that
* context will fail silently, which is ok. As mentioned
* above, that's not expected, but it could happen if the
* driver or AFU didn't do the right thing.
*
* There could still be a bottom half running, but we don't
* need to wait/flush, as it is managing a reference count on
* the mm it reads from the radix tree.
*/
pe_handle = pasid & SPA_PE_MASK;
pe = spa->spa_mem + pe_handle;
mutex_lock(&spa->spa_lock);
if (!(be32_to_cpu(pe->software_state) & SPA_PE_VALID)) {
rc = -EINVAL;
goto unlock;
}
trace_ocxl_context_remove(current->pid, spa->spa_mem, pasid,
be32_to_cpu(pe->pid), be32_to_cpu(pe->tid));
memset(pe, 0, sizeof(struct ocxl_process_element));
/*
* The barrier makes sure the PE is removed from the SPA
* before we clear the NPU context cache below, so that the
* old PE cannot be reloaded erroneously.
*/
mb();
/*
* hook to platform code
* On powerpc, the entry needs to be cleared from the context
* cache of the NPU.
*/
rc = pnv_ocxl_spa_remove_pe_from_cache(link->platform_data, pe_handle);
WARN_ON(rc);
pe_data = radix_tree_delete(&spa->pe_tree, pe_handle);
if (!pe_data) {
WARN(1, "Couldn't find pe data when removing PE\n");
} else {
if (pe_data->mm) {
if (link->arva) {
trace_ocxl_release_mmu_notifier(pasid,
pe_data->mm->context.id);
mmu_notifier_unregister(&pe_data->mmu_notifier,
pe_data->mm);
spin_lock(&link->atsd_lock);
pnv_ocxl_tlb_invalidate(link->arva,
pe_data->mm->context.id,
0ull,
PAGE_SIZE);
spin_unlock(&link->atsd_lock);
}
mm_context_remove_copro(pe_data->mm);
mmdrop(pe_data->mm);
}
kfree_rcu(pe_data, rcu);
}
unlock:
mutex_unlock(&spa->spa_lock);
return rc;
}
EXPORT_SYMBOL_GPL(ocxl_link_remove_pe);
int ocxl_link_irq_alloc(void *link_handle, int *hw_irq)
{
struct ocxl_link *link = link_handle;
int irq;
if (atomic_dec_if_positive(&link->irq_available) < 0)
return -ENOSPC;
irq = xive_native_alloc_irq();
if (!irq) {
atomic_inc(&link->irq_available);
return -ENXIO;
}
*hw_irq = irq;
return 0;
}
EXPORT_SYMBOL_GPL(ocxl_link_irq_alloc);
void ocxl_link_free_irq(void *link_handle, int hw_irq)
{
struct ocxl_link *link = link_handle;
xive_native_free_irq(hw_irq);
atomic_inc(&link->irq_available);
}
EXPORT_SYMBOL_GPL(ocxl_link_free_irq);