| // SPDX-License-Identifier: GPL-2.0 | |
| /* | |
| * Copyright (C) 2024 Loongson Technology Corporation Limited | |
| */ | |
| #include <linux/kvm_host.h> | |
| #include <asm/kvm_ipi.h> | |
| #include <asm/kvm_vcpu.h> | |
| static void ipi_set(struct kvm_vcpu *vcpu, uint32_t data) | |
| { | |
| uint32_t status; | |
| spin_lock(&vcpu->arch.ipi_state.lock); | |
| status = vcpu->arch.ipi_state.status; | |
| vcpu->arch.ipi_state.status |= data; | |
| if ((status == 0) && data) { | |
| kvm_queue_irq(vcpu, LARCH_INT_IPI); | |
| kvm_vcpu_kick(vcpu); | |
| } | |
| spin_unlock(&vcpu->arch.ipi_state.lock); | |
| } | |
| static void ipi_send(struct kvm *kvm, uint64_t data) | |
| { | |
| int cpu; | |
| struct kvm_vcpu *vcpu; | |
| cpu = ((data & 0xffffffff) >> 16) & 0x3ff; | |
| vcpu = kvm_get_vcpu_by_cpuid(kvm, cpu); | |
| if (unlikely(vcpu == NULL)) { | |
| kvm_pr_unimpl("%s: invalid target cpu: %d\n", __func__, cpu); | |
| return; | |
| } | |
| ipi_set(vcpu, BIT(data & 0x1f)); | |
| } | |
| static void ipi_clear(struct kvm_vcpu *vcpu, uint64_t data) | |
| { | |
| uint32_t status; | |
| spin_lock(&vcpu->arch.ipi_state.lock); | |
| vcpu->arch.ipi_state.status &= ~data; | |
| status = vcpu->arch.ipi_state.status; | |
| if (status == 0) { | |
| kvm_dequeue_irq(vcpu, LARCH_INT_IPI); | |
| kvm_vcpu_kick(vcpu); | |
| } | |
| spin_unlock(&vcpu->arch.ipi_state.lock); | |
| } | |
| static uint64_t read_mailbox(struct kvm_vcpu *vcpu, int offset, int len) | |
| { | |
| uint64_t data = 0; | |
| spin_lock(&vcpu->arch.ipi_state.lock); | |
| data = *(ulong *)((void *)vcpu->arch.ipi_state.buf + (offset - 0x20)); | |
| spin_unlock(&vcpu->arch.ipi_state.lock); | |
| switch (len) { | |
| case 1: | |
| return data & 0xff; | |
| case 2: | |
| return data & 0xffff; | |
| case 4: | |
| return data & 0xffffffff; | |
| default: | |
| return data; | |
| } | |
| } | |
| static void write_mailbox(struct kvm_vcpu *vcpu, int offset, uint64_t data, int len) | |
| { | |
| void *pbuf; | |
| spin_lock(&vcpu->arch.ipi_state.lock); | |
| pbuf = (void *)vcpu->arch.ipi_state.buf + (offset - 0x20); | |
| switch (len) { | |
| case 1: | |
| *(unsigned char *)pbuf = (unsigned char)data; | |
| break; | |
| case 2: | |
| *(unsigned short *)pbuf = (unsigned short)data; | |
| break; | |
| case 4: | |
| *(unsigned int *)pbuf = (unsigned int)data; | |
| break; | |
| default: | |
| *(unsigned long *)pbuf = (unsigned long)data; | |
| break; | |
| } | |
| spin_unlock(&vcpu->arch.ipi_state.lock); | |
| } | |
| static int mail_send(struct kvm *kvm, uint64_t data) | |
| { | |
| int i, cpu, mailbox, offset; | |
| uint32_t val = 0, mask = 0; | |
| struct kvm_vcpu *vcpu; | |
| cpu = ((data & 0xffffffff) >> 16) & 0x3ff; | |
| vcpu = kvm_get_vcpu_by_cpuid(kvm, cpu); | |
| if (unlikely(vcpu == NULL)) { | |
| kvm_pr_unimpl("%s: invalid target cpu: %d\n", __func__, cpu); | |
| return 0; | |
| } | |
| mailbox = ((data & 0xffffffff) >> 2) & 0x7; | |
| offset = IOCSR_IPI_BUF_20 + mailbox * 4; | |
| if ((data >> 27) & 0xf) { | |
| val = read_mailbox(vcpu, offset, 4); | |
| for (i = 0; i < 4; i++) | |
| if (data & (BIT(27 + i))) | |
| mask |= (0xff << (i * 8)); | |
| val &= mask; | |
| } | |
| val |= ((uint32_t)(data >> 32) & ~mask); | |
| write_mailbox(vcpu, offset, val, 4); | |
| return 0; | |
| } | |
| static int send_ipi_data(struct kvm_vcpu *vcpu, gpa_t addr, uint64_t data) | |
| { | |
| int i, idx, ret; | |
| uint64_t val = 0, mask = 0; | |
| /* | |
| * Bit 27-30 is mask for byte writing. | |
| * If the mask is 0, we need not to do anything. | |
| */ | |
| if ((data >> 27) & 0xf) { | |
| /* Read the old val */ | |
| idx = srcu_read_lock(&vcpu->kvm->srcu); | |
| ret = kvm_io_bus_read(vcpu, KVM_IOCSR_BUS, addr, 4, &val); | |
| srcu_read_unlock(&vcpu->kvm->srcu, idx); | |
| if (unlikely(ret)) { | |
| kvm_pr_unimpl("%s: : read data from addr %llx failed\n", __func__, addr); | |
| return 0; | |
| } | |
| /* Construct the mask by scanning the bit 27-30 */ | |
| for (i = 0; i < 4; i++) { | |
| if (data & (BIT(27 + i))) | |
| mask |= (0xff << (i * 8)); | |
| } | |
| /* Save the old part of val */ | |
| val &= mask; | |
| } | |
| val |= ((uint32_t)(data >> 32) & ~mask); | |
| idx = srcu_read_lock(&vcpu->kvm->srcu); | |
| ret = kvm_io_bus_write(vcpu, KVM_IOCSR_BUS, addr, 4, &val); | |
| srcu_read_unlock(&vcpu->kvm->srcu, idx); | |
| if (unlikely(ret)) | |
| kvm_pr_unimpl("%s: : write data to addr %llx failed\n", __func__, addr); | |
| return 0; | |
| } | |
| static int any_send(struct kvm *kvm, uint64_t data) | |
| { | |
| int cpu, offset; | |
| struct kvm_vcpu *vcpu; | |
| cpu = ((data & 0xffffffff) >> 16) & 0x3ff; | |
| vcpu = kvm_get_vcpu_by_cpuid(kvm, cpu); | |
| if (unlikely(vcpu == NULL)) { | |
| kvm_pr_unimpl("%s: invalid target cpu: %d\n", __func__, cpu); | |
| return 0; | |
| } | |
| offset = data & 0xffff; | |
| return send_ipi_data(vcpu, offset, data); | |
| } | |
| static int loongarch_ipi_readl(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *val) | |
| { | |
| uint32_t offset; | |
| uint64_t res = 0; | |
| if (addr & (len - 1)) { | |
| *(uint64_t *)val = res; | |
| kvm_pr_unimpl("%s: ipi not aligned addr %llx len %d\n", __func__, addr, len); | |
| return 0; | |
| } | |
| offset = addr - IOCSR_IPI_BASE; | |
| switch (offset) { | |
| case IOCSR_IPI_STATUS: | |
| spin_lock(&vcpu->arch.ipi_state.lock); | |
| res = vcpu->arch.ipi_state.status; | |
| spin_unlock(&vcpu->arch.ipi_state.lock); | |
| break; | |
| case IOCSR_IPI_EN: | |
| spin_lock(&vcpu->arch.ipi_state.lock); | |
| res = vcpu->arch.ipi_state.en; | |
| spin_unlock(&vcpu->arch.ipi_state.lock); | |
| break; | |
| case IOCSR_IPI_SET: | |
| case IOCSR_IPI_CLEAR: | |
| break; | |
| case IOCSR_IPI_BUF_20 ... IOCSR_IPI_BUF_38 + 7: | |
| res = read_mailbox(vcpu, offset, len); | |
| break; | |
| default: | |
| kvm_pr_unimpl("%s: unknown addr: %llx\n", __func__, addr); | |
| break; | |
| } | |
| *(uint64_t *)val = res; | |
| return 0; | |
| } | |
| static int loongarch_ipi_writel(struct kvm_vcpu *vcpu, gpa_t addr, int len, const void *val) | |
| { | |
| uint64_t data; | |
| uint32_t offset; | |
| data = *(uint64_t *)val; | |
| if (addr & (len - 1)) { | |
| kvm_pr_unimpl("%s: ipi not aligned addr %llx len %d\n", __func__, addr, len); | |
| return 0; | |
| } | |
| offset = addr - IOCSR_IPI_BASE; | |
| switch (offset) { | |
| case IOCSR_IPI_STATUS: | |
| break; | |
| case IOCSR_IPI_EN: | |
| spin_lock(&vcpu->arch.ipi_state.lock); | |
| vcpu->arch.ipi_state.en = data; | |
| spin_unlock(&vcpu->arch.ipi_state.lock); | |
| break; | |
| case IOCSR_IPI_SET: | |
| ipi_set(vcpu, data); | |
| break; | |
| case IOCSR_IPI_CLEAR: | |
| /* Just clear the status of the current vcpu */ | |
| ipi_clear(vcpu, data); | |
| break; | |
| case IOCSR_IPI_BUF_20 ... IOCSR_IPI_BUF_38 + 7: | |
| if (offset + len > IOCSR_IPI_BUF_38 + 8) { | |
| kvm_pr_unimpl("%s: invalid offset or len: offset = %d, len = %d\n", | |
| __func__, offset, len); | |
| break; | |
| } | |
| write_mailbox(vcpu, offset, data, len); | |
| break; | |
| case IOCSR_IPI_SEND: | |
| ipi_send(vcpu->kvm, data); | |
| break; | |
| case IOCSR_MAIL_SEND: | |
| mail_send(vcpu->kvm, data); | |
| break; | |
| case IOCSR_ANY_SEND: | |
| any_send(vcpu->kvm, data); | |
| break; | |
| default: | |
| kvm_pr_unimpl("%s: unknown addr: %llx\n", __func__, addr); | |
| break; | |
| } | |
| return 0; | |
| } | |
| static int kvm_ipi_read(struct kvm_vcpu *vcpu, | |
| struct kvm_io_device *dev, | |
| gpa_t addr, int len, void *val) | |
| { | |
| vcpu->stat.ipi_read_exits++; | |
| return loongarch_ipi_readl(vcpu, addr, len, val); | |
| } | |
| static int kvm_ipi_write(struct kvm_vcpu *vcpu, | |
| struct kvm_io_device *dev, | |
| gpa_t addr, int len, const void *val) | |
| { | |
| vcpu->stat.ipi_write_exits++; | |
| return loongarch_ipi_writel(vcpu, addr, len, val); | |
| } | |
| static const struct kvm_io_device_ops kvm_ipi_ops = { | |
| .read = kvm_ipi_read, | |
| .write = kvm_ipi_write, | |
| }; | |
| static int kvm_ipi_regs_access(struct kvm_device *dev, | |
| struct kvm_device_attr *attr, | |
| bool is_write) | |
| { | |
| int len = 4; | |
| int cpu, addr; | |
| uint64_t val; | |
| void *p = NULL; | |
| struct kvm_vcpu *vcpu; | |
| cpu = (attr->attr >> 16) & 0x3ff; | |
| addr = attr->attr & 0xff; | |
| vcpu = kvm_get_vcpu_by_id(dev->kvm, cpu); | |
| if (unlikely(vcpu == NULL)) { | |
| kvm_pr_unimpl("%s: invalid target cpu: %d\n", __func__, cpu); | |
| return -EINVAL; | |
| } | |
| switch (addr) { | |
| case IOCSR_IPI_STATUS: | |
| p = &vcpu->arch.ipi_state.status; | |
| break; | |
| case IOCSR_IPI_EN: | |
| p = &vcpu->arch.ipi_state.en; | |
| break; | |
| case IOCSR_IPI_SET: | |
| p = &vcpu->arch.ipi_state.set; | |
| break; | |
| case IOCSR_IPI_CLEAR: | |
| p = &vcpu->arch.ipi_state.clear; | |
| break; | |
| case IOCSR_IPI_BUF_20: | |
| p = &vcpu->arch.ipi_state.buf[0]; | |
| len = 8; | |
| break; | |
| case IOCSR_IPI_BUF_28: | |
| p = &vcpu->arch.ipi_state.buf[1]; | |
| len = 8; | |
| break; | |
| case IOCSR_IPI_BUF_30: | |
| p = &vcpu->arch.ipi_state.buf[2]; | |
| len = 8; | |
| break; | |
| case IOCSR_IPI_BUF_38: | |
| p = &vcpu->arch.ipi_state.buf[3]; | |
| len = 8; | |
| break; | |
| default: | |
| kvm_pr_unimpl("%s: unknown ipi register, addr = %d\n", __func__, addr); | |
| return -EINVAL; | |
| } | |
| if (is_write) { | |
| if (len == 4) { | |
| if (get_user(val, (uint32_t __user *)attr->addr)) | |
| return -EFAULT; | |
| *(uint32_t *)p = (uint32_t)val; | |
| } else if (len == 8) { | |
| if (get_user(val, (uint64_t __user *)attr->addr)) | |
| return -EFAULT; | |
| *(uint64_t *)p = val; | |
| } | |
| } else { | |
| if (len == 4) { | |
| val = *(uint32_t *)p; | |
| return put_user(val, (uint32_t __user *)attr->addr); | |
| } else if (len == 8) { | |
| val = *(uint64_t *)p; | |
| return put_user(val, (uint64_t __user *)attr->addr); | |
| } | |
| } | |
| return 0; | |
| } | |
| static int kvm_ipi_get_attr(struct kvm_device *dev, | |
| struct kvm_device_attr *attr) | |
| { | |
| switch (attr->group) { | |
| case KVM_DEV_LOONGARCH_IPI_GRP_REGS: | |
| return kvm_ipi_regs_access(dev, attr, false); | |
| default: | |
| kvm_pr_unimpl("%s: unknown group (%d)\n", __func__, attr->group); | |
| return -EINVAL; | |
| } | |
| } | |
| static int kvm_ipi_set_attr(struct kvm_device *dev, | |
| struct kvm_device_attr *attr) | |
| { | |
| switch (attr->group) { | |
| case KVM_DEV_LOONGARCH_IPI_GRP_REGS: | |
| return kvm_ipi_regs_access(dev, attr, true); | |
| default: | |
| kvm_pr_unimpl("%s: unknown group (%d)\n", __func__, attr->group); | |
| return -EINVAL; | |
| } | |
| } | |
| static int kvm_ipi_create(struct kvm_device *dev, u32 type) | |
| { | |
| int ret; | |
| struct kvm *kvm; | |
| struct kvm_io_device *device; | |
| struct loongarch_ipi *s; | |
| if (!dev) { | |
| kvm_pr_unimpl("%s: kvm_device ptr is invalid!\n", __func__); | |
| return -EINVAL; | |
| } | |
| kvm = dev->kvm; | |
| if (kvm->arch.ipi) { | |
| kvm_pr_unimpl("%s: LoongArch IPI has already been created!\n", __func__); | |
| return -EINVAL; | |
| } | |
| s = kzalloc_obj(struct loongarch_ipi); | |
| if (!s) | |
| return -ENOMEM; | |
| spin_lock_init(&s->lock); | |
| s->kvm = kvm; | |
| /* | |
| * Initialize IOCSR device | |
| */ | |
| device = &s->device; | |
| kvm_iodevice_init(device, &kvm_ipi_ops); | |
| mutex_lock(&kvm->slots_lock); | |
| ret = kvm_io_bus_register_dev(kvm, KVM_IOCSR_BUS, IOCSR_IPI_BASE, IOCSR_IPI_SIZE, device); | |
| mutex_unlock(&kvm->slots_lock); | |
| if (ret < 0) { | |
| kvm_pr_unimpl("%s: Initialize IOCSR dev failed, ret = %d\n", __func__, ret); | |
| goto err; | |
| } | |
| kvm->arch.ipi = s; | |
| return 0; | |
| err: | |
| kfree(s); | |
| return -EFAULT; | |
| } | |
| static void kvm_ipi_destroy(struct kvm_device *dev) | |
| { | |
| struct kvm *kvm; | |
| struct loongarch_ipi *ipi; | |
| if (!dev || !dev->kvm || !dev->kvm->arch.ipi) | |
| return; | |
| kvm = dev->kvm; | |
| ipi = kvm->arch.ipi; | |
| mutex_lock(&kvm->slots_lock); | |
| kvm_io_bus_unregister_dev(kvm, KVM_IOCSR_BUS, &ipi->device); | |
| mutex_unlock(&kvm->slots_lock); | |
| kfree(ipi); | |
| kfree(dev); | |
| } | |
| static struct kvm_device_ops kvm_ipi_dev_ops = { | |
| .name = "kvm-loongarch-ipi", | |
| .create = kvm_ipi_create, | |
| .destroy = kvm_ipi_destroy, | |
| .set_attr = kvm_ipi_set_attr, | |
| .get_attr = kvm_ipi_get_attr, | |
| }; | |
| int kvm_loongarch_register_ipi_device(void) | |
| { | |
| return kvm_register_device_ops(&kvm_ipi_dev_ops, KVM_DEV_TYPE_LOONGARCH_IPI); | |
| } | |
| void kvm_loongarch_unregister_ipi_device(void) | |
| { | |
| kvm_unregister_device_ops(KVM_DEV_TYPE_LOONGARCH_IPI); | |
| } |