arch/riscv/kvm/vcpu_insn.c - third_party/linux - Git at Google

 // SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (C) 2019 Western Digital Corporation or its affiliates.
  * Copyright (c) 2022 Ventana Micro Systems Inc.
  */

 #include <linux/bitops.h>
 #include <linux/kvm_host.h>

 #include <asm/cpufeature.h>
 #include <asm/insn.h>

 struct insn_func {
 	unsigned long mask;
 	unsigned long match;
 	/*
 	 * Possible return values are as follows:
 	 * 1) Returns < 0 for error case
 	 * 2) Returns 0 for exit to user-space
 	 * 3) Returns 1 to continue with next sepc
 	 * 4) Returns 2 to continue with same sepc
 	 * 5) Returns 3 to inject illegal instruction trap and continue
 	 * 6) Returns 4 to inject virtual instruction trap and continue
 	 *
 	 * Use enum kvm_insn_return for return values
 	 */
 	int (*func)(struct kvm_vcpu *vcpu, struct kvm_run *run, ulong insn);
 };

 static int truly_illegal_insn(struct kvm_vcpu *vcpu, struct kvm_run *run,
 			      ulong insn)
 {
 	struct kvm_cpu_trap utrap = { 0 };

 	/* Redirect trap to Guest VCPU */
 	utrap.sepc = vcpu->arch.guest_context.sepc;
 	utrap.scause = EXC_INST_ILLEGAL;
 	utrap.stval = insn;
 	utrap.htval = 0;
 	utrap.htinst = 0;
 	kvm_riscv_vcpu_trap_redirect(vcpu, &utrap);

 	return 1;
 }

 static int truly_virtual_insn(struct kvm_vcpu *vcpu, struct kvm_run *run,
 			      ulong insn)
 {
 	struct kvm_cpu_trap utrap = { 0 };

 	/* Redirect trap to Guest VCPU */
 	utrap.sepc = vcpu->arch.guest_context.sepc;
 	utrap.scause = EXC_VIRTUAL_INST_FAULT;
 	utrap.stval = insn;
 	utrap.htval = 0;
 	utrap.htinst = 0;
 	kvm_riscv_vcpu_trap_redirect(vcpu, &utrap);

 	return 1;
 }

 /**
  * kvm_riscv_vcpu_wfi -- Emulate wait for interrupt (WFI) behaviour
  *
  * @vcpu: The VCPU pointer
  */
 void kvm_riscv_vcpu_wfi(struct kvm_vcpu *vcpu)
 {
 	if (!kvm_arch_vcpu_runnable(vcpu)) {
 		kvm_vcpu_srcu_read_unlock(vcpu);
 		kvm_vcpu_halt(vcpu);
 		kvm_vcpu_srcu_read_lock(vcpu);
 	}
 }

 static int wfi_insn(struct kvm_vcpu *vcpu, struct kvm_run *run, ulong insn)
 {
 	vcpu->stat.wfi_exit_stat++;
 	kvm_riscv_vcpu_wfi(vcpu);
 	return KVM_INSN_CONTINUE_NEXT_SEPC;
 }

 static int wrs_insn(struct kvm_vcpu *vcpu, struct kvm_run *run, ulong insn)
 {
 	vcpu->stat.wrs_exit_stat++;
 	kvm_vcpu_on_spin(vcpu, vcpu->arch.guest_context.sstatus & SR_SPP);
 	return KVM_INSN_CONTINUE_NEXT_SEPC;
 }

 struct csr_func {
 	unsigned int base;
 	unsigned int count;
 	/*
 	 * Possible return values are as same as "func" callback in
 	 * "struct insn_func".
 	 */
 	int (*func)(struct kvm_vcpu *vcpu, unsigned int csr_num,
 		    unsigned long *val, unsigned long new_val,
 		    unsigned long wr_mask);
 };

 static int seed_csr_rmw(struct kvm_vcpu *vcpu, unsigned int csr_num,
 			unsigned long *val, unsigned long new_val,
 			unsigned long wr_mask)
 {
 	if (!riscv_isa_extension_available(vcpu->arch.isa, ZKR))
 		return KVM_INSN_ILLEGAL_TRAP;

 	return KVM_INSN_EXIT_TO_USER_SPACE;
 }

 static const struct csr_func csr_funcs[] = {
 	KVM_RISCV_VCPU_AIA_CSR_FUNCS
 	KVM_RISCV_VCPU_HPMCOUNTER_CSR_FUNCS
 	{ .base = CSR_SEED, .count = 1, .func = seed_csr_rmw },
 };

 /**
  * kvm_riscv_vcpu_csr_return -- Handle CSR read/write after user space
  *				emulation or in-kernel emulation
  *
  * @vcpu: The VCPU pointer
  * @run:  The VCPU run struct containing the CSR data
  *
  * Returns > 0 upon failure and 0 upon success
  */
 int kvm_riscv_vcpu_csr_return(struct kvm_vcpu *vcpu, struct kvm_run *run)
 {
 	ulong insn;

 	if (vcpu->arch.csr_decode.return_handled)
 		return 0;
 	vcpu->arch.csr_decode.return_handled = 1;

 	/* Update destination register for CSR reads */
 	insn = vcpu->arch.csr_decode.insn;
 	if ((insn >> SH_RD) & MASK_RX)
 		SET_RD(insn, &vcpu->arch.guest_context,
 		       run->riscv_csr.ret_value);

 	/* Move to next instruction */
 	vcpu->arch.guest_context.sepc += INSN_LEN(insn);

 	return 0;
 }

 static int csr_insn(struct kvm_vcpu *vcpu, struct kvm_run *run, ulong insn)
 {
 	int i, rc = KVM_INSN_ILLEGAL_TRAP;
 	unsigned int csr_num = insn >> SH_RS2;
 	unsigned int rs1_num = (insn >> SH_RS1) & MASK_RX;
 	ulong rs1_val = GET_RS1(insn, &vcpu->arch.guest_context);
 	const struct csr_func *tcfn, *cfn = NULL;
 	ulong val = 0, wr_mask = 0, new_val = 0;

 	/* Decode the CSR instruction */
 	switch (GET_FUNCT3(insn)) {
 	case GET_FUNCT3(INSN_MATCH_CSRRW):
 		wr_mask = -1UL;
 		new_val = rs1_val;
 		break;
 	case GET_FUNCT3(INSN_MATCH_CSRRS):
 		wr_mask = rs1_val;
 		new_val = -1UL;
 		break;
 	case GET_FUNCT3(INSN_MATCH_CSRRC):
 		wr_mask = rs1_val;
 		new_val = 0;
 		break;
 	case GET_FUNCT3(INSN_MATCH_CSRRWI):
 		wr_mask = -1UL;
 		new_val = rs1_num;
 		break;
 	case GET_FUNCT3(INSN_MATCH_CSRRSI):
 		wr_mask = rs1_num;
 		new_val = -1UL;
 		break;
 	case GET_FUNCT3(INSN_MATCH_CSRRCI):
 		wr_mask = rs1_num;
 		new_val = 0;
 		break;
 	default:
 		return rc;
 	}

 	/* Save instruction decode info */
 	vcpu->arch.csr_decode.insn = insn;
 	vcpu->arch.csr_decode.return_handled = 0;

 	/* Update CSR details in kvm_run struct */
 	run->riscv_csr.csr_num = csr_num;
 	run->riscv_csr.new_value = new_val;
 	run->riscv_csr.write_mask = wr_mask;
 	run->riscv_csr.ret_value = 0;

 	/* Find in-kernel CSR function */
 	for (i = 0; i < ARRAY_SIZE(csr_funcs); i++) {
 		tcfn = &csr_funcs[i];
 		if ((tcfn->base <= csr_num) &&
 		    (csr_num < (tcfn->base + tcfn->count))) {
 			cfn = tcfn;
 			break;
 		}
 	}

 	/* First try in-kernel CSR emulation */
 	if (cfn && cfn->func) {
 		rc = cfn->func(vcpu, csr_num, &val, new_val, wr_mask);
 		if (rc > KVM_INSN_EXIT_TO_USER_SPACE) {
 			if (rc == KVM_INSN_CONTINUE_NEXT_SEPC) {
 				run->riscv_csr.ret_value = val;
 				vcpu->stat.csr_exit_kernel++;
 				kvm_riscv_vcpu_csr_return(vcpu, run);
 				rc = KVM_INSN_CONTINUE_SAME_SEPC;
 			}
 			return rc;
 		}
 	}

 	/* Exit to user-space for CSR emulation */
 	if (rc <= KVM_INSN_EXIT_TO_USER_SPACE) {
 		vcpu->stat.csr_exit_user++;
 		run->exit_reason = KVM_EXIT_RISCV_CSR;
 	}

 	return rc;
 }

 static const struct insn_func system_opcode_funcs[] = {
 	{
 		.mask  = INSN_MASK_CSRRW,
 		.match = INSN_MATCH_CSRRW,
 		.func  = csr_insn,
 	},
 	{
 		.mask  = INSN_MASK_CSRRS,
 		.match = INSN_MATCH_CSRRS,
 		.func  = csr_insn,
 	},
 	{
 		.mask  = INSN_MASK_CSRRC,
 		.match = INSN_MATCH_CSRRC,
 		.func  = csr_insn,
 	},
 	{
 		.mask  = INSN_MASK_CSRRWI,
 		.match = INSN_MATCH_CSRRWI,
 		.func  = csr_insn,
 	},
 	{
 		.mask  = INSN_MASK_CSRRSI,
 		.match = INSN_MATCH_CSRRSI,
 		.func  = csr_insn,
 	},
 	{
 		.mask  = INSN_MASK_CSRRCI,
 		.match = INSN_MATCH_CSRRCI,
 		.func  = csr_insn,
 	},
 	{
 		.mask  = INSN_MASK_WFI,
 		.match = INSN_MATCH_WFI,
 		.func  = wfi_insn,
 	},
 	{
 		.mask  = INSN_MASK_WRS,
 		.match = INSN_MATCH_WRS,
 		.func  = wrs_insn,
 	},
 };

 static int system_opcode_insn(struct kvm_vcpu *vcpu, struct kvm_run *run,
 			      ulong insn)
 {
 	int i, rc = KVM_INSN_ILLEGAL_TRAP;
 	const struct insn_func *ifn;

 	for (i = 0; i < ARRAY_SIZE(system_opcode_funcs); i++) {
 		ifn = &system_opcode_funcs[i];
 		if ((insn & ifn->mask) == ifn->match) {
 			rc = ifn->func(vcpu, run, insn);
 			break;
 		}
 	}

 	switch (rc) {
 	case KVM_INSN_ILLEGAL_TRAP:
 		return truly_illegal_insn(vcpu, run, insn);
 	case KVM_INSN_VIRTUAL_TRAP:
 		return truly_virtual_insn(vcpu, run, insn);
 	case KVM_INSN_CONTINUE_NEXT_SEPC:
 		vcpu->arch.guest_context.sepc += INSN_LEN(insn);
 		break;
 	default:
 		break;
 	}

 	return (rc <= 0) ? rc : 1;
 }

 static bool is_load_guest_page_fault(unsigned long scause)
 {
 	/**
 	 * If a g-stage page fault occurs, the direct approach
 	 * is to let the g-stage page fault handler handle it
 	 * naturally, however, calling the g-stage page fault
 	 * handler here seems rather strange.
 	 * Considering this is a corner case, we can directly
 	 * return to the guest and re-execute the same PC, this
 	 * will trigger a g-stage page fault again and then the
 	 * regular g-stage page fault handler will populate
 	 * g-stage page table.
 	 */
 	return (scause == EXC_LOAD_GUEST_PAGE_FAULT);
 }

 /**
  * kvm_riscv_vcpu_virtual_insn -- Handle virtual instruction trap
  *
  * @vcpu: The VCPU pointer
  * @run:  The VCPU run struct containing the mmio data
  * @trap: Trap details
  *
  * Returns > 0 to continue run-loop
  * Returns   0 to exit run-loop and handle in user-space.
  * Returns < 0 to report failure and exit run-loop
  */
 int kvm_riscv_vcpu_virtual_insn(struct kvm_vcpu *vcpu, struct kvm_run *run,
 				struct kvm_cpu_trap *trap)
 {
 	unsigned long insn = trap->stval;
 	struct kvm_cpu_trap utrap = { 0 };
 	struct kvm_cpu_context *ct;

 	if (unlikely(INSN_IS_16BIT(insn))) {
 		if (insn == 0) {
 			ct = &vcpu->arch.guest_context;
 			insn = kvm_riscv_vcpu_unpriv_read(vcpu, true,
 							  ct->sepc,
 							  &utrap);
 			if (utrap.scause) {
 				if (is_load_guest_page_fault(utrap.scause))
 					return 1;
 				utrap.sepc = ct->sepc;
 				kvm_riscv_vcpu_trap_redirect(vcpu, &utrap);
 				return 1;
 			}
 		}
 		if (INSN_IS_16BIT(insn))
 			return truly_illegal_insn(vcpu, run, insn);
 	}

 	switch ((insn & INSN_OPCODE_MASK) >> INSN_OPCODE_SHIFT) {
 	case INSN_OPCODE_SYSTEM:
 		return system_opcode_insn(vcpu, run, insn);
 	default:
 		return truly_illegal_insn(vcpu, run, insn);
 	}
 }

 /**
  * kvm_riscv_vcpu_mmio_load -- Emulate MMIO load instruction
  *
  * @vcpu: The VCPU pointer
  * @run:  The VCPU run struct containing the mmio data
  * @fault_addr: Guest physical address to load
  * @htinst: Transformed encoding of the load instruction
  *
  * Returns > 0 to continue run-loop
  * Returns   0 to exit run-loop and handle in user-space.
  * Returns < 0 to report failure and exit run-loop
  */
 int kvm_riscv_vcpu_mmio_load(struct kvm_vcpu *vcpu, struct kvm_run *run,
 			     unsigned long fault_addr,
 			     unsigned long htinst)
 {
 	u8 data_buf[8];
 	unsigned long insn;
 	int shift = 0, len = 0, insn_len = 0;
 	struct kvm_cpu_trap utrap = { 0 };
 	struct kvm_cpu_context *ct = &vcpu->arch.guest_context;

 	/* Determine trapped instruction */
 	if (htinst & 0x1) {
 		/*
 		 * Bit[0] == 1 implies trapped instruction value is
 		 * transformed instruction or custom instruction.
 		 */
 		insn = htinst | INSN_16BIT_MASK;
 		insn_len = (htinst & BIT(1)) ? INSN_LEN(insn) : 2;
 	} else {
 		/*
 		 * Bit[0] == 0 implies trapped instruction value is
 		 * zero or special value.
 		 */
 		insn = kvm_riscv_vcpu_unpriv_read(vcpu, true, ct->sepc,
 						  &utrap);
 		if (utrap.scause) {
 			if (is_load_guest_page_fault(utrap.scause))
 				return 1;
 			/* Redirect trap if we failed to read instruction */
 			utrap.sepc = ct->sepc;
 			kvm_riscv_vcpu_trap_redirect(vcpu, &utrap);
 			return 1;
 		}
 		insn_len = INSN_LEN(insn);
 	}

 	/* Decode length of MMIO and shift */
 	if ((insn & INSN_MASK_LW) == INSN_MATCH_LW) {
 		len = 4;
 		shift = 8 * (sizeof(ulong) - len);
 	} else if ((insn & INSN_MASK_LB) == INSN_MATCH_LB) {
 		len = 1;
 		shift = 8 * (sizeof(ulong) - len);
 	} else if ((insn & INSN_MASK_LBU) == INSN_MATCH_LBU) {
 		len = 1;
 		shift = 8 * (sizeof(ulong) - len);
 #ifdef CONFIG_64BIT
 	} else if ((insn & INSN_MASK_LD) == INSN_MATCH_LD) {
 		len = 8;
 		shift = 8 * (sizeof(ulong) - len);
 	} else if ((insn & INSN_MASK_LWU) == INSN_MATCH_LWU) {
 		len = 4;
 #endif
 	} else if ((insn & INSN_MASK_LH) == INSN_MATCH_LH) {
 		len = 2;
 		shift = 8 * (sizeof(ulong) - len);
 	} else if ((insn & INSN_MASK_LHU) == INSN_MATCH_LHU) {
 		len = 2;
 #ifdef CONFIG_64BIT
 	} else if ((insn & INSN_MASK_C_LD) == INSN_MATCH_C_LD) {
 		len = 8;
 		shift = 8 * (sizeof(ulong) - len);
 		insn = RVC_RS2S(insn) << SH_RD;
 	} else if ((insn & INSN_MASK_C_LDSP) == INSN_MATCH_C_LDSP &&
 		   ((insn >> SH_RD) & 0x1f)) {
 		len = 8;
 		shift = 8 * (sizeof(ulong) - len);
 #endif
 	} else if ((insn & INSN_MASK_C_LW) == INSN_MATCH_C_LW) {
 		len = 4;
 		shift = 8 * (sizeof(ulong) - len);
 		insn = RVC_RS2S(insn) << SH_RD;
 	} else if ((insn & INSN_MASK_C_LWSP) == INSN_MATCH_C_LWSP &&
 		   ((insn >> SH_RD) & 0x1f)) {
 		len = 4;
 		shift = 8 * (sizeof(ulong) - len);
 	} else {
 		return -EOPNOTSUPP;
 	}

 	/* Fault address should be aligned to length of MMIO */
 	if (fault_addr & (len - 1))
 		return -EIO;

 	/* Save instruction decode info */
 	vcpu->arch.mmio_decode.insn = insn;
 	vcpu->arch.mmio_decode.insn_len = insn_len;
 	vcpu->arch.mmio_decode.shift = shift;
 	vcpu->arch.mmio_decode.len = len;
 	vcpu->arch.mmio_decode.return_handled = 0;

 	/* Update MMIO details in kvm_run struct */
 	run->mmio.is_write = false;
 	run->mmio.phys_addr = fault_addr;
 	run->mmio.len = len;

 	/* Try to handle MMIO access in the kernel */
 	if (!kvm_io_bus_read(vcpu, KVM_MMIO_BUS, fault_addr, len, data_buf)) {
 		/* Successfully handled MMIO access in the kernel so resume */
 		memcpy(run->mmio.data, data_buf, len);
 		vcpu->stat.mmio_exit_kernel++;
 		kvm_riscv_vcpu_mmio_return(vcpu, run);
 		return 1;
 	}

 	/* Exit to userspace for MMIO emulation */
 	vcpu->stat.mmio_exit_user++;
 	run->exit_reason = KVM_EXIT_MMIO;

 	return 0;
 }

 /**
  * kvm_riscv_vcpu_mmio_store -- Emulate MMIO store instruction
  *
  * @vcpu: The VCPU pointer
  * @run:  The VCPU run struct containing the mmio data
  * @fault_addr: Guest physical address to store
  * @htinst: Transformed encoding of the store instruction
  *
  * Returns > 0 to continue run-loop
  * Returns   0 to exit run-loop and handle in user-space.
  * Returns < 0 to report failure and exit run-loop
  */
 int kvm_riscv_vcpu_mmio_store(struct kvm_vcpu *vcpu, struct kvm_run *run,
 			      unsigned long fault_addr,
 			      unsigned long htinst)
 {
 	u8 data8;
 	u16 data16;
 	u32 data32;
 	u64 data64;
 	ulong data;
 	unsigned long insn;
 	int len = 0, insn_len = 0;
 	struct kvm_cpu_trap utrap = { 0 };
 	struct kvm_cpu_context *ct = &vcpu->arch.guest_context;

 	/* Determine trapped instruction */
 	if (htinst & 0x1) {
 		/*
 		 * Bit[0] == 1 implies trapped instruction value is
 		 * transformed instruction or custom instruction.
 		 */
 		insn = htinst | INSN_16BIT_MASK;
 		insn_len = (htinst & BIT(1)) ? INSN_LEN(insn) : 2;
 	} else {
 		/*
 		 * Bit[0] == 0 implies trapped instruction value is
 		 * zero or special value.
 		 */
 		insn = kvm_riscv_vcpu_unpriv_read(vcpu, true, ct->sepc,
 						  &utrap);
 		if (utrap.scause) {
 			if (is_load_guest_page_fault(utrap.scause))
 				return 1;
 			/* Redirect trap if we failed to read instruction */
 			utrap.sepc = ct->sepc;
 			kvm_riscv_vcpu_trap_redirect(vcpu, &utrap);
 			return 1;
 		}
 		insn_len = INSN_LEN(insn);
 	}

 	data = GET_RS2(insn, &vcpu->arch.guest_context);
 	data8 = data16 = data32 = data64 = data;

 	if ((insn & INSN_MASK_SW) == INSN_MATCH_SW) {
 		len = 4;
 	} else if ((insn & INSN_MASK_SB) == INSN_MATCH_SB) {
 		len = 1;
 #ifdef CONFIG_64BIT
 	} else if ((insn & INSN_MASK_SD) == INSN_MATCH_SD) {
 		len = 8;
 #endif
 	} else if ((insn & INSN_MASK_SH) == INSN_MATCH_SH) {
 		len = 2;
 #ifdef CONFIG_64BIT
 	} else if ((insn & INSN_MASK_C_SD) == INSN_MATCH_C_SD) {
 		len = 8;
 		data64 = GET_RS2S(insn, &vcpu->arch.guest_context);
 	} else if ((insn & INSN_MASK_C_SDSP) == INSN_MATCH_C_SDSP &&
 		   ((insn >> SH_RD) & 0x1f)) {
 		len = 8;
 		data64 = GET_RS2C(insn, &vcpu->arch.guest_context);
 #endif
 	} else if ((insn & INSN_MASK_C_SW) == INSN_MATCH_C_SW) {
 		len = 4;
 		data32 = GET_RS2S(insn, &vcpu->arch.guest_context);
 	} else if ((insn & INSN_MASK_C_SWSP) == INSN_MATCH_C_SWSP &&
 		   ((insn >> SH_RD) & 0x1f)) {
 		len = 4;
 		data32 = GET_RS2C(insn, &vcpu->arch.guest_context);
 	} else {
 		return -EOPNOTSUPP;
 	}

 	/* Fault address should be aligned to length of MMIO */
 	if (fault_addr & (len - 1))
 		return -EIO;

 	/* Save instruction decode info */
 	vcpu->arch.mmio_decode.insn = insn;
 	vcpu->arch.mmio_decode.insn_len = insn_len;
 	vcpu->arch.mmio_decode.shift = 0;
 	vcpu->arch.mmio_decode.len = len;
 	vcpu->arch.mmio_decode.return_handled = 0;

 	/* Copy data to kvm_run instance */
 	switch (len) {
 	case 1:
 		*((u8 *)run->mmio.data) = data8;
 		break;
 	case 2:
 		*((u16 *)run->mmio.data) = data16;
 		break;
 	case 4:
 		*((u32 *)run->mmio.data) = data32;
 		break;
 	case 8:
 		*((u64 *)run->mmio.data) = data64;
 		break;
 	default:
 		return -EOPNOTSUPP;
 	}

 	/* Update MMIO details in kvm_run struct */
 	run->mmio.is_write = true;
 	run->mmio.phys_addr = fault_addr;
 	run->mmio.len = len;

 	/* Try to handle MMIO access in the kernel */
 	if (!kvm_io_bus_write(vcpu, KVM_MMIO_BUS,
 			      fault_addr, len, run->mmio.data)) {
 		/* Successfully handled MMIO access in the kernel so resume */
 		vcpu->stat.mmio_exit_kernel++;
 		kvm_riscv_vcpu_mmio_return(vcpu, run);
 		return 1;
 	}

 	/* Exit to userspace for MMIO emulation */
 	vcpu->stat.mmio_exit_user++;
 	run->exit_reason = KVM_EXIT_MMIO;

 	return 0;
 }

 /**
  * kvm_riscv_vcpu_mmio_return -- Handle MMIO loads after user space emulation
  *			     or in-kernel IO emulation
  *
  * @vcpu: The VCPU pointer
  * @run:  The VCPU run struct containing the mmio data
  */
 int kvm_riscv_vcpu_mmio_return(struct kvm_vcpu *vcpu, struct kvm_run *run)
 {
 	u8 data8;
 	u16 data16;
 	u32 data32;
 	u64 data64;
 	ulong insn;
 	int len, shift;

 	if (vcpu->arch.mmio_decode.return_handled)
 		return 0;

 	vcpu->arch.mmio_decode.return_handled = 1;
 	insn = vcpu->arch.mmio_decode.insn;

 	if (run->mmio.is_write)
 		goto done;

 	len = vcpu->arch.mmio_decode.len;
 	shift = vcpu->arch.mmio_decode.shift;

 	switch (len) {
 	case 1:
 		data8 = *((u8 *)run->mmio.data);
 		SET_RD(insn, &vcpu->arch.guest_context,
 			(ulong)data8 << shift >> shift);
 		break;
 	case 2:
 		data16 = *((u16 *)run->mmio.data);
 		SET_RD(insn, &vcpu->arch.guest_context,
 			(ulong)data16 << shift >> shift);
 		break;
 	case 4:
 		data32 = *((u32 *)run->mmio.data);
 		SET_RD(insn, &vcpu->arch.guest_context,
 			(ulong)data32 << shift >> shift);
 		break;
 	case 8:
 		data64 = *((u64 *)run->mmio.data);
 		SET_RD(insn, &vcpu->arch.guest_context,
 			(ulong)data64 << shift >> shift);
 		break;
 	default:
 		return -EOPNOTSUPP;
 	}

 done:
 	/* Move to next instruction */
 	vcpu->arch.guest_context.sepc += vcpu->arch.mmio_decode.insn_len;

 	return 0;
 }
	// SPDX-License-Identifier: GPL-2.0
	/*
	* Copyright (C) 2019 Western Digital Corporation or its affiliates.
	* Copyright (c) 2022 Ventana Micro Systems Inc.
	*/

	#include <linux/bitops.h>
	#include <linux/kvm_host.h>

	#include <asm/cpufeature.h>
	#include <asm/insn.h>

	struct insn_func {
	unsigned long mask;
	unsigned long match;
	/*
	* Possible return values are as follows:
	* 1) Returns < 0 for error case
	* 2) Returns 0 for exit to user-space
	* 3) Returns 1 to continue with next sepc
	* 4) Returns 2 to continue with same sepc
	* 5) Returns 3 to inject illegal instruction trap and continue
	* 6) Returns 4 to inject virtual instruction trap and continue
	*
	* Use enum kvm_insn_return for return values
	*/
	int (func)(struct kvm_vcpu vcpu, struct kvm_run *run, ulong insn);
	};

	static int truly_illegal_insn(struct kvm_vcpu vcpu, struct kvm_run run,
	ulong insn)
	{
	struct kvm_cpu_trap utrap = { 0 };

	/* Redirect trap to Guest VCPU */
	utrap.sepc = vcpu->arch.guest_context.sepc;
	utrap.scause = EXC_INST_ILLEGAL;
	utrap.stval = insn;
	utrap.htval = 0;
	utrap.htinst = 0;
	kvm_riscv_vcpu_trap_redirect(vcpu, &utrap);

	return 1;
	}

	static int truly_virtual_insn(struct kvm_vcpu vcpu, struct kvm_run run,
	ulong insn)
	{
	struct kvm_cpu_trap utrap = { 0 };

	/* Redirect trap to Guest VCPU */
	utrap.sepc = vcpu->arch.guest_context.sepc;
	utrap.scause = EXC_VIRTUAL_INST_FAULT;
	utrap.stval = insn;
	utrap.htval = 0;
	utrap.htinst = 0;
	kvm_riscv_vcpu_trap_redirect(vcpu, &utrap);

	return 1;
	}

	/**
	* kvm_riscv_vcpu_wfi -- Emulate wait for interrupt (WFI) behaviour
	*
	* @vcpu: The VCPU pointer
	*/
	void kvm_riscv_vcpu_wfi(struct kvm_vcpu *vcpu)
	{
	if (!kvm_arch_vcpu_runnable(vcpu)) {
	kvm_vcpu_srcu_read_unlock(vcpu);
	kvm_vcpu_halt(vcpu);
	kvm_vcpu_srcu_read_lock(vcpu);
	}
	}

	static int wfi_insn(struct kvm_vcpu vcpu, struct kvm_run run, ulong insn)
	{
	vcpu->stat.wfi_exit_stat++;
	kvm_riscv_vcpu_wfi(vcpu);
	return KVM_INSN_CONTINUE_NEXT_SEPC;
	}

	static int wrs_insn(struct kvm_vcpu vcpu, struct kvm_run run, ulong insn)
	{
	vcpu->stat.wrs_exit_stat++;
	kvm_vcpu_on_spin(vcpu, vcpu->arch.guest_context.sstatus & SR_SPP);
	return KVM_INSN_CONTINUE_NEXT_SEPC;
	}

	struct csr_func {
	unsigned int base;
	unsigned int count;
	/*
	* Possible return values are as same as "func" callback in
	* "struct insn_func".
	*/
	int (func)(struct kvm_vcpu vcpu, unsigned int csr_num,
	unsigned long *val, unsigned long new_val,
	unsigned long wr_mask);
	};

	static int seed_csr_rmw(struct kvm_vcpu *vcpu, unsigned int csr_num,
	unsigned long *val, unsigned long new_val,
	unsigned long wr_mask)
	{
	if (!riscv_isa_extension_available(vcpu->arch.isa, ZKR))
	return KVM_INSN_ILLEGAL_TRAP;

	return KVM_INSN_EXIT_TO_USER_SPACE;
	}

	static const struct csr_func csr_funcs[] = {
	KVM_RISCV_VCPU_AIA_CSR_FUNCS
	KVM_RISCV_VCPU_HPMCOUNTER_CSR_FUNCS
	{ .base = CSR_SEED, .count = 1, .func = seed_csr_rmw },
	};

	/**
	* kvm_riscv_vcpu_csr_return -- Handle CSR read/write after user space
	* emulation or in-kernel emulation
	*
	* @vcpu: The VCPU pointer
	* @run: The VCPU run struct containing the CSR data
	*
	* Returns > 0 upon failure and 0 upon success
	*/
	int kvm_riscv_vcpu_csr_return(struct kvm_vcpu vcpu, struct kvm_run run)
	{
	ulong insn;

	if (vcpu->arch.csr_decode.return_handled)
	return 0;
	vcpu->arch.csr_decode.return_handled = 1;

	/* Update destination register for CSR reads */
	insn = vcpu->arch.csr_decode.insn;
	if ((insn >> SH_RD) & MASK_RX)
	SET_RD(insn, &vcpu->arch.guest_context,
	run->riscv_csr.ret_value);

	/* Move to next instruction */
	vcpu->arch.guest_context.sepc += INSN_LEN(insn);

	return 0;
	}

	static int csr_insn(struct kvm_vcpu vcpu, struct kvm_run run, ulong insn)
	{
	int i, rc = KVM_INSN_ILLEGAL_TRAP;
	unsigned int csr_num = insn >> SH_RS2;
	unsigned int rs1_num = (insn >> SH_RS1) & MASK_RX;
	ulong rs1_val = GET_RS1(insn, &vcpu->arch.guest_context);
	const struct csr_func tcfn, cfn = NULL;
	ulong val = 0, wr_mask = 0, new_val = 0;

	/* Decode the CSR instruction */
	switch (GET_FUNCT3(insn)) {
	case GET_FUNCT3(INSN_MATCH_CSRRW):
	wr_mask = -1UL;
	new_val = rs1_val;
	break;
	case GET_FUNCT3(INSN_MATCH_CSRRS):
	wr_mask = rs1_val;
	new_val = -1UL;
	break;
	case GET_FUNCT3(INSN_MATCH_CSRRC):
	wr_mask = rs1_val;
	new_val = 0;
	break;
	case GET_FUNCT3(INSN_MATCH_CSRRWI):
	wr_mask = -1UL;
	new_val = rs1_num;
	break;
	case GET_FUNCT3(INSN_MATCH_CSRRSI):
	wr_mask = rs1_num;
	new_val = -1UL;
	break;
	case GET_FUNCT3(INSN_MATCH_CSRRCI):
	wr_mask = rs1_num;
	new_val = 0;
	break;
	default:
	return rc;
	}

	/* Save instruction decode info */
	vcpu->arch.csr_decode.insn = insn;
	vcpu->arch.csr_decode.return_handled = 0;

	/* Update CSR details in kvm_run struct */
	run->riscv_csr.csr_num = csr_num;
	run->riscv_csr.new_value = new_val;
	run->riscv_csr.write_mask = wr_mask;
	run->riscv_csr.ret_value = 0;

	/* Find in-kernel CSR function */
	for (i = 0; i < ARRAY_SIZE(csr_funcs); i++) {
	tcfn = &csr_funcs[i];
	if ((tcfn->base <= csr_num) &&
	(csr_num < (tcfn->base + tcfn->count))) {
	cfn = tcfn;
	break;
	}
	}

	/* First try in-kernel CSR emulation */
	if (cfn && cfn->func) {
	rc = cfn->func(vcpu, csr_num, &val, new_val, wr_mask);
	if (rc > KVM_INSN_EXIT_TO_USER_SPACE) {
	if (rc == KVM_INSN_CONTINUE_NEXT_SEPC) {
	run->riscv_csr.ret_value = val;
	vcpu->stat.csr_exit_kernel++;
	kvm_riscv_vcpu_csr_return(vcpu, run);
	rc = KVM_INSN_CONTINUE_SAME_SEPC;
	}
	return rc;
	}
	}

	/* Exit to user-space for CSR emulation */
	if (rc <= KVM_INSN_EXIT_TO_USER_SPACE) {
	vcpu->stat.csr_exit_user++;
	run->exit_reason = KVM_EXIT_RISCV_CSR;
	}

	return rc;
	}

	static const struct insn_func system_opcode_funcs[] = {
	{
	.mask = INSN_MASK_CSRRW,
	.match = INSN_MATCH_CSRRW,
	.func = csr_insn,
	},
	{
	.mask = INSN_MASK_CSRRS,
	.match = INSN_MATCH_CSRRS,
	.func = csr_insn,
	},
	{
	.mask = INSN_MASK_CSRRC,
	.match = INSN_MATCH_CSRRC,
	.func = csr_insn,
	},
	{
	.mask = INSN_MASK_CSRRWI,
	.match = INSN_MATCH_CSRRWI,
	.func = csr_insn,
	},
	{
	.mask = INSN_MASK_CSRRSI,
	.match = INSN_MATCH_CSRRSI,
	.func = csr_insn,
	},
	{
	.mask = INSN_MASK_CSRRCI,
	.match = INSN_MATCH_CSRRCI,
	.func = csr_insn,
	},
	{
	.mask = INSN_MASK_WFI,
	.match = INSN_MATCH_WFI,
	.func = wfi_insn,
	},
	{
	.mask = INSN_MASK_WRS,
	.match = INSN_MATCH_WRS,
	.func = wrs_insn,
	},
	};

	static int system_opcode_insn(struct kvm_vcpu vcpu, struct kvm_run run,
	ulong insn)
	{
	int i, rc = KVM_INSN_ILLEGAL_TRAP;
	const struct insn_func *ifn;

	for (i = 0; i < ARRAY_SIZE(system_opcode_funcs); i++) {
	ifn = &system_opcode_funcs[i];
	if ((insn & ifn->mask) == ifn->match) {
	rc = ifn->func(vcpu, run, insn);
	break;
	}
	}

	switch (rc) {
	case KVM_INSN_ILLEGAL_TRAP:
	return truly_illegal_insn(vcpu, run, insn);
	case KVM_INSN_VIRTUAL_TRAP:
	return truly_virtual_insn(vcpu, run, insn);
	case KVM_INSN_CONTINUE_NEXT_SEPC:
	vcpu->arch.guest_context.sepc += INSN_LEN(insn);
	break;
	default:
	break;
	}

	return (rc <= 0) ? rc : 1;
	}

	static bool is_load_guest_page_fault(unsigned long scause)
	{
	/**
	* If a g-stage page fault occurs, the direct approach
	* is to let the g-stage page fault handler handle it
	* naturally, however, calling the g-stage page fault
	* handler here seems rather strange.
	* Considering this is a corner case, we can directly
	* return to the guest and re-execute the same PC, this
	* will trigger a g-stage page fault again and then the
	* regular g-stage page fault handler will populate
	* g-stage page table.
	*/
	return (scause == EXC_LOAD_GUEST_PAGE_FAULT);
	}

	/**
	* kvm_riscv_vcpu_virtual_insn -- Handle virtual instruction trap
	*
	* @vcpu: The VCPU pointer
	* @run: The VCPU run struct containing the mmio data
	* @trap: Trap details
	*
	* Returns > 0 to continue run-loop
	* Returns 0 to exit run-loop and handle in user-space.
	* Returns < 0 to report failure and exit run-loop
	*/
	int kvm_riscv_vcpu_virtual_insn(struct kvm_vcpu vcpu, struct kvm_run run,
	struct kvm_cpu_trap *trap)
	{
	unsigned long insn = trap->stval;
	struct kvm_cpu_trap utrap = { 0 };
	struct kvm_cpu_context *ct;

	if (unlikely(INSN_IS_16BIT(insn))) {
	if (insn == 0) {
	ct = &vcpu->arch.guest_context;
	insn = kvm_riscv_vcpu_unpriv_read(vcpu, true,
	ct->sepc,
	&utrap);
	if (utrap.scause) {
	if (is_load_guest_page_fault(utrap.scause))
	return 1;
	utrap.sepc = ct->sepc;
	kvm_riscv_vcpu_trap_redirect(vcpu, &utrap);
	return 1;
	}
	}
	if (INSN_IS_16BIT(insn))
	return truly_illegal_insn(vcpu, run, insn);
	}

	switch ((insn & INSN_OPCODE_MASK) >> INSN_OPCODE_SHIFT) {
	case INSN_OPCODE_SYSTEM:
	return system_opcode_insn(vcpu, run, insn);
	default:
	return truly_illegal_insn(vcpu, run, insn);
	}
	}

	/**
	* kvm_riscv_vcpu_mmio_load -- Emulate MMIO load instruction
	*
	* @vcpu: The VCPU pointer
	* @run: The VCPU run struct containing the mmio data
	* @fault_addr: Guest physical address to load
	* @htinst: Transformed encoding of the load instruction
	*
	* Returns > 0 to continue run-loop
	* Returns 0 to exit run-loop and handle in user-space.
	* Returns < 0 to report failure and exit run-loop
	*/
	int kvm_riscv_vcpu_mmio_load(struct kvm_vcpu vcpu, struct kvm_run run,
	unsigned long fault_addr,
	unsigned long htinst)
	{
	u8 data_buf[8];
	unsigned long insn;
	int shift = 0, len = 0, insn_len = 0;
	struct kvm_cpu_trap utrap = { 0 };
	struct kvm_cpu_context *ct = &vcpu->arch.guest_context;

	/* Determine trapped instruction */
	if (htinst & 0x1) {
	/*
	* Bit[0] == 1 implies trapped instruction value is
	* transformed instruction or custom instruction.
	*/
	insn = htinst \| INSN_16BIT_MASK;
	insn_len = (htinst & BIT(1)) ? INSN_LEN(insn) : 2;
	} else {
	/*
	* Bit[0] == 0 implies trapped instruction value is
	* zero or special value.
	*/
	insn = kvm_riscv_vcpu_unpriv_read(vcpu, true, ct->sepc,
	&utrap);
	if (utrap.scause) {
	if (is_load_guest_page_fault(utrap.scause))
	return 1;
	/* Redirect trap if we failed to read instruction */
	utrap.sepc = ct->sepc;
	kvm_riscv_vcpu_trap_redirect(vcpu, &utrap);
	return 1;
	}
	insn_len = INSN_LEN(insn);
	}

	/* Decode length of MMIO and shift */
	if ((insn & INSN_MASK_LW) == INSN_MATCH_LW) {
	len = 4;
	shift = 8 * (sizeof(ulong) - len);
	} else if ((insn & INSN_MASK_LB) == INSN_MATCH_LB) {
	len = 1;
	shift = 8 * (sizeof(ulong) - len);
	} else if ((insn & INSN_MASK_LBU) == INSN_MATCH_LBU) {
	len = 1;
	shift = 8 * (sizeof(ulong) - len);
	#ifdef CONFIG_64BIT
	} else if ((insn & INSN_MASK_LD) == INSN_MATCH_LD) {
	len = 8;
	shift = 8 * (sizeof(ulong) - len);
	} else if ((insn & INSN_MASK_LWU) == INSN_MATCH_LWU) {
	len = 4;
	#endif
	} else if ((insn & INSN_MASK_LH) == INSN_MATCH_LH) {
	len = 2;
	shift = 8 * (sizeof(ulong) - len);
	} else if ((insn & INSN_MASK_LHU) == INSN_MATCH_LHU) {
	len = 2;
	#ifdef CONFIG_64BIT
	} else if ((insn & INSN_MASK_C_LD) == INSN_MATCH_C_LD) {
	len = 8;
	shift = 8 * (sizeof(ulong) - len);
	insn = RVC_RS2S(insn) << SH_RD;
	} else if ((insn & INSN_MASK_C_LDSP) == INSN_MATCH_C_LDSP &&
	((insn >> SH_RD) & 0x1f)) {
	len = 8;
	shift = 8 * (sizeof(ulong) - len);
	#endif
	} else if ((insn & INSN_MASK_C_LW) == INSN_MATCH_C_LW) {
	len = 4;
	shift = 8 * (sizeof(ulong) - len);
	insn = RVC_RS2S(insn) << SH_RD;
	} else if ((insn & INSN_MASK_C_LWSP) == INSN_MATCH_C_LWSP &&
	((insn >> SH_RD) & 0x1f)) {
	len = 4;
	shift = 8 * (sizeof(ulong) - len);
	} else {
	return -EOPNOTSUPP;
	}

	/* Fault address should be aligned to length of MMIO */
	if (fault_addr & (len - 1))
	return -EIO;

	/* Save instruction decode info */
	vcpu->arch.mmio_decode.insn = insn;
	vcpu->arch.mmio_decode.insn_len = insn_len;
	vcpu->arch.mmio_decode.shift = shift;
	vcpu->arch.mmio_decode.len = len;
	vcpu->arch.mmio_decode.return_handled = 0;

	/* Update MMIO details in kvm_run struct */
	run->mmio.is_write = false;
	run->mmio.phys_addr = fault_addr;
	run->mmio.len = len;

	/* Try to handle MMIO access in the kernel */
	if (!kvm_io_bus_read(vcpu, KVM_MMIO_BUS, fault_addr, len, data_buf)) {
	/* Successfully handled MMIO access in the kernel so resume */
	memcpy(run->mmio.data, data_buf, len);
	vcpu->stat.mmio_exit_kernel++;
	kvm_riscv_vcpu_mmio_return(vcpu, run);
	return 1;
	}

	/* Exit to userspace for MMIO emulation */
	vcpu->stat.mmio_exit_user++;
	run->exit_reason = KVM_EXIT_MMIO;

	return 0;
	}

	/**
	* kvm_riscv_vcpu_mmio_store -- Emulate MMIO store instruction
	*
	* @vcpu: The VCPU pointer
	* @run: The VCPU run struct containing the mmio data
	* @fault_addr: Guest physical address to store
	* @htinst: Transformed encoding of the store instruction
	*
	* Returns > 0 to continue run-loop
	* Returns 0 to exit run-loop and handle in user-space.
	* Returns < 0 to report failure and exit run-loop
	*/
	int kvm_riscv_vcpu_mmio_store(struct kvm_vcpu vcpu, struct kvm_run run,
	unsigned long fault_addr,
	unsigned long htinst)
	{
	u8 data8;
	u16 data16;
	u32 data32;
	u64 data64;
	ulong data;
	unsigned long insn;
	int len = 0, insn_len = 0;
	struct kvm_cpu_trap utrap = { 0 };
	struct kvm_cpu_context *ct = &vcpu->arch.guest_context;

	/* Determine trapped instruction */
	if (htinst & 0x1) {
	/*
	* Bit[0] == 1 implies trapped instruction value is
	* transformed instruction or custom instruction.
	*/
	insn = htinst \| INSN_16BIT_MASK;
	insn_len = (htinst & BIT(1)) ? INSN_LEN(insn) : 2;
	} else {
	/*
	* Bit[0] == 0 implies trapped instruction value is
	* zero or special value.
	*/
	insn = kvm_riscv_vcpu_unpriv_read(vcpu, true, ct->sepc,
	&utrap);
	if (utrap.scause) {
	if (is_load_guest_page_fault(utrap.scause))
	return 1;
	/* Redirect trap if we failed to read instruction */
	utrap.sepc = ct->sepc;
	kvm_riscv_vcpu_trap_redirect(vcpu, &utrap);
	return 1;
	}
	insn_len = INSN_LEN(insn);
	}

	data = GET_RS2(insn, &vcpu->arch.guest_context);
	data8 = data16 = data32 = data64 = data;

	if ((insn & INSN_MASK_SW) == INSN_MATCH_SW) {
	len = 4;
	} else if ((insn & INSN_MASK_SB) == INSN_MATCH_SB) {
	len = 1;
	#ifdef CONFIG_64BIT
	} else if ((insn & INSN_MASK_SD) == INSN_MATCH_SD) {
	len = 8;
	#endif
	} else if ((insn & INSN_MASK_SH) == INSN_MATCH_SH) {
	len = 2;
	#ifdef CONFIG_64BIT
	} else if ((insn & INSN_MASK_C_SD) == INSN_MATCH_C_SD) {
	len = 8;
	data64 = GET_RS2S(insn, &vcpu->arch.guest_context);
	} else if ((insn & INSN_MASK_C_SDSP) == INSN_MATCH_C_SDSP &&
	((insn >> SH_RD) & 0x1f)) {
	len = 8;
	data64 = GET_RS2C(insn, &vcpu->arch.guest_context);
	#endif
	} else if ((insn & INSN_MASK_C_SW) == INSN_MATCH_C_SW) {
	len = 4;
	data32 = GET_RS2S(insn, &vcpu->arch.guest_context);
	} else if ((insn & INSN_MASK_C_SWSP) == INSN_MATCH_C_SWSP &&
	((insn >> SH_RD) & 0x1f)) {
	len = 4;
	data32 = GET_RS2C(insn, &vcpu->arch.guest_context);
	} else {
	return -EOPNOTSUPP;
	}

	/* Fault address should be aligned to length of MMIO */
	if (fault_addr & (len - 1))
	return -EIO;

	/* Save instruction decode info */
	vcpu->arch.mmio_decode.insn = insn;
	vcpu->arch.mmio_decode.insn_len = insn_len;
	vcpu->arch.mmio_decode.shift = 0;
	vcpu->arch.mmio_decode.len = len;
	vcpu->arch.mmio_decode.return_handled = 0;

	/* Copy data to kvm_run instance */
	switch (len) {
	case 1:
	((u8 )run->mmio.data) = data8;
	break;
	case 2:
	((u16 )run->mmio.data) = data16;
	break;
	case 4:
	((u32 )run->mmio.data) = data32;
	break;
	case 8:
	((u64 )run->mmio.data) = data64;
	break;
	default:
	return -EOPNOTSUPP;
	}

	/* Update MMIO details in kvm_run struct */
	run->mmio.is_write = true;
	run->mmio.phys_addr = fault_addr;
	run->mmio.len = len;

	/* Try to handle MMIO access in the kernel */
	if (!kvm_io_bus_write(vcpu, KVM_MMIO_BUS,
	fault_addr, len, run->mmio.data)) {
	/* Successfully handled MMIO access in the kernel so resume */
	vcpu->stat.mmio_exit_kernel++;
	kvm_riscv_vcpu_mmio_return(vcpu, run);
	return 1;
	}

	/* Exit to userspace for MMIO emulation */
	vcpu->stat.mmio_exit_user++;
	run->exit_reason = KVM_EXIT_MMIO;

	return 0;
	}

	/**
	* kvm_riscv_vcpu_mmio_return -- Handle MMIO loads after user space emulation
	* or in-kernel IO emulation
	*
	* @vcpu: The VCPU pointer
	* @run: The VCPU run struct containing the mmio data
	*/
	int kvm_riscv_vcpu_mmio_return(struct kvm_vcpu vcpu, struct kvm_run run)
	{
	u8 data8;
	u16 data16;
	u32 data32;
	u64 data64;
	ulong insn;
	int len, shift;

	if (vcpu->arch.mmio_decode.return_handled)
	return 0;

	vcpu->arch.mmio_decode.return_handled = 1;
	insn = vcpu->arch.mmio_decode.insn;

	if (run->mmio.is_write)
	goto done;

	len = vcpu->arch.mmio_decode.len;
	shift = vcpu->arch.mmio_decode.shift;

	switch (len) {
	case 1:
	data8 = ((u8 )run->mmio.data);
	SET_RD(insn, &vcpu->arch.guest_context,
	(ulong)data8 << shift >> shift);
	break;
	case 2:
	data16 = ((u16 )run->mmio.data);
	SET_RD(insn, &vcpu->arch.guest_context,
	(ulong)data16 << shift >> shift);
	break;
	case 4:
	data32 = ((u32 )run->mmio.data);
	SET_RD(insn, &vcpu->arch.guest_context,
	(ulong)data32 << shift >> shift);
	break;
	case 8:
	data64 = ((u64 )run->mmio.data);
	SET_RD(insn, &vcpu->arch.guest_context,
	(ulong)data64 << shift >> shift);
	break;
	default:
	return -EOPNOTSUPP;
	}

	done:
	/* Move to next instruction */
	vcpu->arch.guest_context.sepc += vcpu->arch.mmio_decode.insn_len;

	return 0;
	}