| /* SPDX-License-Identifier: GPL-2.0 */ | |
| #ifndef __ARCH_S390_PERCPU__ | |
| #define __ARCH_S390_PERCPU__ | |
| #include <linux/preempt.h> | |
| #include <asm/cmpxchg.h> | |
| #include <asm/march.h> | |
| /* | |
| * s390 uses its own implementation for per cpu data, the offset of | |
| * the cpu local data area is cached in the cpu's lowcore memory. | |
| */ | |
| #define __my_cpu_offset get_lowcore()->percpu_offset | |
| #define arch_raw_cpu_ptr(_ptr) \ | |
| ({ \ | |
| unsigned long lc_percpu, tcp_ptr__; \ | |
| \ | |
| tcp_ptr__ = (__force unsigned long)(_ptr); \ | |
| lc_percpu = offsetof(struct lowcore, percpu_offset); \ | |
| asm_inline volatile( \ | |
| ALTERNATIVE("ag %[__ptr__],%[offzero](%%r0)\n", \ | |
| "ag %[__ptr__],%[offalt](%%r0)\n", \ | |
| ALT_FEATURE(MFEATURE_LOWCORE)) \ | |
| : [__ptr__] "+d" (tcp_ptr__) \ | |
| : [offzero] "i" (lc_percpu), \ | |
| [offalt] "i" (lc_percpu + LOWCORE_ALT_ADDRESS), \ | |
| "m" (((struct lowcore *)0)->percpu_offset) \ | |
| : "cc"); \ | |
| (TYPEOF_UNQUAL(*(_ptr)) __force __kernel *)tcp_ptr__; \ | |
| }) | |
| /* | |
| * We use a compare-and-swap loop since that uses less cpu cycles than | |
| * disabling and enabling interrupts like the generic variant would do. | |
| */ | |
| #define arch_this_cpu_to_op_simple(pcp, val, op) \ | |
| ({ \ | |
| typedef typeof(pcp) pcp_op_T__; \ | |
| pcp_op_T__ old__, new__, prev__; \ | |
| pcp_op_T__ *ptr__; \ | |
| preempt_disable_notrace(); \ | |
| ptr__ = raw_cpu_ptr(&(pcp)); \ | |
| prev__ = READ_ONCE(*ptr__); \ | |
| do { \ | |
| old__ = prev__; \ | |
| new__ = old__ op (val); \ | |
| prev__ = cmpxchg(ptr__, old__, new__); \ | |
| } while (prev__ != old__); \ | |
| preempt_enable_notrace(); \ | |
| new__; \ | |
| }) | |
| #define this_cpu_add_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) | |
| #define this_cpu_add_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) | |
| #define this_cpu_add_return_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) | |
| #define this_cpu_add_return_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) | |
| #define this_cpu_and_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, &) | |
| #define this_cpu_and_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, &) | |
| #define this_cpu_or_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |) | |
| #define this_cpu_or_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |) | |
| /* | |
| * Macros to be used for percpu code section based on atomic instructions. | |
| * | |
| * Avoid the need to use preempt_disable() / preempt_disable() pairs and the | |
| * conditional preempt_schedule_notrace() function calls which come with | |
| * this. The idea is that this_cpu operations based on atomic instructions are | |
| * guarded with mviy instructions: | |
| * | |
| * - The first mviy instruction writes the register number, which contains the | |
| * percpu address variable to lowcore. This also indicates that a percpu | |
| * code section is executed. | |
| * | |
| * - The first mviy instruction following the mviy instruction must be the ag | |
| * instruction which adds the percpu offset to the percpu address register. | |
| * | |
| * - Afterwards the atomic percpu operation follows. | |
| * | |
| * - Then a second mviy instruction writes a zero to lowcore, which indicates | |
| * the end of the percpu code section. | |
| * | |
| * - In case of an interrupt/exception/nmi the register number which was | |
| * written to lowcore is copied to the exception frame (pt_regs), and a zero | |
| * is written to lowcore. | |
| * | |
| * - On return to the previous context it is checked if a percpu code section | |
| * was executed (saved register number not zero), and if the process was | |
| * migrated to a different cpu. If the percpu offset was already added to | |
| * the percpu address register (instruction address does _not_ point to the | |
| * ag instruction) the content of the percpu address register is adjusted so | |
| * it points to percpu variable of the new cpu. | |
| * | |
| * Inline assemblies making use of this typically have a code sequence like: | |
| * | |
| * MVIY_PERCPU(...) <- start of percpu code section | |
| * AG_ALT(...) <- add percpu offset; must be the second instruction | |
| * atomic_op <- atomic op | |
| * MVIY_ALT(...) <- end of percpu code section | |
| */ | |
| #define MVIY_PERCPU(disp, dispalt, reg) \ | |
| ".macro GEN_MVIY disp reg\n" \ | |
| ".irp rs,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15\n" \ | |
| " .ifc \\reg,%%r\\rs\n" \ | |
| " mviy \\disp(%%r0),\\rs\n" \ | |
| " .endif\n" \ | |
| ".endr\n" \ | |
| ".endm\n" \ | |
| ALTERNATIVE("GEN_MVIY " disp ", " reg "\n", \ | |
| "GEN_MVIY " dispalt ", " reg "\n", \ | |
| ALT_FEATURE(MFEATURE_LOWCORE)) \ | |
| ".purgem GEN_MVIY\n" | |
| #define MVIY_ALT(disp, dispalt) \ | |
| ALTERNATIVE(" mviy " disp "(%%r0),0\n", \ | |
| " mviy " dispalt "(%%r0),0\n", \ | |
| ALT_FEATURE(MFEATURE_LOWCORE)) | |
| #define AG_ALT(disp, dispalt, reg) \ | |
| ALTERNATIVE(" ag " reg ", " disp "(%%r0)\n", \ | |
| " ag " reg ", " dispalt "(%%r0)\n", \ | |
| ALT_FEATURE(MFEATURE_LOWCORE)) | |
| #ifndef MARCH_HAS_Z196_FEATURES | |
| #define this_cpu_add_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) | |
| #define this_cpu_add_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) | |
| #define this_cpu_add_return_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) | |
| #define this_cpu_add_return_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) | |
| #define this_cpu_and_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, &) | |
| #define this_cpu_and_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, &) | |
| #define this_cpu_or_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |) | |
| #define this_cpu_or_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |) | |
| #else /* MARCH_HAS_Z196_FEATURES */ | |
| #define arch_this_cpu_add(pcp, val, op1, op2, szcast) \ | |
| do { \ | |
| unsigned long lc_pcpr, lc_pcpo; \ | |
| typedef typeof(pcp) pcp_op_T__; \ | |
| pcp_op_T__ val__ = (val); \ | |
| pcp_op_T__ old__, *ptr__; \ | |
| \ | |
| lc_pcpr = offsetof(struct lowcore, percpu_register); \ | |
| lc_pcpo = offsetof(struct lowcore, percpu_offset); \ | |
| ptr__ = PERCPU_PTR(&(pcp)); \ | |
| if (__builtin_constant_p(val__) && \ | |
| ((szcast)val__ > -129) && ((szcast)val__ < 128)) { \ | |
| asm volatile( \ | |
| MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\ | |
| AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]") \ | |
| op2 " 0(%[ptr__]),%[val__]\n" \ | |
| MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]") \ | |
| : [ptr__] "+&a" (ptr__), "+m" (*ptr__), \ | |
| "=m" (((struct lowcore *)0)->percpu_register) \ | |
| : [val__] "i" ((szcast)val__), \ | |
| [disppcpr] "i" (lc_pcpr), \ | |
| [disppcpo] "i" (lc_pcpo), \ | |
| [dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS), \ | |
| [dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS), \ | |
| "m" (((struct lowcore *)0)->percpu_offset) \ | |
| : "cc"); \ | |
| } else { \ | |
| asm volatile( \ | |
| MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\ | |
| AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]") \ | |
| op1 " %[old__],%[val__],0(%[ptr__])\n" \ | |
| MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]") \ | |
| : [old__] "=&d" (old__), \ | |
| [ptr__] "+&a" (ptr__), "+m" (*ptr__), \ | |
| "=m" (((struct lowcore *)0)->percpu_register) \ | |
| : [val__] "d" (val__), \ | |
| [disppcpr] "i" (lc_pcpr), \ | |
| [disppcpo] "i" (lc_pcpo), \ | |
| [dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS), \ | |
| [dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS), \ | |
| "m" (((struct lowcore *)0)->percpu_offset) \ | |
| : "cc"); \ | |
| } \ | |
| } while (0) | |
| #define this_cpu_add_4(pcp, val) arch_this_cpu_add(pcp, val, "laa", "asi", int) | |
| #define this_cpu_add_8(pcp, val) arch_this_cpu_add(pcp, val, "laag", "agsi", long) | |
| #define arch_this_cpu_add_return(pcp, val, op) \ | |
| ({ \ | |
| unsigned long lc_pcpr, lc_pcpo; \ | |
| typedef typeof(pcp) pcp_op_T__; \ | |
| pcp_op_T__ val__ = (val); \ | |
| pcp_op_T__ old__, *ptr__; \ | |
| \ | |
| lc_pcpr = offsetof(struct lowcore, percpu_register); \ | |
| lc_pcpo = offsetof(struct lowcore, percpu_offset); \ | |
| ptr__ = PERCPU_PTR(&(pcp)); \ | |
| asm_inline volatile( \ | |
| MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\ | |
| AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]") \ | |
| op " %[old__],%[val__],0(%[ptr__])\n" \ | |
| MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]") \ | |
| : [old__] "=&d" (old__), \ | |
| [ptr__] "+&a" (ptr__), "+m" (*ptr__), \ | |
| "=m" (((struct lowcore *)0)->percpu_register) \ | |
| : [val__] "d" (val__), \ | |
| [disppcpr] "i" (lc_pcpr), \ | |
| [disppcpo] "i" (lc_pcpo), \ | |
| [dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS), \ | |
| [dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS), \ | |
| "m" (((struct lowcore *)0)->percpu_offset) \ | |
| : "cc"); \ | |
| old__ + val__; \ | |
| }) | |
| #define this_cpu_add_return_4(pcp, val) arch_this_cpu_add_return(pcp, val, "laa") | |
| #define this_cpu_add_return_8(pcp, val) arch_this_cpu_add_return(pcp, val, "laag") | |
| #define arch_this_cpu_to_op(pcp, val, op) \ | |
| do { \ | |
| unsigned long lc_pcpr, lc_pcpo; \ | |
| typedef typeof(pcp) pcp_op_T__; \ | |
| pcp_op_T__ val__ = (val); \ | |
| pcp_op_T__ old__, *ptr__; \ | |
| \ | |
| lc_pcpr = offsetof(struct lowcore, percpu_register); \ | |
| lc_pcpo = offsetof(struct lowcore, percpu_offset); \ | |
| ptr__ = PERCPU_PTR(&(pcp)); \ | |
| asm_inline volatile( \ | |
| MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\ | |
| AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]") \ | |
| op " %[old__],%[val__],0(%[ptr__])\n" \ | |
| MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]") \ | |
| : [old__] "=&d" (old__), \ | |
| [ptr__] "+&a" (ptr__), "+m" (*ptr__), \ | |
| "=m" (((struct lowcore *)0)->percpu_register) \ | |
| : [val__] "d" (val__), \ | |
| [disppcpr] "i" (lc_pcpr), \ | |
| [disppcpo] "i" (lc_pcpo), \ | |
| [dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS), \ | |
| [dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS), \ | |
| "m" (((struct lowcore *)0)->percpu_offset) \ | |
| : "cc"); \ | |
| } while (0) | |
| #define this_cpu_and_4(pcp, val) arch_this_cpu_to_op(pcp, val, "lan") | |
| #define this_cpu_and_8(pcp, val) arch_this_cpu_to_op(pcp, val, "lang") | |
| #define this_cpu_or_4(pcp, val) arch_this_cpu_to_op(pcp, val, "lao") | |
| #define this_cpu_or_8(pcp, val) arch_this_cpu_to_op(pcp, val, "laog") | |
| #endif /* MARCH_HAS_Z196_FEATURES */ | |
| #define arch_this_cpu_read(pcp, op) \ | |
| ({ \ | |
| unsigned long lc_pcpr, lc_pcpo, res__; \ | |
| typedef typeof(pcp) pcp_op_T__; \ | |
| pcp_op_T__ *ptr__; \ | |
| \ | |
| lc_pcpr = offsetof(struct lowcore, percpu_register); \ | |
| lc_pcpo = offsetof(struct lowcore, percpu_offset); \ | |
| ptr__ = PERCPU_PTR(&(pcp)); \ | |
| asm_inline volatile( \ | |
| MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\ | |
| AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]") \ | |
| op " %[res__],0(%[ptr__])\n" \ | |
| MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]") \ | |
| : [res__] "=&d" (res__), [ptr__] "+&a" (ptr__), \ | |
| "=m" (((struct lowcore *)0)->percpu_register) \ | |
| : [disppcpr] "i" (lc_pcpr), \ | |
| [disppcpo] "i" (lc_pcpo), \ | |
| [dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS), \ | |
| [dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS), \ | |
| "m" (*ptr__), \ | |
| "m" (((struct lowcore *)0)->percpu_offset) \ | |
| : "cc"); \ | |
| (pcp_op_T__)res__; \ | |
| }) | |
| #define this_cpu_read_1(pcp) arch_this_cpu_read(pcp, "llgc") | |
| #define this_cpu_read_2(pcp) arch_this_cpu_read(pcp, "llgh") | |
| #define this_cpu_read_4(pcp) arch_this_cpu_read(pcp, "llgf") | |
| #define this_cpu_read_8(pcp) arch_this_cpu_read(pcp, "lg") | |
| #define arch_this_cpu_write(pcp, val, op) \ | |
| do { \ | |
| unsigned long lc_pcpr, lc_pcpo; \ | |
| typedef typeof(pcp) pcp_op_T__; \ | |
| pcp_op_T__ *ptr__, val__ = (val); \ | |
| \ | |
| lc_pcpr = offsetof(struct lowcore, percpu_register); \ | |
| lc_pcpo = offsetof(struct lowcore, percpu_offset); \ | |
| ptr__ = PERCPU_PTR(&(pcp)); \ | |
| asm_inline volatile( \ | |
| MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\ | |
| AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]") \ | |
| op " %[val__],0(%[ptr__])\n" \ | |
| MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]") \ | |
| : [ptr__] "+&a" (ptr__), "=m" (*ptr__), \ | |
| "=m" (((struct lowcore *)0)->percpu_register) \ | |
| : [val__] "d" (val__), \ | |
| [disppcpr] "i" (lc_pcpr), \ | |
| [disppcpo] "i" (lc_pcpo), \ | |
| [dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS), \ | |
| [dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS), \ | |
| "m" (((struct lowcore *)0)->percpu_offset) \ | |
| : "cc"); \ | |
| } while (0) | |
| #define this_cpu_write_1(pcp, val) arch_this_cpu_write(pcp, val, "stc") | |
| #define this_cpu_write_2(pcp, val) arch_this_cpu_write(pcp, val, "sth") | |
| #define this_cpu_write_4(pcp, val) arch_this_cpu_write(pcp, val, "st") | |
| #define this_cpu_write_8(pcp, val) arch_this_cpu_write(pcp, val, "stg") | |
| #define arch_this_cpu_cmpxchg(pcp, oval, nval) \ | |
| ({ \ | |
| typedef typeof(pcp) pcp_op_T__; \ | |
| pcp_op_T__ ret__; \ | |
| pcp_op_T__ *ptr__; \ | |
| preempt_disable_notrace(); \ | |
| ptr__ = raw_cpu_ptr(&(pcp)); \ | |
| ret__ = cmpxchg(ptr__, oval, nval); \ | |
| preempt_enable_notrace(); \ | |
| ret__; \ | |
| }) | |
| #define this_cpu_cmpxchg_1(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval) | |
| #define this_cpu_cmpxchg_2(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval) | |
| #define this_cpu_cmpxchg_4(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval) | |
| #define this_cpu_cmpxchg_8(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval) | |
| #define this_cpu_cmpxchg64(pcp, o, n) this_cpu_cmpxchg_8(pcp, o, n) | |
| #define this_cpu_cmpxchg128(pcp, oval, nval) \ | |
| ({ \ | |
| typedef typeof(pcp) pcp_op_T__; \ | |
| u128 old__, new__, ret__; \ | |
| pcp_op_T__ *ptr__; \ | |
| old__ = oval; \ | |
| new__ = nval; \ | |
| preempt_disable_notrace(); \ | |
| ptr__ = raw_cpu_ptr(&(pcp)); \ | |
| ret__ = cmpxchg128((void *)ptr__, old__, new__); \ | |
| preempt_enable_notrace(); \ | |
| ret__; \ | |
| }) | |
| #define arch_this_cpu_xchg(pcp, nval) \ | |
| ({ \ | |
| typeof(pcp) *ptr__; \ | |
| typeof(pcp) ret__; \ | |
| preempt_disable_notrace(); \ | |
| ptr__ = raw_cpu_ptr(&(pcp)); \ | |
| ret__ = xchg(ptr__, nval); \ | |
| preempt_enable_notrace(); \ | |
| ret__; \ | |
| }) | |
| #define this_cpu_xchg_1(pcp, nval) arch_this_cpu_xchg(pcp, nval) | |
| #define this_cpu_xchg_2(pcp, nval) arch_this_cpu_xchg(pcp, nval) | |
| #define this_cpu_xchg_4(pcp, nval) arch_this_cpu_xchg(pcp, nval) | |
| #define this_cpu_xchg_8(pcp, nval) arch_this_cpu_xchg(pcp, nval) | |
| #include <asm-generic/percpu.h> | |
| #endif /* __ARCH_S390_PERCPU__ */ |