Greg Kroah-Hartman | b244131 | 2017-11-01 15:07:57 +0100 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0 |
Andy Lutomirski | b7b7c78 | 2015-07-20 11:49:06 -0400 | [diff] [blame] | 2 | #include <linux/perf_event.h> |
Peter Zijlstra | 06ce6e9 | 2018-04-20 14:23:36 +0200 | [diff] [blame] | 3 | #include <linux/nospec.h> |
Dave Hansen | 353bf60 | 2016-06-02 17:19:33 -0700 | [diff] [blame] | 4 | #include <asm/intel-family.h> |
Andy Lutomirski | b7b7c78 | 2015-07-20 11:49:06 -0400 | [diff] [blame] | 5 | |
| 6 | enum perf_msr_id { |
| 7 | PERF_MSR_TSC = 0, |
| 8 | PERF_MSR_APERF = 1, |
| 9 | PERF_MSR_MPERF = 2, |
| 10 | PERF_MSR_PPERF = 3, |
| 11 | PERF_MSR_SMI = 4, |
Huang Rui | 8a22426 | 2016-01-29 16:29:56 +0800 | [diff] [blame] | 12 | PERF_MSR_PTSC = 5, |
Huang Rui | aaf2488 | 2016-01-29 16:29:57 +0800 | [diff] [blame] | 13 | PERF_MSR_IRPERF = 6, |
Stephane Eranian | 9ae21dd | 2018-01-05 08:18:52 -0800 | [diff] [blame] | 14 | PERF_MSR_THERM = 7, |
| 15 | PERF_MSR_THERM_SNAP = 8, |
| 16 | PERF_MSR_THERM_UNIT = 9, |
Andy Lutomirski | b7b7c78 | 2015-07-20 11:49:06 -0400 | [diff] [blame] | 17 | PERF_MSR_EVENT_MAX, |
| 18 | }; |
| 19 | |
Geliang Tang | 7e5560a | 2015-09-24 04:48:53 -0700 | [diff] [blame] | 20 | static bool test_aperfmperf(int idx) |
Peter Zijlstra | 19b3340 | 2015-08-06 17:26:58 +0200 | [diff] [blame] | 21 | { |
| 22 | return boot_cpu_has(X86_FEATURE_APERFMPERF); |
| 23 | } |
Andy Lutomirski | b7b7c78 | 2015-07-20 11:49:06 -0400 | [diff] [blame] | 24 | |
Huang Rui | 8a22426 | 2016-01-29 16:29:56 +0800 | [diff] [blame] | 25 | static bool test_ptsc(int idx) |
| 26 | { |
| 27 | return boot_cpu_has(X86_FEATURE_PTSC); |
| 28 | } |
| 29 | |
Huang Rui | aaf2488 | 2016-01-29 16:29:57 +0800 | [diff] [blame] | 30 | static bool test_irperf(int idx) |
| 31 | { |
| 32 | return boot_cpu_has(X86_FEATURE_IRPERF); |
| 33 | } |
| 34 | |
Stephane Eranian | 9ae21dd | 2018-01-05 08:18:52 -0800 | [diff] [blame] | 35 | static bool test_therm_status(int idx) |
| 36 | { |
| 37 | return boot_cpu_has(X86_FEATURE_DTHERM); |
| 38 | } |
| 39 | |
Geliang Tang | 7e5560a | 2015-09-24 04:48:53 -0700 | [diff] [blame] | 40 | static bool test_intel(int idx) |
Peter Zijlstra | 19b3340 | 2015-08-06 17:26:58 +0200 | [diff] [blame] | 41 | { |
| 42 | if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL || |
| 43 | boot_cpu_data.x86 != 6) |
| 44 | return false; |
| 45 | |
| 46 | switch (boot_cpu_data.x86_model) { |
Dave Hansen | 353bf60 | 2016-06-02 17:19:33 -0700 | [diff] [blame] | 47 | case INTEL_FAM6_NEHALEM: |
Linus Torvalds | b325e04 | 2016-07-30 12:56:26 -0700 | [diff] [blame] | 48 | case INTEL_FAM6_NEHALEM_G: |
Dave Hansen | 353bf60 | 2016-06-02 17:19:33 -0700 | [diff] [blame] | 49 | case INTEL_FAM6_NEHALEM_EP: |
| 50 | case INTEL_FAM6_NEHALEM_EX: |
Peter Zijlstra | 19b3340 | 2015-08-06 17:26:58 +0200 | [diff] [blame] | 51 | |
Dave Hansen | 353bf60 | 2016-06-02 17:19:33 -0700 | [diff] [blame] | 52 | case INTEL_FAM6_WESTMERE: |
| 53 | case INTEL_FAM6_WESTMERE_EP: |
| 54 | case INTEL_FAM6_WESTMERE_EX: |
Peter Zijlstra | 19b3340 | 2015-08-06 17:26:58 +0200 | [diff] [blame] | 55 | |
Dave Hansen | 353bf60 | 2016-06-02 17:19:33 -0700 | [diff] [blame] | 56 | case INTEL_FAM6_SANDYBRIDGE: |
| 57 | case INTEL_FAM6_SANDYBRIDGE_X: |
Peter Zijlstra | 19b3340 | 2015-08-06 17:26:58 +0200 | [diff] [blame] | 58 | |
Dave Hansen | 353bf60 | 2016-06-02 17:19:33 -0700 | [diff] [blame] | 59 | case INTEL_FAM6_IVYBRIDGE: |
| 60 | case INTEL_FAM6_IVYBRIDGE_X: |
Peter Zijlstra | 19b3340 | 2015-08-06 17:26:58 +0200 | [diff] [blame] | 61 | |
Dave Hansen | 353bf60 | 2016-06-02 17:19:33 -0700 | [diff] [blame] | 62 | case INTEL_FAM6_HASWELL_CORE: |
| 63 | case INTEL_FAM6_HASWELL_X: |
| 64 | case INTEL_FAM6_HASWELL_ULT: |
| 65 | case INTEL_FAM6_HASWELL_GT3E: |
Peter Zijlstra | 19b3340 | 2015-08-06 17:26:58 +0200 | [diff] [blame] | 66 | |
Dave Hansen | 353bf60 | 2016-06-02 17:19:33 -0700 | [diff] [blame] | 67 | case INTEL_FAM6_BROADWELL_CORE: |
| 68 | case INTEL_FAM6_BROADWELL_XEON_D: |
| 69 | case INTEL_FAM6_BROADWELL_GT3E: |
| 70 | case INTEL_FAM6_BROADWELL_X: |
Peter Zijlstra | 19b3340 | 2015-08-06 17:26:58 +0200 | [diff] [blame] | 71 | |
Dave Hansen | 353bf60 | 2016-06-02 17:19:33 -0700 | [diff] [blame] | 72 | case INTEL_FAM6_ATOM_SILVERMONT1: |
| 73 | case INTEL_FAM6_ATOM_SILVERMONT2: |
| 74 | case INTEL_FAM6_ATOM_AIRMONT: |
Kan Liang | 1aaccc4 | 2017-09-08 17:34:48 -0400 | [diff] [blame] | 75 | |
| 76 | case INTEL_FAM6_ATOM_GOLDMONT: |
| 77 | case INTEL_FAM6_ATOM_DENVERTON: |
| 78 | |
| 79 | case INTEL_FAM6_ATOM_GEMINI_LAKE: |
| 80 | |
| 81 | case INTEL_FAM6_XEON_PHI_KNL: |
| 82 | case INTEL_FAM6_XEON_PHI_KNM: |
Peter Zijlstra | 19b3340 | 2015-08-06 17:26:58 +0200 | [diff] [blame] | 83 | if (idx == PERF_MSR_SMI) |
| 84 | return true; |
| 85 | break; |
| 86 | |
Dave Hansen | 353bf60 | 2016-06-02 17:19:33 -0700 | [diff] [blame] | 87 | case INTEL_FAM6_SKYLAKE_MOBILE: |
| 88 | case INTEL_FAM6_SKYLAKE_DESKTOP: |
Dave Hansen | 5134596 | 2016-06-02 17:19:35 -0700 | [diff] [blame] | 89 | case INTEL_FAM6_SKYLAKE_X: |
| 90 | case INTEL_FAM6_KABYLAKE_MOBILE: |
| 91 | case INTEL_FAM6_KABYLAKE_DESKTOP: |
Peter Zijlstra | 19b3340 | 2015-08-06 17:26:58 +0200 | [diff] [blame] | 92 | if (idx == PERF_MSR_SMI || idx == PERF_MSR_PPERF) |
| 93 | return true; |
| 94 | break; |
| 95 | } |
| 96 | |
| 97 | return false; |
| 98 | } |
| 99 | |
| 100 | struct perf_msr { |
| 101 | u64 msr; |
| 102 | struct perf_pmu_events_attr *attr; |
| 103 | bool (*test)(int idx); |
Andy Lutomirski | b7b7c78 | 2015-07-20 11:49:06 -0400 | [diff] [blame] | 104 | }; |
| 105 | |
Ingo Molnar | 9128d3e | 2018-01-05 08:18:52 -0800 | [diff] [blame] | 106 | PMU_EVENT_ATTR_STRING(tsc, evattr_tsc, "event=0x00" ); |
| 107 | PMU_EVENT_ATTR_STRING(aperf, evattr_aperf, "event=0x01" ); |
| 108 | PMU_EVENT_ATTR_STRING(mperf, evattr_mperf, "event=0x02" ); |
| 109 | PMU_EVENT_ATTR_STRING(pperf, evattr_pperf, "event=0x03" ); |
| 110 | PMU_EVENT_ATTR_STRING(smi, evattr_smi, "event=0x04" ); |
| 111 | PMU_EVENT_ATTR_STRING(ptsc, evattr_ptsc, "event=0x05" ); |
| 112 | PMU_EVENT_ATTR_STRING(irperf, evattr_irperf, "event=0x06" ); |
| 113 | PMU_EVENT_ATTR_STRING(cpu_thermal_margin, evattr_therm, "event=0x07" ); |
| 114 | PMU_EVENT_ATTR_STRING(cpu_thermal_margin.snapshot, evattr_therm_snap, "1" ); |
| 115 | PMU_EVENT_ATTR_STRING(cpu_thermal_margin.unit, evattr_therm_unit, "C" ); |
Andy Lutomirski | b7b7c78 | 2015-07-20 11:49:06 -0400 | [diff] [blame] | 116 | |
Peter Zijlstra | 19b3340 | 2015-08-06 17:26:58 +0200 | [diff] [blame] | 117 | static struct perf_msr msr[] = { |
Ingo Molnar | 9128d3e | 2018-01-05 08:18:52 -0800 | [diff] [blame] | 118 | [PERF_MSR_TSC] = { 0, &evattr_tsc, NULL, }, |
| 119 | [PERF_MSR_APERF] = { MSR_IA32_APERF, &evattr_aperf, test_aperfmperf, }, |
| 120 | [PERF_MSR_MPERF] = { MSR_IA32_MPERF, &evattr_mperf, test_aperfmperf, }, |
| 121 | [PERF_MSR_PPERF] = { MSR_PPERF, &evattr_pperf, test_intel, }, |
| 122 | [PERF_MSR_SMI] = { MSR_SMI_COUNT, &evattr_smi, test_intel, }, |
| 123 | [PERF_MSR_PTSC] = { MSR_F15H_PTSC, &evattr_ptsc, test_ptsc, }, |
| 124 | [PERF_MSR_IRPERF] = { MSR_F17H_IRPERF, &evattr_irperf, test_irperf, }, |
| 125 | [PERF_MSR_THERM] = { MSR_IA32_THERM_STATUS, &evattr_therm, test_therm_status, }, |
| 126 | [PERF_MSR_THERM_SNAP] = { MSR_IA32_THERM_STATUS, &evattr_therm_snap, test_therm_status, }, |
| 127 | [PERF_MSR_THERM_UNIT] = { MSR_IA32_THERM_STATUS, &evattr_therm_unit, test_therm_status, }, |
Peter Zijlstra | 19b3340 | 2015-08-06 17:26:58 +0200 | [diff] [blame] | 128 | }; |
| 129 | |
Andy Lutomirski | b7b7c78 | 2015-07-20 11:49:06 -0400 | [diff] [blame] | 130 | static struct attribute *events_attrs[PERF_MSR_EVENT_MAX + 1] = { |
Peter Zijlstra | 19b3340 | 2015-08-06 17:26:58 +0200 | [diff] [blame] | 131 | NULL, |
Andy Lutomirski | b7b7c78 | 2015-07-20 11:49:06 -0400 | [diff] [blame] | 132 | }; |
| 133 | |
| 134 | static struct attribute_group events_attr_group = { |
| 135 | .name = "events", |
| 136 | .attrs = events_attrs, |
| 137 | }; |
| 138 | |
| 139 | PMU_FORMAT_ATTR(event, "config:0-63"); |
| 140 | static struct attribute *format_attrs[] = { |
| 141 | &format_attr_event.attr, |
| 142 | NULL, |
| 143 | }; |
| 144 | static struct attribute_group format_attr_group = { |
| 145 | .name = "format", |
| 146 | .attrs = format_attrs, |
| 147 | }; |
| 148 | |
| 149 | static const struct attribute_group *attr_groups[] = { |
| 150 | &events_attr_group, |
| 151 | &format_attr_group, |
| 152 | NULL, |
| 153 | }; |
| 154 | |
| 155 | static int msr_event_init(struct perf_event *event) |
| 156 | { |
| 157 | u64 cfg = event->attr.config; |
| 158 | |
| 159 | if (event->attr.type != event->pmu->type) |
| 160 | return -ENOENT; |
| 161 | |
Andy Lutomirski | b7b7c78 | 2015-07-20 11:49:06 -0400 | [diff] [blame] | 162 | /* unsupported modes and filters */ |
| 163 | if (event->attr.exclude_user || |
| 164 | event->attr.exclude_kernel || |
| 165 | event->attr.exclude_hv || |
| 166 | event->attr.exclude_idle || |
| 167 | event->attr.exclude_host || |
| 168 | event->attr.exclude_guest || |
| 169 | event->attr.sample_period) /* no sampling */ |
| 170 | return -EINVAL; |
| 171 | |
Peter Zijlstra | 06ce6e9 | 2018-04-20 14:23:36 +0200 | [diff] [blame] | 172 | if (cfg >= PERF_MSR_EVENT_MAX) |
| 173 | return -EINVAL; |
| 174 | |
| 175 | cfg = array_index_nospec((unsigned long)cfg, PERF_MSR_EVENT_MAX); |
| 176 | |
Peter Zijlstra | 19b3340 | 2015-08-06 17:26:58 +0200 | [diff] [blame] | 177 | if (!msr[cfg].attr) |
| 178 | return -EINVAL; |
| 179 | |
Ingo Molnar | 9128d3e | 2018-01-05 08:18:52 -0800 | [diff] [blame] | 180 | event->hw.idx = -1; |
| 181 | event->hw.event_base = msr[cfg].msr; |
| 182 | event->hw.config = cfg; |
Andy Lutomirski | b7b7c78 | 2015-07-20 11:49:06 -0400 | [diff] [blame] | 183 | |
| 184 | return 0; |
| 185 | } |
| 186 | |
| 187 | static inline u64 msr_read_counter(struct perf_event *event) |
| 188 | { |
| 189 | u64 now; |
| 190 | |
| 191 | if (event->hw.event_base) |
| 192 | rdmsrl(event->hw.event_base, now); |
| 193 | else |
Thomas Gleixner | ea89c06 | 2018-03-23 00:05:29 +0100 | [diff] [blame] | 194 | now = rdtsc_ordered(); |
Andy Lutomirski | b7b7c78 | 2015-07-20 11:49:06 -0400 | [diff] [blame] | 195 | |
| 196 | return now; |
| 197 | } |
Thomas Gleixner | ea89c06 | 2018-03-23 00:05:29 +0100 | [diff] [blame] | 198 | |
Andy Lutomirski | b7b7c78 | 2015-07-20 11:49:06 -0400 | [diff] [blame] | 199 | static void msr_event_update(struct perf_event *event) |
| 200 | { |
| 201 | u64 prev, now; |
| 202 | s64 delta; |
| 203 | |
Ingo Molnar | 9128d3e | 2018-01-05 08:18:52 -0800 | [diff] [blame] | 204 | /* Careful, an NMI might modify the previous event value: */ |
Andy Lutomirski | b7b7c78 | 2015-07-20 11:49:06 -0400 | [diff] [blame] | 205 | again: |
| 206 | prev = local64_read(&event->hw.prev_count); |
| 207 | now = msr_read_counter(event); |
| 208 | |
| 209 | if (local64_cmpxchg(&event->hw.prev_count, prev, now) != prev) |
| 210 | goto again; |
| 211 | |
| 212 | delta = now - prev; |
Stephane Eranian | 9ae21dd | 2018-01-05 08:18:52 -0800 | [diff] [blame] | 213 | if (unlikely(event->hw.event_base == MSR_SMI_COUNT)) { |
Martin Kepplinger | 78e3c79 | 2015-11-06 16:31:08 -0800 | [diff] [blame] | 214 | delta = sign_extend64(delta, 31); |
Stephane Eranian | 9ae21dd | 2018-01-05 08:18:52 -0800 | [diff] [blame] | 215 | local64_add(delta, &event->count); |
| 216 | } else if (unlikely(event->hw.event_base == MSR_IA32_THERM_STATUS)) { |
Ingo Molnar | 9128d3e | 2018-01-05 08:18:52 -0800 | [diff] [blame] | 217 | /* If valid, extract digital readout, otherwise set to -1: */ |
Stephane Eranian | 9ae21dd | 2018-01-05 08:18:52 -0800 | [diff] [blame] | 218 | now = now & (1ULL << 31) ? (now >> 16) & 0x3f : -1; |
| 219 | local64_set(&event->count, now); |
Ingo Molnar | 9128d3e | 2018-01-05 08:18:52 -0800 | [diff] [blame] | 220 | } else { |
Stephane Eranian | 9ae21dd | 2018-01-05 08:18:52 -0800 | [diff] [blame] | 221 | local64_add(delta, &event->count); |
Ingo Molnar | 9128d3e | 2018-01-05 08:18:52 -0800 | [diff] [blame] | 222 | } |
Andy Lutomirski | b7b7c78 | 2015-07-20 11:49:06 -0400 | [diff] [blame] | 223 | } |
| 224 | |
| 225 | static void msr_event_start(struct perf_event *event, int flags) |
| 226 | { |
Ingo Molnar | 9128d3e | 2018-01-05 08:18:52 -0800 | [diff] [blame] | 227 | u64 now = msr_read_counter(event); |
Andy Lutomirski | b7b7c78 | 2015-07-20 11:49:06 -0400 | [diff] [blame] | 228 | |
Andy Lutomirski | b7b7c78 | 2015-07-20 11:49:06 -0400 | [diff] [blame] | 229 | local64_set(&event->hw.prev_count, now); |
| 230 | } |
| 231 | |
| 232 | static void msr_event_stop(struct perf_event *event, int flags) |
| 233 | { |
| 234 | msr_event_update(event); |
| 235 | } |
| 236 | |
| 237 | static void msr_event_del(struct perf_event *event, int flags) |
| 238 | { |
| 239 | msr_event_stop(event, PERF_EF_UPDATE); |
| 240 | } |
| 241 | |
| 242 | static int msr_event_add(struct perf_event *event, int flags) |
| 243 | { |
| 244 | if (flags & PERF_EF_START) |
| 245 | msr_event_start(event, flags); |
| 246 | |
| 247 | return 0; |
| 248 | } |
| 249 | |
| 250 | static struct pmu pmu_msr = { |
| 251 | .task_ctx_nr = perf_sw_context, |
| 252 | .attr_groups = attr_groups, |
| 253 | .event_init = msr_event_init, |
| 254 | .add = msr_event_add, |
| 255 | .del = msr_event_del, |
| 256 | .start = msr_event_start, |
| 257 | .stop = msr_event_stop, |
| 258 | .read = msr_event_update, |
| 259 | .capabilities = PERF_PMU_CAP_NO_INTERRUPT, |
| 260 | }; |
| 261 | |
Andy Lutomirski | b7b7c78 | 2015-07-20 11:49:06 -0400 | [diff] [blame] | 262 | static int __init msr_init(void) |
| 263 | { |
Peter Zijlstra | 19b3340 | 2015-08-06 17:26:58 +0200 | [diff] [blame] | 264 | int i, j = 0; |
Andy Lutomirski | b7b7c78 | 2015-07-20 11:49:06 -0400 | [diff] [blame] | 265 | |
Peter Zijlstra | 19b3340 | 2015-08-06 17:26:58 +0200 | [diff] [blame] | 266 | if (!boot_cpu_has(X86_FEATURE_TSC)) { |
| 267 | pr_cont("no MSR PMU driver.\n"); |
Andy Lutomirski | b7b7c78 | 2015-07-20 11:49:06 -0400 | [diff] [blame] | 268 | return 0; |
| 269 | } |
| 270 | |
Peter Zijlstra | 19b3340 | 2015-08-06 17:26:58 +0200 | [diff] [blame] | 271 | /* Probe the MSRs. */ |
| 272 | for (i = PERF_MSR_TSC + 1; i < PERF_MSR_EVENT_MAX; i++) { |
| 273 | u64 val; |
| 274 | |
Ingo Molnar | 9128d3e | 2018-01-05 08:18:52 -0800 | [diff] [blame] | 275 | /* Virt sucks; you cannot tell if a R/O MSR is present :/ */ |
Peter Zijlstra | 19b3340 | 2015-08-06 17:26:58 +0200 | [diff] [blame] | 276 | if (!msr[i].test(i) || rdmsrl_safe(msr[i].msr, &val)) |
| 277 | msr[i].attr = NULL; |
| 278 | } |
| 279 | |
| 280 | /* List remaining MSRs in the sysfs attrs. */ |
| 281 | for (i = 0; i < PERF_MSR_EVENT_MAX; i++) { |
| 282 | if (msr[i].attr) |
| 283 | events_attrs[j++] = &msr[i].attr->attr.attr; |
| 284 | } |
| 285 | events_attrs[j] = NULL; |
| 286 | |
Andy Lutomirski | b7b7c78 | 2015-07-20 11:49:06 -0400 | [diff] [blame] | 287 | perf_pmu_register(&pmu_msr, "msr", -1); |
| 288 | |
| 289 | return 0; |
| 290 | } |
| 291 | device_initcall(msr_init); |