rjw | 1f88458 | 2022-01-06 17:20:42 +0800 | [diff] [blame^] | 1 | // SPDX-License-Identifier: GPL-2.0 |
| 2 | /* |
| 3 | * cpuidle-powernv - idle state cpuidle driver. |
| 4 | * Adapted from drivers/cpuidle/cpuidle-pseries |
| 5 | * |
| 6 | */ |
| 7 | |
| 8 | #include <linux/kernel.h> |
| 9 | #include <linux/module.h> |
| 10 | #include <linux/init.h> |
| 11 | #include <linux/moduleparam.h> |
| 12 | #include <linux/cpuidle.h> |
| 13 | #include <linux/cpu.h> |
| 14 | #include <linux/notifier.h> |
| 15 | #include <linux/clockchips.h> |
| 16 | #include <linux/of.h> |
| 17 | #include <linux/slab.h> |
| 18 | |
| 19 | #include <asm/machdep.h> |
| 20 | #include <asm/firmware.h> |
| 21 | #include <asm/opal.h> |
| 22 | #include <asm/runlatch.h> |
| 23 | #include <asm/cpuidle.h> |
| 24 | |
| 25 | /* |
| 26 | * Expose only those Hardware idle states via the cpuidle framework |
| 27 | * that have latency value below POWERNV_THRESHOLD_LATENCY_NS. |
| 28 | */ |
| 29 | #define POWERNV_THRESHOLD_LATENCY_NS 200000 |
| 30 | |
| 31 | static struct cpuidle_driver powernv_idle_driver = { |
| 32 | .name = "powernv_idle", |
| 33 | .owner = THIS_MODULE, |
| 34 | }; |
| 35 | |
| 36 | static int max_idle_state __read_mostly; |
| 37 | static struct cpuidle_state *cpuidle_state_table __read_mostly; |
| 38 | |
| 39 | struct stop_psscr_table { |
| 40 | u64 val; |
| 41 | u64 mask; |
| 42 | }; |
| 43 | |
| 44 | static struct stop_psscr_table stop_psscr_table[CPUIDLE_STATE_MAX] __read_mostly; |
| 45 | |
| 46 | static u64 default_snooze_timeout __read_mostly; |
| 47 | static bool snooze_timeout_en __read_mostly; |
| 48 | |
| 49 | static u64 get_snooze_timeout(struct cpuidle_device *dev, |
| 50 | struct cpuidle_driver *drv, |
| 51 | int index) |
| 52 | { |
| 53 | int i; |
| 54 | |
| 55 | if (unlikely(!snooze_timeout_en)) |
| 56 | return default_snooze_timeout; |
| 57 | |
| 58 | for (i = index + 1; i < drv->state_count; i++) { |
| 59 | struct cpuidle_state *s = &drv->states[i]; |
| 60 | struct cpuidle_state_usage *su = &dev->states_usage[i]; |
| 61 | |
| 62 | if (s->disabled || su->disable) |
| 63 | continue; |
| 64 | |
| 65 | return s->target_residency * tb_ticks_per_usec; |
| 66 | } |
| 67 | |
| 68 | return default_snooze_timeout; |
| 69 | } |
| 70 | |
| 71 | static int snooze_loop(struct cpuidle_device *dev, |
| 72 | struct cpuidle_driver *drv, |
| 73 | int index) |
| 74 | { |
| 75 | u64 snooze_exit_time; |
| 76 | |
| 77 | set_thread_flag(TIF_POLLING_NRFLAG); |
| 78 | |
| 79 | local_irq_enable(); |
| 80 | |
| 81 | snooze_exit_time = get_tb() + get_snooze_timeout(dev, drv, index); |
| 82 | ppc64_runlatch_off(); |
| 83 | HMT_very_low(); |
| 84 | while (!need_resched()) { |
| 85 | if (likely(snooze_timeout_en) && get_tb() > snooze_exit_time) { |
| 86 | /* |
| 87 | * Task has not woken up but we are exiting the polling |
| 88 | * loop anyway. Require a barrier after polling is |
| 89 | * cleared to order subsequent test of need_resched(). |
| 90 | */ |
| 91 | clear_thread_flag(TIF_POLLING_NRFLAG); |
| 92 | smp_mb(); |
| 93 | break; |
| 94 | } |
| 95 | } |
| 96 | |
| 97 | HMT_medium(); |
| 98 | ppc64_runlatch_on(); |
| 99 | clear_thread_flag(TIF_POLLING_NRFLAG); |
| 100 | |
| 101 | return index; |
| 102 | } |
| 103 | |
| 104 | static int nap_loop(struct cpuidle_device *dev, |
| 105 | struct cpuidle_driver *drv, |
| 106 | int index) |
| 107 | { |
| 108 | power7_idle_type(PNV_THREAD_NAP); |
| 109 | |
| 110 | return index; |
| 111 | } |
| 112 | |
| 113 | /* Register for fastsleep only in oneshot mode of broadcast */ |
| 114 | #ifdef CONFIG_TICK_ONESHOT |
| 115 | static int fastsleep_loop(struct cpuidle_device *dev, |
| 116 | struct cpuidle_driver *drv, |
| 117 | int index) |
| 118 | { |
| 119 | unsigned long old_lpcr = mfspr(SPRN_LPCR); |
| 120 | unsigned long new_lpcr; |
| 121 | |
| 122 | if (unlikely(system_state < SYSTEM_RUNNING)) |
| 123 | return index; |
| 124 | |
| 125 | new_lpcr = old_lpcr; |
| 126 | /* Do not exit powersave upon decrementer as we've setup the timer |
| 127 | * offload. |
| 128 | */ |
| 129 | new_lpcr &= ~LPCR_PECE1; |
| 130 | |
| 131 | mtspr(SPRN_LPCR, new_lpcr); |
| 132 | |
| 133 | power7_idle_type(PNV_THREAD_SLEEP); |
| 134 | |
| 135 | mtspr(SPRN_LPCR, old_lpcr); |
| 136 | |
| 137 | return index; |
| 138 | } |
| 139 | #endif |
| 140 | |
| 141 | static int stop_loop(struct cpuidle_device *dev, |
| 142 | struct cpuidle_driver *drv, |
| 143 | int index) |
| 144 | { |
| 145 | power9_idle_type(stop_psscr_table[index].val, |
| 146 | stop_psscr_table[index].mask); |
| 147 | return index; |
| 148 | } |
| 149 | |
| 150 | /* |
| 151 | * States for dedicated partition case. |
| 152 | */ |
| 153 | static struct cpuidle_state powernv_states[CPUIDLE_STATE_MAX] = { |
| 154 | { /* Snooze */ |
| 155 | .name = "snooze", |
| 156 | .desc = "snooze", |
| 157 | .exit_latency = 0, |
| 158 | .target_residency = 0, |
| 159 | .enter = snooze_loop }, |
| 160 | }; |
| 161 | |
| 162 | static int powernv_cpuidle_cpu_online(unsigned int cpu) |
| 163 | { |
| 164 | struct cpuidle_device *dev = per_cpu(cpuidle_devices, cpu); |
| 165 | |
| 166 | if (dev && cpuidle_get_driver()) { |
| 167 | cpuidle_pause_and_lock(); |
| 168 | cpuidle_enable_device(dev); |
| 169 | cpuidle_resume_and_unlock(); |
| 170 | } |
| 171 | return 0; |
| 172 | } |
| 173 | |
| 174 | static int powernv_cpuidle_cpu_dead(unsigned int cpu) |
| 175 | { |
| 176 | struct cpuidle_device *dev = per_cpu(cpuidle_devices, cpu); |
| 177 | |
| 178 | if (dev && cpuidle_get_driver()) { |
| 179 | cpuidle_pause_and_lock(); |
| 180 | cpuidle_disable_device(dev); |
| 181 | cpuidle_resume_and_unlock(); |
| 182 | } |
| 183 | return 0; |
| 184 | } |
| 185 | |
| 186 | /* |
| 187 | * powernv_cpuidle_driver_init() |
| 188 | */ |
| 189 | static int powernv_cpuidle_driver_init(void) |
| 190 | { |
| 191 | int idle_state; |
| 192 | struct cpuidle_driver *drv = &powernv_idle_driver; |
| 193 | |
| 194 | drv->state_count = 0; |
| 195 | |
| 196 | for (idle_state = 0; idle_state < max_idle_state; ++idle_state) { |
| 197 | /* Is the state not enabled? */ |
| 198 | if (cpuidle_state_table[idle_state].enter == NULL) |
| 199 | continue; |
| 200 | |
| 201 | drv->states[drv->state_count] = /* structure copy */ |
| 202 | cpuidle_state_table[idle_state]; |
| 203 | |
| 204 | drv->state_count += 1; |
| 205 | } |
| 206 | |
| 207 | /* |
| 208 | * On the PowerNV platform cpu_present may be less than cpu_possible in |
| 209 | * cases when firmware detects the CPU, but it is not available to the |
| 210 | * OS. If CONFIG_HOTPLUG_CPU=n, then such CPUs are not hotplugable at |
| 211 | * run time and hence cpu_devices are not created for those CPUs by the |
| 212 | * generic topology_init(). |
| 213 | * |
| 214 | * drv->cpumask defaults to cpu_possible_mask in |
| 215 | * __cpuidle_driver_init(). This breaks cpuidle on PowerNV where |
| 216 | * cpu_devices are not created for CPUs in cpu_possible_mask that |
| 217 | * cannot be hot-added later at run time. |
| 218 | * |
| 219 | * Trying cpuidle_register_device() on a CPU without a cpu_device is |
| 220 | * incorrect, so pass a correct CPU mask to the generic cpuidle driver. |
| 221 | */ |
| 222 | |
| 223 | drv->cpumask = (struct cpumask *)cpu_present_mask; |
| 224 | |
| 225 | return 0; |
| 226 | } |
| 227 | |
| 228 | static inline void add_powernv_state(int index, const char *name, |
| 229 | unsigned int flags, |
| 230 | int (*idle_fn)(struct cpuidle_device *, |
| 231 | struct cpuidle_driver *, |
| 232 | int), |
| 233 | unsigned int target_residency, |
| 234 | unsigned int exit_latency, |
| 235 | u64 psscr_val, u64 psscr_mask) |
| 236 | { |
| 237 | strlcpy(powernv_states[index].name, name, CPUIDLE_NAME_LEN); |
| 238 | strlcpy(powernv_states[index].desc, name, CPUIDLE_NAME_LEN); |
| 239 | powernv_states[index].flags = flags; |
| 240 | powernv_states[index].target_residency = target_residency; |
| 241 | powernv_states[index].exit_latency = exit_latency; |
| 242 | powernv_states[index].enter = idle_fn; |
| 243 | stop_psscr_table[index].val = psscr_val; |
| 244 | stop_psscr_table[index].mask = psscr_mask; |
| 245 | } |
| 246 | |
| 247 | /* |
| 248 | * Returns 0 if prop1_len == prop2_len. Else returns -1 |
| 249 | */ |
| 250 | static inline int validate_dt_prop_sizes(const char *prop1, int prop1_len, |
| 251 | const char *prop2, int prop2_len) |
| 252 | { |
| 253 | if (prop1_len == prop2_len) |
| 254 | return 0; |
| 255 | |
| 256 | pr_warn("cpuidle-powernv: array sizes don't match for %s and %s\n", |
| 257 | prop1, prop2); |
| 258 | return -1; |
| 259 | } |
| 260 | |
| 261 | extern u32 pnv_get_supported_cpuidle_states(void); |
| 262 | static int powernv_add_idle_states(void) |
| 263 | { |
| 264 | struct device_node *power_mgt; |
| 265 | int nr_idle_states = 1; /* Snooze */ |
| 266 | int dt_idle_states, count; |
| 267 | u32 latency_ns[CPUIDLE_STATE_MAX]; |
| 268 | u32 residency_ns[CPUIDLE_STATE_MAX]; |
| 269 | u32 flags[CPUIDLE_STATE_MAX]; |
| 270 | u64 psscr_val[CPUIDLE_STATE_MAX]; |
| 271 | u64 psscr_mask[CPUIDLE_STATE_MAX]; |
| 272 | const char *names[CPUIDLE_STATE_MAX]; |
| 273 | u32 has_stop_states = 0; |
| 274 | int i, rc; |
| 275 | u32 supported_flags = pnv_get_supported_cpuidle_states(); |
| 276 | |
| 277 | |
| 278 | /* Currently we have snooze statically defined */ |
| 279 | |
| 280 | power_mgt = of_find_node_by_path("/ibm,opal/power-mgt"); |
| 281 | if (!power_mgt) { |
| 282 | pr_warn("opal: PowerMgmt Node not found\n"); |
| 283 | goto out; |
| 284 | } |
| 285 | |
| 286 | /* Read values of any property to determine the num of idle states */ |
| 287 | dt_idle_states = of_property_count_u32_elems(power_mgt, "ibm,cpu-idle-state-flags"); |
| 288 | if (dt_idle_states < 0) { |
| 289 | pr_warn("cpuidle-powernv: no idle states found in the DT\n"); |
| 290 | goto out; |
| 291 | } |
| 292 | |
| 293 | count = of_property_count_u32_elems(power_mgt, |
| 294 | "ibm,cpu-idle-state-latencies-ns"); |
| 295 | |
| 296 | if (validate_dt_prop_sizes("ibm,cpu-idle-state-flags", dt_idle_states, |
| 297 | "ibm,cpu-idle-state-latencies-ns", |
| 298 | count) != 0) |
| 299 | goto out; |
| 300 | |
| 301 | count = of_property_count_strings(power_mgt, |
| 302 | "ibm,cpu-idle-state-names"); |
| 303 | if (validate_dt_prop_sizes("ibm,cpu-idle-state-flags", dt_idle_states, |
| 304 | "ibm,cpu-idle-state-names", |
| 305 | count) != 0) |
| 306 | goto out; |
| 307 | |
| 308 | /* |
| 309 | * Since snooze is used as first idle state, max idle states allowed is |
| 310 | * CPUIDLE_STATE_MAX -1 |
| 311 | */ |
| 312 | if (dt_idle_states > CPUIDLE_STATE_MAX - 1) { |
| 313 | pr_warn("cpuidle-powernv: discovered idle states more than allowed"); |
| 314 | dt_idle_states = CPUIDLE_STATE_MAX - 1; |
| 315 | } |
| 316 | |
| 317 | if (of_property_read_u32_array(power_mgt, |
| 318 | "ibm,cpu-idle-state-flags", flags, dt_idle_states)) { |
| 319 | pr_warn("cpuidle-powernv : missing ibm,cpu-idle-state-flags in DT\n"); |
| 320 | goto out; |
| 321 | } |
| 322 | |
| 323 | if (of_property_read_u32_array(power_mgt, |
| 324 | "ibm,cpu-idle-state-latencies-ns", latency_ns, |
| 325 | dt_idle_states)) { |
| 326 | pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-latencies-ns in DT\n"); |
| 327 | goto out; |
| 328 | } |
| 329 | if (of_property_read_string_array(power_mgt, |
| 330 | "ibm,cpu-idle-state-names", names, dt_idle_states) < 0) { |
| 331 | pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-names in DT\n"); |
| 332 | goto out; |
| 333 | } |
| 334 | |
| 335 | /* |
| 336 | * If the idle states use stop instruction, probe for psscr values |
| 337 | * and psscr mask which are necessary to specify required stop level. |
| 338 | */ |
| 339 | has_stop_states = (flags[0] & |
| 340 | (OPAL_PM_STOP_INST_FAST | OPAL_PM_STOP_INST_DEEP)); |
| 341 | if (has_stop_states) { |
| 342 | count = of_property_count_u64_elems(power_mgt, |
| 343 | "ibm,cpu-idle-state-psscr"); |
| 344 | if (validate_dt_prop_sizes("ibm,cpu-idle-state-flags", |
| 345 | dt_idle_states, |
| 346 | "ibm,cpu-idle-state-psscr", |
| 347 | count) != 0) |
| 348 | goto out; |
| 349 | |
| 350 | count = of_property_count_u64_elems(power_mgt, |
| 351 | "ibm,cpu-idle-state-psscr-mask"); |
| 352 | if (validate_dt_prop_sizes("ibm,cpu-idle-state-flags", |
| 353 | dt_idle_states, |
| 354 | "ibm,cpu-idle-state-psscr-mask", |
| 355 | count) != 0) |
| 356 | goto out; |
| 357 | |
| 358 | if (of_property_read_u64_array(power_mgt, |
| 359 | "ibm,cpu-idle-state-psscr", psscr_val, dt_idle_states)) { |
| 360 | pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-psscr in DT\n"); |
| 361 | goto out; |
| 362 | } |
| 363 | |
| 364 | if (of_property_read_u64_array(power_mgt, |
| 365 | "ibm,cpu-idle-state-psscr-mask", |
| 366 | psscr_mask, dt_idle_states)) { |
| 367 | pr_warn("cpuidle-powernv:Missing ibm,cpu-idle-state-psscr-mask in DT\n"); |
| 368 | goto out; |
| 369 | } |
| 370 | } |
| 371 | |
| 372 | count = of_property_count_u32_elems(power_mgt, |
| 373 | "ibm,cpu-idle-state-residency-ns"); |
| 374 | |
| 375 | if (count < 0) { |
| 376 | rc = count; |
| 377 | } else if (validate_dt_prop_sizes("ibm,cpu-idle-state-flags", |
| 378 | dt_idle_states, |
| 379 | "ibm,cpu-idle-state-residency-ns", |
| 380 | count) != 0) { |
| 381 | goto out; |
| 382 | } else { |
| 383 | rc = of_property_read_u32_array(power_mgt, |
| 384 | "ibm,cpu-idle-state-residency-ns", |
| 385 | residency_ns, dt_idle_states); |
| 386 | } |
| 387 | |
| 388 | for (i = 0; i < dt_idle_states; i++) { |
| 389 | unsigned int exit_latency, target_residency; |
| 390 | bool stops_timebase = false; |
| 391 | |
| 392 | /* |
| 393 | * Skip the platform idle state whose flag isn't in |
| 394 | * the supported_cpuidle_states flag mask. |
| 395 | */ |
| 396 | if ((flags[i] & supported_flags) != flags[i]) |
| 397 | continue; |
| 398 | /* |
| 399 | * If an idle state has exit latency beyond |
| 400 | * POWERNV_THRESHOLD_LATENCY_NS then don't use it |
| 401 | * in cpu-idle. |
| 402 | */ |
| 403 | if (latency_ns[i] > POWERNV_THRESHOLD_LATENCY_NS) |
| 404 | continue; |
| 405 | /* |
| 406 | * Firmware passes residency and latency values in ns. |
| 407 | * cpuidle expects it in us. |
| 408 | */ |
| 409 | exit_latency = DIV_ROUND_UP(latency_ns[i], 1000); |
| 410 | if (!rc) |
| 411 | target_residency = DIV_ROUND_UP(residency_ns[i], 1000); |
| 412 | else |
| 413 | target_residency = 0; |
| 414 | |
| 415 | if (has_stop_states) { |
| 416 | int err = validate_psscr_val_mask(&psscr_val[i], |
| 417 | &psscr_mask[i], |
| 418 | flags[i]); |
| 419 | if (err) { |
| 420 | report_invalid_psscr_val(psscr_val[i], err); |
| 421 | continue; |
| 422 | } |
| 423 | } |
| 424 | |
| 425 | if (flags[i] & OPAL_PM_TIMEBASE_STOP) |
| 426 | stops_timebase = true; |
| 427 | |
| 428 | /* |
| 429 | * For nap and fastsleep, use default target_residency |
| 430 | * values if f/w does not expose it. |
| 431 | */ |
| 432 | if (flags[i] & OPAL_PM_NAP_ENABLED) { |
| 433 | if (!rc) |
| 434 | target_residency = 100; |
| 435 | /* Add NAP state */ |
| 436 | add_powernv_state(nr_idle_states, "Nap", |
| 437 | CPUIDLE_FLAG_NONE, nap_loop, |
| 438 | target_residency, exit_latency, 0, 0); |
| 439 | } else if (has_stop_states && !stops_timebase) { |
| 440 | add_powernv_state(nr_idle_states, names[i], |
| 441 | CPUIDLE_FLAG_NONE, stop_loop, |
| 442 | target_residency, exit_latency, |
| 443 | psscr_val[i], psscr_mask[i]); |
| 444 | } |
| 445 | |
| 446 | /* |
| 447 | * All cpuidle states with CPUIDLE_FLAG_TIMER_STOP set must come |
| 448 | * within this config dependency check. |
| 449 | */ |
| 450 | #ifdef CONFIG_TICK_ONESHOT |
| 451 | else if (flags[i] & OPAL_PM_SLEEP_ENABLED || |
| 452 | flags[i] & OPAL_PM_SLEEP_ENABLED_ER1) { |
| 453 | if (!rc) |
| 454 | target_residency = 300000; |
| 455 | /* Add FASTSLEEP state */ |
| 456 | add_powernv_state(nr_idle_states, "FastSleep", |
| 457 | CPUIDLE_FLAG_TIMER_STOP, |
| 458 | fastsleep_loop, |
| 459 | target_residency, exit_latency, 0, 0); |
| 460 | } else if (has_stop_states && stops_timebase) { |
| 461 | add_powernv_state(nr_idle_states, names[i], |
| 462 | CPUIDLE_FLAG_TIMER_STOP, stop_loop, |
| 463 | target_residency, exit_latency, |
| 464 | psscr_val[i], psscr_mask[i]); |
| 465 | } |
| 466 | #endif |
| 467 | else |
| 468 | continue; |
| 469 | nr_idle_states++; |
| 470 | } |
| 471 | out: |
| 472 | return nr_idle_states; |
| 473 | } |
| 474 | |
| 475 | /* |
| 476 | * powernv_idle_probe() |
| 477 | * Choose state table for shared versus dedicated partition |
| 478 | */ |
| 479 | static int powernv_idle_probe(void) |
| 480 | { |
| 481 | if (cpuidle_disable != IDLE_NO_OVERRIDE) |
| 482 | return -ENODEV; |
| 483 | |
| 484 | if (firmware_has_feature(FW_FEATURE_OPAL)) { |
| 485 | cpuidle_state_table = powernv_states; |
| 486 | /* Device tree can indicate more idle states */ |
| 487 | max_idle_state = powernv_add_idle_states(); |
| 488 | default_snooze_timeout = TICK_USEC * tb_ticks_per_usec; |
| 489 | if (max_idle_state > 1) |
| 490 | snooze_timeout_en = true; |
| 491 | } else |
| 492 | return -ENODEV; |
| 493 | |
| 494 | return 0; |
| 495 | } |
| 496 | |
| 497 | static int __init powernv_processor_idle_init(void) |
| 498 | { |
| 499 | int retval; |
| 500 | |
| 501 | retval = powernv_idle_probe(); |
| 502 | if (retval) |
| 503 | return retval; |
| 504 | |
| 505 | powernv_cpuidle_driver_init(); |
| 506 | retval = cpuidle_register(&powernv_idle_driver, NULL); |
| 507 | if (retval) { |
| 508 | printk(KERN_DEBUG "Registration of powernv driver failed.\n"); |
| 509 | return retval; |
| 510 | } |
| 511 | |
| 512 | retval = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, |
| 513 | "cpuidle/powernv:online", |
| 514 | powernv_cpuidle_cpu_online, NULL); |
| 515 | WARN_ON(retval < 0); |
| 516 | retval = cpuhp_setup_state_nocalls(CPUHP_CPUIDLE_DEAD, |
| 517 | "cpuidle/powernv:dead", NULL, |
| 518 | powernv_cpuidle_cpu_dead); |
| 519 | WARN_ON(retval < 0); |
| 520 | printk(KERN_DEBUG "powernv_idle_driver registered\n"); |
| 521 | return 0; |
| 522 | } |
| 523 | |
| 524 | device_initcall(powernv_processor_idle_init); |