drivers/clocksource/arm_arch_timer.c

   1 /*
   2  *  linux/drivers/clocksource/arm_arch_timer.c
   3  *
   4  *  Copyright (C) 2011 ARM Ltd.
   5  *  All Rights Reserved
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License version 2 as
   9  * published by the Free Software Foundation.
  10  */
  11 #include <linux/init.h>
  12 #include <linux/kernel.h>
  13 #include <linux/device.h>
  14 #include <linux/smp.h>
  15 #include <linux/cpu.h>
  16 #include <linux/cpu_pm.h>
  17 #include <linux/clockchips.h>
  18 #include <linux/clocksource.h>
  19 #include <linux/interrupt.h>
  20 #include <linux/of_irq.h>
  21 #include <linux/of_address.h>
  22 #include <linux/io.h>
  23 #include <linux/slab.h>
  24 #include <linux/sched_clock.h>
  25 #include <linux/acpi.h>
  26
  27 #include <asm/arch_timer.h>
  28 #include <asm/virt.h>
  29
  30 #include <clocksource/arm_arch_timer.h>
  31
  32 #define CNTTIDR         0x08
  33 #define CNTTIDR_VIRT(n) (BIT(1) << ((n) * 4))
  34
  35 #define CNTACR(n)       (0x40 + ((n) * 4))
  36 #define CNTACR_RPCT     BIT(0)
  37 #define CNTACR_RVCT     BIT(1)
  38 #define CNTACR_RFRQ     BIT(2)
  39 #define CNTACR_RVOFF    BIT(3)
  40 #define CNTACR_RWVT     BIT(4)
  41 #define CNTACR_RWPT     BIT(5)
  42
  43 #define CNTVCT_LO       0x08
  44 #define CNTVCT_HI       0x0c
  45 #define CNTFRQ          0x10
  46 #define CNTP_TVAL       0x28
  47 #define CNTP_CTL        0x2c
  48 #define CNTV_TVAL       0x38
  49 #define CNTV_CTL        0x3c
  50
  51 #define ARCH_CP15_TIMER BIT(0)
  52 #define ARCH_MEM_TIMER  BIT(1)
  53 static unsigned arch_timers_present __initdata;
  54
  55 static void __iomem *arch_counter_base;
  56
  57 struct arch_timer {
  58         void __iomem *base;
  59         struct clock_event_device evt;
  60 };
  61
  62 #define to_arch_timer(e) container_of(e, struct arch_timer, evt)
  63
  64 static u32 arch_timer_rate;
  65
  66 enum ppi_nr {
  67         PHYS_SECURE_PPI,
  68         PHYS_NONSECURE_PPI,
  69         VIRT_PPI,
  70         HYP_PPI,
  71         MAX_TIMER_PPI
  72 };
  73
  74 static int arch_timer_ppi[MAX_TIMER_PPI];
  75
  76 static struct clock_event_device __percpu *arch_timer_evt;
  77
  78 static enum ppi_nr arch_timer_uses_ppi = VIRT_PPI;
  79 static bool arch_timer_c3stop;
  80 static bool arch_timer_mem_use_virtual;
  81
  82 /*
  83  * Architected system timer support.
  84  */
  85
  86 static __always_inline
  87 void arch_timer_reg_write(int access, enum arch_timer_reg reg, u32 val,
  88                           struct clock_event_device *clk)
  89 {
  90         if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
  91                 struct arch_timer *timer = to_arch_timer(clk);
  92                 switch (reg) {
  93                 case ARCH_TIMER_REG_CTRL:
  94                         writel_relaxed(val, timer->base + CNTP_CTL);
  95                         break;
  96                 case ARCH_TIMER_REG_TVAL:
  97                         writel_relaxed(val, timer->base + CNTP_TVAL);
  98                         break;
  99                 }
 100         } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
 101                 struct arch_timer *timer = to_arch_timer(clk);
 102                 switch (reg) {
 103                 case ARCH_TIMER_REG_CTRL:
 104                         writel_relaxed(val, timer->base + CNTV_CTL);
 105                         break;
 106                 case ARCH_TIMER_REG_TVAL:
 107                         writel_relaxed(val, timer->base + CNTV_TVAL);
 108                         break;
 109                 }
 110         } else {
 111                 arch_timer_reg_write_cp15(access, reg, val);
 112         }
 113 }
 114
 115 static __always_inline
 116 u32 arch_timer_reg_read(int access, enum arch_timer_reg reg,
 117                         struct clock_event_device *clk)
 118 {
 119         u32 val;
 120
 121         if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
 122                 struct arch_timer *timer = to_arch_timer(clk);
 123                 switch (reg) {
 124                 case ARCH_TIMER_REG_CTRL:
 125                         val = readl_relaxed(timer->base + CNTP_CTL);
 126                         break;
 127                 case ARCH_TIMER_REG_TVAL:
 128                         val = readl_relaxed(timer->base + CNTP_TVAL);
 129                         break;
 130                 }
 131         } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
 132                 struct arch_timer *timer = to_arch_timer(clk);
 133                 switch (reg) {
 134                 case ARCH_TIMER_REG_CTRL:
 135                         val = readl_relaxed(timer->base + CNTV_CTL);
 136                         break;
 137                 case ARCH_TIMER_REG_TVAL:
 138                         val = readl_relaxed(timer->base + CNTV_TVAL);
 139                         break;
 140                 }
 141         } else {
 142                 val = arch_timer_reg_read_cp15(access, reg);
 143         }
 144
 145         return val;
 146 }
 147
 148 static __always_inline irqreturn_t timer_handler(const int access,
 149                                         struct clock_event_device *evt)
 150 {
 151         unsigned long ctrl;
 152
 153         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, evt);
 154         if (ctrl & ARCH_TIMER_CTRL_IT_STAT) {
 155                 ctrl |= ARCH_TIMER_CTRL_IT_MASK;
 156                 arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, evt);
 157                 evt->event_handler(evt);
 158                 return IRQ_HANDLED;
 159         }
 160
 161         return IRQ_NONE;
 162 }
 163
 164 static irqreturn_t arch_timer_handler_virt(int irq, void *dev_id)
 165 {
 166         struct clock_event_device *evt = dev_id;
 167
 168         return timer_handler(ARCH_TIMER_VIRT_ACCESS, evt);
 169 }
 170
 171 static irqreturn_t arch_timer_handler_phys(int irq, void *dev_id)
 172 {
 173         struct clock_event_device *evt = dev_id;
 174
 175         return timer_handler(ARCH_TIMER_PHYS_ACCESS, evt);
 176 }
 177
 178 static irqreturn_t arch_timer_handler_phys_mem(int irq, void *dev_id)
 179 {
 180         struct clock_event_device *evt = dev_id;
 181
 182         return timer_handler(ARCH_TIMER_MEM_PHYS_ACCESS, evt);
 183 }
 184
 185 static irqreturn_t arch_timer_handler_virt_mem(int irq, void *dev_id)
 186 {
 187         struct clock_event_device *evt = dev_id;
 188
 189         return timer_handler(ARCH_TIMER_MEM_VIRT_ACCESS, evt);
 190 }
 191
 192 static __always_inline int timer_shutdown(const int access,
 193                                           struct clock_event_device *clk)
 194 {
 195         unsigned long ctrl;
 196
 197         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
 198         ctrl &= ~ARCH_TIMER_CTRL_ENABLE;
 199         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
 200
 201         return 0;
 202 }
 203
 204 static int arch_timer_shutdown_virt(struct clock_event_device *clk)
 205 {
 206         return timer_shutdown(ARCH_TIMER_VIRT_ACCESS, clk);
 207 }
 208
 209 static int arch_timer_shutdown_phys(struct clock_event_device *clk)
 210 {
 211         return timer_shutdown(ARCH_TIMER_PHYS_ACCESS, clk);
 212 }
 213
 214 static int arch_timer_shutdown_virt_mem(struct clock_event_device *clk)
 215 {
 216         return timer_shutdown(ARCH_TIMER_MEM_VIRT_ACCESS, clk);
 217 }
 218
 219 static int arch_timer_shutdown_phys_mem(struct clock_event_device *clk)
 220 {
 221         return timer_shutdown(ARCH_TIMER_MEM_PHYS_ACCESS, clk);
 222 }
 223
 224 static __always_inline void set_next_event(const int access, unsigned long evt,
 225                                            struct clock_event_device *clk)
 226 {
 227         unsigned long ctrl;
 228         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
 229         ctrl |= ARCH_TIMER_CTRL_ENABLE;
 230         ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
 231         arch_timer_reg_write(access, ARCH_TIMER_REG_TVAL, evt, clk);
 232         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
 233 }
 234
 235 static int arch_timer_set_next_event_virt(unsigned long evt,
 236                                           struct clock_event_device *clk)
 237 {
 238         set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk);
 239         return 0;
 240 }
 241
 242 static int arch_timer_set_next_event_phys(unsigned long evt,
 243                                           struct clock_event_device *clk)
 244 {
 245         set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk);
 246         return 0;
 247 }
 248
 249 static int arch_timer_set_next_event_virt_mem(unsigned long evt,
 250                                               struct clock_event_device *clk)
 251 {
 252         set_next_event(ARCH_TIMER_MEM_VIRT_ACCESS, evt, clk);
 253         return 0;
 254 }
 255
 256 static int arch_timer_set_next_event_phys_mem(unsigned long evt,
 257                                               struct clock_event_device *clk)
 258 {
 259         set_next_event(ARCH_TIMER_MEM_PHYS_ACCESS, evt, clk);
 260         return 0;
 261 }
 262
 263 static void __arch_timer_setup(unsigned type,
 264                                struct clock_event_device *clk)
 265 {
 266         clk->features = CLOCK_EVT_FEAT_ONESHOT;
 267
 268         if (type == ARCH_CP15_TIMER) {
 269                 if (arch_timer_c3stop)
 270                         clk->features |= CLOCK_EVT_FEAT_C3STOP;
 271                 clk->name = "arch_sys_timer";
 272                 clk->rating = 450;
 273                 clk->cpumask = cpumask_of(smp_processor_id());
 274                 clk->irq = arch_timer_ppi[arch_timer_uses_ppi];
 275                 switch (arch_timer_uses_ppi) {
 276                 case VIRT_PPI:
 277                         clk->set_state_shutdown = arch_timer_shutdown_virt;
 278                         clk->set_state_oneshot_stopped = arch_timer_shutdown_virt;
 279                         clk->set_next_event = arch_timer_set_next_event_virt;
 280                         break;
 281                 case PHYS_SECURE_PPI:
 282                 case PHYS_NONSECURE_PPI:
 283                 case HYP_PPI:
 284                         clk->set_state_shutdown = arch_timer_shutdown_phys;
 285                         clk->set_state_oneshot_stopped = arch_timer_shutdown_phys;
 286                         clk->set_next_event = arch_timer_set_next_event_phys;
 287                         break;
 288                 default:
 289                         BUG();
 290                 }
 291         } else {
 292                 clk->features |= CLOCK_EVT_FEAT_DYNIRQ;
 293                 clk->name = "arch_mem_timer";
 294                 clk->rating = 400;
 295                 clk->cpumask = cpu_all_mask;
 296                 if (arch_timer_mem_use_virtual) {
 297                         clk->set_state_shutdown = arch_timer_shutdown_virt_mem;
 298                         clk->set_state_oneshot_stopped = arch_timer_shutdown_virt_mem;
 299                         clk->set_next_event =
 300                                 arch_timer_set_next_event_virt_mem;
 301                 } else {
 302                         clk->set_state_shutdown = arch_timer_shutdown_phys_mem;
 303                         clk->set_state_oneshot_stopped = arch_timer_shutdown_phys_mem;
 304                         clk->set_next_event =
 305                                 arch_timer_set_next_event_phys_mem;
 306                 }
 307         }
 308
 309         clk->set_state_shutdown(clk);
 310
 311         clockevents_config_and_register(clk, arch_timer_rate, 0xf, 0x7fffffff);
 312 }
 313
 314 static void arch_timer_evtstrm_enable(int divider)
 315 {
 316         u32 cntkctl = arch_timer_get_cntkctl();
 317
 318         cntkctl &= ~ARCH_TIMER_EVT_TRIGGER_MASK;
 319         /* Set the divider and enable virtual event stream */
 320         cntkctl |= (divider << ARCH_TIMER_EVT_TRIGGER_SHIFT)
 321                         | ARCH_TIMER_VIRT_EVT_EN;
 322         arch_timer_set_cntkctl(cntkctl);
 323         elf_hwcap |= HWCAP_EVTSTRM;
 324 #ifdef CONFIG_COMPAT
 325         compat_elf_hwcap |= COMPAT_HWCAP_EVTSTRM;
 326 #endif
 327 }
 328
 329 static void arch_timer_configure_evtstream(void)
 330 {
 331         int evt_stream_div, pos;
 332
 333         /* Find the closest power of two to the divisor */
 334         evt_stream_div = arch_timer_rate / ARCH_TIMER_EVT_STREAM_FREQ;
 335         pos = fls(evt_stream_div);
 336         if (pos > 1 && !(evt_stream_div & (1 << (pos - 2))))
 337                 pos--;
 338         /* enable event stream */
 339         arch_timer_evtstrm_enable(min(pos, 15));
 340 }
 341
 342 static void arch_counter_set_user_access(void)
 343 {
 344         u32 cntkctl = arch_timer_get_cntkctl();
 345
 346         /* Disable user access to the timers and the physical counter */
 347         /* Also disable virtual event stream */
 348         cntkctl &= ~(ARCH_TIMER_USR_PT_ACCESS_EN
 349                         | ARCH_TIMER_USR_VT_ACCESS_EN
 350                         | ARCH_TIMER_VIRT_EVT_EN
 351                         | ARCH_TIMER_USR_PCT_ACCESS_EN);
 352
 353         /* Enable user access to the virtual counter */
 354         cntkctl |= ARCH_TIMER_USR_VCT_ACCESS_EN;
 355
 356         arch_timer_set_cntkctl(cntkctl);
 357 }
 358
 359 static bool arch_timer_has_nonsecure_ppi(void)
 360 {
 361         return (arch_timer_uses_ppi == PHYS_SECURE_PPI &&
 362                 arch_timer_ppi[PHYS_NONSECURE_PPI]);
 363 }
 364
 365 static int arch_timer_setup(struct clock_event_device *clk)
 366 {
 367         __arch_timer_setup(ARCH_CP15_TIMER, clk);
 368
 369         enable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], 0);
 370
 371         if (arch_timer_has_nonsecure_ppi())
 372                 enable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI], 0);
 373
 374         arch_counter_set_user_access();
 375         if (IS_ENABLED(CONFIG_ARM_ARCH_TIMER_EVTSTREAM))
 376                 arch_timer_configure_evtstream();
 377
 378         return 0;
 379 }
 380
 381 static void
 382 arch_timer_detect_rate(void __iomem *cntbase, struct device_node *np)
 383 {
 384         /* Who has more than one independent system counter? */
 385         if (arch_timer_rate)
 386                 return;
 387
 388         /*
 389          * Try to determine the frequency from the device tree or CNTFRQ,
 390          * if ACPI is enabled, get the frequency from CNTFRQ ONLY.
 391          */
 392         if (!acpi_disabled ||
 393             of_property_read_u32(np, "clock-frequency", &arch_timer_rate)) {
 394                 if (cntbase)
 395                         arch_timer_rate = readl_relaxed(cntbase + CNTFRQ);
 396                 else
 397                         arch_timer_rate = arch_timer_get_cntfrq();
 398         }
 399
 400         /* Check the timer frequency. */
 401         if (arch_timer_rate == 0)
 402                 pr_warn("Architected timer frequency not available\n");
 403 }
 404
 405 static void arch_timer_banner(unsigned type)
 406 {
 407         pr_info("Architected %s%s%s timer(s) running at %lu.%02luMHz (%s%s%s).\n",
 408                      type & ARCH_CP15_TIMER ? "cp15" : "",
 409                      type == (ARCH_CP15_TIMER | ARCH_MEM_TIMER) ?  " and " : "",
 410                      type & ARCH_MEM_TIMER ? "mmio" : "",
 411                      (unsigned long)arch_timer_rate / 1000000,
 412                      (unsigned long)(arch_timer_rate / 10000) % 100,
 413                      type & ARCH_CP15_TIMER ?
 414                      (arch_timer_uses_ppi == VIRT_PPI) ? "virt" : "phys" :
 415                         "",
 416                      type == (ARCH_CP15_TIMER | ARCH_MEM_TIMER) ?  "/" : "",
 417                      type & ARCH_MEM_TIMER ?
 418                         arch_timer_mem_use_virtual ? "virt" : "phys" :
 419                         "");
 420 }
 421
 422 u32 arch_timer_get_rate(void)
 423 {
 424         return arch_timer_rate;
 425 }
 426
 427 static u64 arch_counter_get_cntvct_mem(void)
 428 {
 429         u32 vct_lo, vct_hi, tmp_hi;
 430
 431         do {
 432                 vct_hi = readl_relaxed(arch_counter_base + CNTVCT_HI);
 433                 vct_lo = readl_relaxed(arch_counter_base + CNTVCT_LO);
 434                 tmp_hi = readl_relaxed(arch_counter_base + CNTVCT_HI);
 435         } while (vct_hi != tmp_hi);
 436
 437         return ((u64) vct_hi << 32) | vct_lo;
 438 }
 439
 440 /*
 441  * Default to cp15 based access because arm64 uses this function for
 442  * sched_clock() before DT is probed and the cp15 method is guaranteed
 443  * to exist on arm64. arm doesn't use this before DT is probed so even
 444  * if we don't have the cp15 accessors we won't have a problem.
 445  */
 446 u64 (*arch_timer_read_counter)(void) = arch_counter_get_cntvct;
 447
 448 static cycle_t arch_counter_read(struct clocksource *cs)
 449 {
 450         return arch_timer_read_counter();
 451 }
 452
 453 static cycle_t arch_counter_read_cc(const struct cyclecounter *cc)
 454 {
 455         return arch_timer_read_counter();
 456 }
 457
 458 static struct clocksource clocksource_counter = {
 459         .name   = "arch_sys_counter",
 460         .rating = 400,
 461         .read   = arch_counter_read,
 462         .mask   = CLOCKSOURCE_MASK(56),
 463         .flags  = CLOCK_SOURCE_IS_CONTINUOUS | CLOCK_SOURCE_SUSPEND_NONSTOP,
 464 };
 465
 466 static struct cyclecounter cyclecounter = {
 467         .read   = arch_counter_read_cc,
 468         .mask   = CLOCKSOURCE_MASK(56),
 469 };
 470
 471 static struct arch_timer_kvm_info arch_timer_kvm_info;
 472
 473 struct arch_timer_kvm_info *arch_timer_get_kvm_info(void)
 474 {
 475         return &arch_timer_kvm_info;
 476 }
 477
 478 static void __init arch_counter_register(unsigned type)
 479 {
 480         u64 start_count;
 481
 482         /* Register the CP15 based counter if we have one */
 483         if (type & ARCH_CP15_TIMER) {
 484                 if (IS_ENABLED(CONFIG_ARM64) || arch_timer_uses_ppi == VIRT_PPI)
 485                         arch_timer_read_counter = arch_counter_get_cntvct;
 486                 else
 487                         arch_timer_read_counter = arch_counter_get_cntpct;
 488         } else {
 489                 arch_timer_read_counter = arch_counter_get_cntvct_mem;
 490
 491                 /* If the clocksource name is "arch_sys_counter" the
 492                  * VDSO will attempt to read the CP15-based counter.
 493                  * Ensure this does not happen when CP15-based
 494                  * counter is not available.
 495                  */
 496                 clocksource_counter.name = "arch_mem_counter";
 497         }
 498
 499         start_count = arch_timer_read_counter();
 500         clocksource_register_hz(&clocksource_counter, arch_timer_rate);
 501         cyclecounter.mult = clocksource_counter.mult;
 502         cyclecounter.shift = clocksource_counter.shift;
 503         timecounter_init(&arch_timer_kvm_info.timecounter,
 504                          &cyclecounter, start_count);
 505
 506         /* 56 bits minimum, so we assume worst case rollover */
 507         sched_clock_register(arch_timer_read_counter, 56, arch_timer_rate);
 508 }
 509
 510 static void arch_timer_stop(struct clock_event_device *clk)
 511 {
 512         pr_debug("arch_timer_teardown disable IRQ%d cpu #%d\n",
 513                  clk->irq, smp_processor_id());
 514
 515         disable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi]);
 516         if (arch_timer_has_nonsecure_ppi())
 517                 disable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI]);
 518
 519         clk->set_state_shutdown(clk);
 520 }
 521
 522 static int arch_timer_cpu_notify(struct notifier_block *self,
 523                                            unsigned long action, void *hcpu)
 524 {
 525         /*
 526          * Grab cpu pointer in each case to avoid spurious
 527          * preemptible warnings
 528          */
 529         switch (action & ~CPU_TASKS_FROZEN) {
 530         case CPU_STARTING:
 531                 arch_timer_setup(this_cpu_ptr(arch_timer_evt));
 532                 break;
 533         case CPU_DYING:
 534                 arch_timer_stop(this_cpu_ptr(arch_timer_evt));
 535                 break;
 536         }
 537
 538         return NOTIFY_OK;
 539 }
 540
 541 static struct notifier_block arch_timer_cpu_nb = {
 542         .notifier_call = arch_timer_cpu_notify,
 543 };
 544
 545 #ifdef CONFIG_CPU_PM
 546 static unsigned int saved_cntkctl;
 547 static int arch_timer_cpu_pm_notify(struct notifier_block *self,
 548                                     unsigned long action, void *hcpu)
 549 {
 550         if (action == CPU_PM_ENTER)
 551                 saved_cntkctl = arch_timer_get_cntkctl();
 552         else if (action == CPU_PM_ENTER_FAILED || action == CPU_PM_EXIT)
 553                 arch_timer_set_cntkctl(saved_cntkctl);
 554         return NOTIFY_OK;
 555 }
 556
 557 static struct notifier_block arch_timer_cpu_pm_notifier = {
 558         .notifier_call = arch_timer_cpu_pm_notify,
 559 };
 560
 561 static int __init arch_timer_cpu_pm_init(void)
 562 {
 563         return cpu_pm_register_notifier(&arch_timer_cpu_pm_notifier);
 564 }
 565 #else
 566 static int __init arch_timer_cpu_pm_init(void)
 567 {
 568         return 0;
 569 }
 570 #endif
 571
 572 static int __init arch_timer_register(void)
 573 {
 574         int err;
 575         int ppi;
 576
 577         arch_timer_evt = alloc_percpu(struct clock_event_device);
 578         if (!arch_timer_evt) {
 579                 err = -ENOMEM;
 580                 goto out;
 581         }
 582
 583         ppi = arch_timer_ppi[arch_timer_uses_ppi];
 584         switch (arch_timer_uses_ppi) {
 585         case VIRT_PPI:
 586                 err = request_percpu_irq(ppi, arch_timer_handler_virt,
 587                                          "arch_timer", arch_timer_evt);
 588                 break;
 589         case PHYS_SECURE_PPI:
 590         case PHYS_NONSECURE_PPI:
 591                 err = request_percpu_irq(ppi, arch_timer_handler_phys,
 592                                          "arch_timer", arch_timer_evt);
 593                 if (!err && arch_timer_ppi[PHYS_NONSECURE_PPI]) {
 594                         ppi = arch_timer_ppi[PHYS_NONSECURE_PPI];
 595                         err = request_percpu_irq(ppi, arch_timer_handler_phys,
 596                                                  "arch_timer", arch_timer_evt);
 597                         if (err)
 598                                 free_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI],
 599                                                 arch_timer_evt);
 600                 }
 601                 break;
 602         case HYP_PPI:
 603                 err = request_percpu_irq(ppi, arch_timer_handler_phys,
 604                                          "arch_timer", arch_timer_evt);
 605                 break;
 606         default:
 607                 BUG();
 608         }
 609
 610         if (err) {
 611                 pr_err("arch_timer: can't register interrupt %d (%d)\n",
 612                        ppi, err);
 613                 goto out_free;
 614         }
 615
 616         err = register_cpu_notifier(&arch_timer_cpu_nb);
 617         if (err)
 618                 goto out_free_irq;
 619
 620         err = arch_timer_cpu_pm_init();
 621         if (err)
 622                 goto out_unreg_notify;
 623
 624         /* Immediately configure the timer on the boot CPU */
 625         arch_timer_setup(this_cpu_ptr(arch_timer_evt));
 626
 627         return 0;
 628
 629 out_unreg_notify:
 630         unregister_cpu_notifier(&arch_timer_cpu_nb);
 631 out_free_irq:
 632         free_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], arch_timer_evt);
 633         if (arch_timer_has_nonsecure_ppi())
 634                 free_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI],
 635                                 arch_timer_evt);
 636
 637 out_free:
 638         free_percpu(arch_timer_evt);
 639 out:
 640         return err;
 641 }
 642
 643 static int __init arch_timer_mem_register(void __iomem *base, unsigned int irq)
 644 {
 645         int ret;
 646         irq_handler_t func;
 647         struct arch_timer *t;
 648
 649         t = kzalloc(sizeof(*t), GFP_KERNEL);
 650         if (!t)
 651                 return -ENOMEM;
 652
 653         t->base = base;
 654         t->evt.irq = irq;
 655         __arch_timer_setup(ARCH_MEM_TIMER, &t->evt);
 656
 657         if (arch_timer_mem_use_virtual)
 658                 func = arch_timer_handler_virt_mem;
 659         else
 660                 func = arch_timer_handler_phys_mem;
 661
 662         ret = request_irq(irq, func, IRQF_TIMER, "arch_mem_timer", &t->evt);
 663         if (ret) {
 664                 pr_err("arch_timer: Failed to request mem timer irq\n");
 665                 kfree(t);
 666         }
 667
 668         return ret;
 669 }
 670
 671 static const struct of_device_id arch_timer_of_match[] __initconst = {
 672         { .compatible   = "arm,armv7-timer",    },
 673         { .compatible   = "arm,armv8-timer",    },
 674         {},
 675 };
 676
 677 static const struct of_device_id arch_timer_mem_of_match[] __initconst = {
 678         { .compatible   = "arm,armv7-timer-mem", },
 679         {},
 680 };
 681
 682 static bool __init
 683 arch_timer_needs_probing(int type, const struct of_device_id *matches)
 684 {
 685         struct device_node *dn;
 686         bool needs_probing = false;
 687
 688         dn = of_find_matching_node(NULL, matches);
 689         if (dn && of_device_is_available(dn) && !(arch_timers_present & type))
 690                 needs_probing = true;
 691         of_node_put(dn);
 692
 693         return needs_probing;
 694 }
 695
 696 static void __init arch_timer_common_init(void)
 697 {
 698         unsigned mask = ARCH_CP15_TIMER | ARCH_MEM_TIMER;
 699
 700         /* Wait until both nodes are probed if we have two timers */
 701         if ((arch_timers_present & mask) != mask) {
 702                 if (arch_timer_needs_probing(ARCH_MEM_TIMER, arch_timer_mem_of_match))
 703                         return;
 704                 if (arch_timer_needs_probing(ARCH_CP15_TIMER, arch_timer_of_match))
 705                         return;
 706         }
 707
 708         arch_timer_banner(arch_timers_present);
 709         arch_counter_register(arch_timers_present);
 710         arch_timer_arch_init();
 711 }
 712
 713 static void __init arch_timer_init(void)
 714 {
 715         /*
 716          * If HYP mode is available, we know that the physical timer
 717          * has been configured to be accessible from PL1. Use it, so
 718          * that a guest can use the virtual timer instead.
 719          *
 720          * If no interrupt provided for virtual timer, we'll have to
 721          * stick to the physical timer. It'd better be accessible...
 722          *
 723          * On ARMv8.1 with VH extensions, the kernel runs in HYP. VHE
 724          * accesses to CNTP_*_EL1 registers are silently redirected to
 725          * their CNTHP_*_EL2 counterparts, and use a different PPI
 726          * number.
 727          */
 728         if (is_hyp_mode_available() || !arch_timer_ppi[VIRT_PPI]) {
 729                 bool has_ppi;
 730
 731                 if (is_kernel_in_hyp_mode()) {
 732                         arch_timer_uses_ppi = HYP_PPI;
 733                         has_ppi = !!arch_timer_ppi[HYP_PPI];
 734                 } else {
 735                         arch_timer_uses_ppi = PHYS_SECURE_PPI;
 736                         has_ppi = (!!arch_timer_ppi[PHYS_SECURE_PPI] ||
 737                                    !!arch_timer_ppi[PHYS_NONSECURE_PPI]);
 738                 }
 739
 740                 if (!has_ppi) {
 741                         pr_warn("arch_timer: No interrupt available, giving up\n");
 742                         return;
 743                 }
 744         }
 745
 746         arch_timer_register();
 747         arch_timer_common_init();
 748
 749         arch_timer_kvm_info.virtual_irq = arch_timer_ppi[VIRT_PPI];
 750 }
 751
 752 static void __init arch_timer_of_init(struct device_node *np)
 753 {
 754         int i;
 755
 756         if (arch_timers_present & ARCH_CP15_TIMER) {
 757                 pr_warn("arch_timer: multiple nodes in dt, skipping\n");
 758                 return;
 759         }
 760
 761         arch_timers_present |= ARCH_CP15_TIMER;
 762         for (i = PHYS_SECURE_PPI; i < MAX_TIMER_PPI; i++)
 763                 arch_timer_ppi[i] = irq_of_parse_and_map(np, i);
 764
 765         arch_timer_detect_rate(NULL, np);
 766
 767         arch_timer_c3stop = !of_property_read_bool(np, "always-on");
 768
 769         /*
 770          * If we cannot rely on firmware initializing the timer registers then
 771          * we should use the physical timers instead.
 772          */
 773         if (IS_ENABLED(CONFIG_ARM) &&
 774             of_property_read_bool(np, "arm,cpu-registers-not-fw-configured"))
 775                 arch_timer_uses_ppi = PHYS_SECURE_PPI;
 776
 777         arch_timer_init();
 778 }
 779 CLOCKSOURCE_OF_DECLARE(armv7_arch_timer, "arm,armv7-timer", arch_timer_of_init);
 780 CLOCKSOURCE_OF_DECLARE(armv8_arch_timer, "arm,armv8-timer", arch_timer_of_init);
 781
 782 static void __init arch_timer_mem_init(struct device_node *np)
 783 {
 784         struct device_node *frame, *best_frame = NULL;
 785         void __iomem *cntctlbase, *base;
 786         unsigned int irq;
 787         u32 cnttidr;
 788
 789         arch_timers_present |= ARCH_MEM_TIMER;
 790         cntctlbase = of_iomap(np, 0);
 791         if (!cntctlbase) {
 792                 pr_err("arch_timer: Can't find CNTCTLBase\n");
 793                 return;
 794         }
 795
 796         cnttidr = readl_relaxed(cntctlbase + CNTTIDR);
 797
 798         /*
 799          * Try to find a virtual capable frame. Otherwise fall back to a
 800          * physical capable frame.
 801          */
 802         for_each_available_child_of_node(np, frame) {
 803                 int n;
 804                 u32 cntacr;
 805
 806                 if (of_property_read_u32(frame, "frame-number", &n)) {
 807                         pr_err("arch_timer: Missing frame-number\n");
 808                         of_node_put(frame);
 809                         goto out;
 810                 }
 811
 812                 /* Try enabling everything, and see what sticks */
 813                 cntacr = CNTACR_RFRQ | CNTACR_RWPT | CNTACR_RPCT |
 814                          CNTACR_RWVT | CNTACR_RVOFF | CNTACR_RVCT;
 815                 writel_relaxed(cntacr, cntctlbase + CNTACR(n));
 816                 cntacr = readl_relaxed(cntctlbase + CNTACR(n));
 817
 818                 if ((cnttidr & CNTTIDR_VIRT(n)) &&
 819                     !(~cntacr & (CNTACR_RWVT | CNTACR_RVCT))) {
 820                         of_node_put(best_frame);
 821                         best_frame = frame;
 822                         arch_timer_mem_use_virtual = true;
 823                         break;
 824                 }
 825
 826                 if (~cntacr & (CNTACR_RWPT | CNTACR_RPCT))
 827                         continue;
 828
 829                 of_node_put(best_frame);
 830                 best_frame = of_node_get(frame);
 831         }
 832
 833         base = arch_counter_base = of_iomap(best_frame, 0);
 834         if (!base) {
 835                 pr_err("arch_timer: Can't map frame's registers\n");
 836                 goto out;
 837         }
 838
 839         if (arch_timer_mem_use_virtual)
 840                 irq = irq_of_parse_and_map(best_frame, 1);
 841         else
 842                 irq = irq_of_parse_and_map(best_frame, 0);
 843
 844         if (!irq) {
 845                 pr_err("arch_timer: Frame missing %s irq",
 846                        arch_timer_mem_use_virtual ? "virt" : "phys");
 847                 goto out;
 848         }
 849
 850         arch_timer_detect_rate(base, np);
 851         arch_timer_mem_register(base, irq);
 852         arch_timer_common_init();
 853 out:
 854         iounmap(cntctlbase);
 855         of_node_put(best_frame);
 856 }
 857 CLOCKSOURCE_OF_DECLARE(armv7_arch_timer_mem, "arm,armv7-timer-mem",
 858                        arch_timer_mem_init);
 859
 860 #ifdef CONFIG_ACPI
 861 static int __init map_generic_timer_interrupt(u32 interrupt, u32 flags)
 862 {
 863         int trigger, polarity;
 864
 865         if (!interrupt)
 866                 return 0;
 867
 868         trigger = (flags & ACPI_GTDT_INTERRUPT_MODE) ? ACPI_EDGE_SENSITIVE
 869                         : ACPI_LEVEL_SENSITIVE;
 870
 871         polarity = (flags & ACPI_GTDT_INTERRUPT_POLARITY) ? ACPI_ACTIVE_LOW
 872                         : ACPI_ACTIVE_HIGH;
 873
 874         return acpi_register_gsi(NULL, interrupt, trigger, polarity);
 875 }
 876
 877 /* Initialize per-processor generic timer */
 878 static int __init arch_timer_acpi_init(struct acpi_table_header *table)
 879 {
 880         struct acpi_table_gtdt *gtdt;
 881
 882         if (arch_timers_present & ARCH_CP15_TIMER) {
 883                 pr_warn("arch_timer: already initialized, skipping\n");
 884                 return -EINVAL;
 885         }
 886
 887         gtdt = container_of(table, struct acpi_table_gtdt, header);
 888
 889         arch_timers_present |= ARCH_CP15_TIMER;
 890
 891         arch_timer_ppi[PHYS_SECURE_PPI] =
 892                 map_generic_timer_interrupt(gtdt->secure_el1_interrupt,
 893                 gtdt->secure_el1_flags);
 894
 895         arch_timer_ppi[PHYS_NONSECURE_PPI] =
 896                 map_generic_timer_interrupt(gtdt->non_secure_el1_interrupt,
 897                 gtdt->non_secure_el1_flags);
 898
 899         arch_timer_ppi[VIRT_PPI] =
 900                 map_generic_timer_interrupt(gtdt->virtual_timer_interrupt,
 901                 gtdt->virtual_timer_flags);
 902
 903         arch_timer_ppi[HYP_PPI] =
 904                 map_generic_timer_interrupt(gtdt->non_secure_el2_interrupt,
 905                 gtdt->non_secure_el2_flags);
 906
 907         /* Get the frequency from CNTFRQ */
 908         arch_timer_detect_rate(NULL, NULL);
 909
 910         /* Always-on capability */
 911         arch_timer_c3stop = !(gtdt->non_secure_el1_flags & ACPI_GTDT_ALWAYS_ON);
 912
 913         arch_timer_init();
 914         return 0;
 915 }
 916 CLOCKSOURCE_ACPI_DECLARE(arch_timer, ACPI_SIG_GTDT, arch_timer_acpi_init);
 917 #endif