arch/x86/mm/kmemcheck/kmemcheck.c

   1 /**
   2  * kmemcheck - a heavyweight memory checker for the linux kernel
   3  * Copyright (C) 2007, 2008  Vegard Nossum <vegardno@ifi.uio.no>
   4  * (With a lot of help from Ingo Molnar and Pekka Enberg.)
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License (version 2) as
   8  * published by the Free Software Foundation.
   9  */
  10
  11 #include <linux/init.h>
  12 #include <linux/interrupt.h>
  13 #include <linux/kallsyms.h>
  14 #include <linux/kernel.h>
  15 #include <linux/kmemcheck.h>
  16 #include <linux/mm.h>
  17 #include <linux/page-flags.h>
  18 #include <linux/percpu.h>
  19 #include <linux/ptrace.h>
  20 #include <linux/string.h>
  21 #include <linux/types.h>
  22
  23 #include <asm/cacheflush.h>
  24 #include <asm/kmemcheck.h>
  25 #include <asm/pgtable.h>
  26 #include <asm/tlbflush.h>
  27
  28 #include "error.h"
  29 #include "opcode.h"
  30 #include "pte.h"
  31 #include "selftest.h"
  32 #include "shadow.h"
  33
  34
  35 #ifdef CONFIG_KMEMCHECK_DISABLED_BY_DEFAULT
  36 #  define KMEMCHECK_ENABLED 0
  37 #endif
  38
  39 #ifdef CONFIG_KMEMCHECK_ENABLED_BY_DEFAULT
  40 #  define KMEMCHECK_ENABLED 1
  41 #endif
  42
  43 #ifdef CONFIG_KMEMCHECK_ONESHOT_BY_DEFAULT
  44 #  define KMEMCHECK_ENABLED 2
  45 #endif
  46
  47 int kmemcheck_enabled = KMEMCHECK_ENABLED;
  48
  49 int __init kmemcheck_init(void)
  50 {
  51 #ifdef CONFIG_SMP
  52         /*
  53          * Limit SMP to use a single CPU. We rely on the fact that this code
  54          * runs before SMP is set up.
  55          */
  56         if (setup_max_cpus > 1) {
  57                 printk(KERN_INFO
  58                         "kmemcheck: Limiting number of CPUs to 1.\n");
  59                 setup_max_cpus = 1;
  60         }
  61 #endif
  62
  63         if (!kmemcheck_selftest()) {
  64                 printk(KERN_INFO "kmemcheck: self-tests failed; disabling\n");
  65                 kmemcheck_enabled = 0;
  66                 return -EINVAL;
  67         }
  68
  69         printk(KERN_INFO "kmemcheck: Initialized\n");
  70         return 0;
  71 }
  72
  73 early_initcall(kmemcheck_init);
  74
  75 /*
  76  * We need to parse the kmemcheck= option before any memory is allocated.
  77  */
  78 static int __init param_kmemcheck(char *str)
  79 {
  80         int val;
  81         int ret;
  82
  83         if (!str)
  84                 return -EINVAL;
  85
  86         ret = kstrtoint(str, 0, &val);
  87         if (ret)
  88                 return ret;
  89         kmemcheck_enabled = val;
  90         return 0;
  91 }
  92
  93 early_param("kmemcheck", param_kmemcheck);
  94
  95 int kmemcheck_show_addr(unsigned long address)
  96 {
  97         pte_t *pte;
  98
  99         pte = kmemcheck_pte_lookup(address);
 100         if (!pte)
 101                 return 0;
 102
 103         set_pte(pte, __pte(pte_val(*pte) | _PAGE_PRESENT));
 104         __flush_tlb_one(address);
 105         return 1;
 106 }
 107
 108 int kmemcheck_hide_addr(unsigned long address)
 109 {
 110         pte_t *pte;
 111
 112         pte = kmemcheck_pte_lookup(address);
 113         if (!pte)
 114                 return 0;
 115
 116         set_pte(pte, __pte(pte_val(*pte) & ~_PAGE_PRESENT));
 117         __flush_tlb_one(address);
 118         return 1;
 119 }
 120
 121 struct kmemcheck_context {
 122         bool busy;
 123         int balance;
 124
 125         /*
 126          * There can be at most two memory operands to an instruction, but
 127          * each address can cross a page boundary -- so we may need up to
 128          * four addresses that must be hidden/revealed for each fault.
 129          */
 130         unsigned long addr[4];
 131         unsigned long n_addrs;
 132         unsigned long flags;
 133
 134         /* Data size of the instruction that caused a fault. */
 135         unsigned int size;
 136 };
 137
 138 static DEFINE_PER_CPU(struct kmemcheck_context, kmemcheck_context);
 139
 140 bool kmemcheck_active(struct pt_regs *regs)
 141 {
 142         struct kmemcheck_context *data = this_cpu_ptr(&kmemcheck_context);
 143
 144         return data->balance > 0;
 145 }
 146
 147 /* Save an address that needs to be shown/hidden */
 148 static void kmemcheck_save_addr(unsigned long addr)
 149 {
 150         struct kmemcheck_context *data = this_cpu_ptr(&kmemcheck_context);
 151
 152         BUG_ON(data->n_addrs >= ARRAY_SIZE(data->addr));
 153         data->addr[data->n_addrs++] = addr;
 154 }
 155
 156 static unsigned int kmemcheck_show_all(void)
 157 {
 158         struct kmemcheck_context *data = this_cpu_ptr(&kmemcheck_context);
 159         unsigned int i;
 160         unsigned int n;
 161
 162         n = 0;
 163         for (i = 0; i < data->n_addrs; ++i)
 164                 n += kmemcheck_show_addr(data->addr[i]);
 165
 166         return n;
 167 }
 168
 169 static unsigned int kmemcheck_hide_all(void)
 170 {
 171         struct kmemcheck_context *data = this_cpu_ptr(&kmemcheck_context);
 172         unsigned int i;
 173         unsigned int n;
 174
 175         n = 0;
 176         for (i = 0; i < data->n_addrs; ++i)
 177                 n += kmemcheck_hide_addr(data->addr[i]);
 178
 179         return n;
 180 }
 181
 182 /*
 183  * Called from the #PF handler.
 184  */
 185 void kmemcheck_show(struct pt_regs *regs)
 186 {
 187         struct kmemcheck_context *data = this_cpu_ptr(&kmemcheck_context);
 188
 189         BUG_ON(!irqs_disabled());
 190
 191         if (unlikely(data->balance != 0)) {
 192                 kmemcheck_show_all();
 193                 kmemcheck_error_save_bug(regs);
 194                 data->balance = 0;
 195                 return;
 196         }
 197
 198         /*
 199          * None of the addresses actually belonged to kmemcheck. Note that
 200          * this is not an error.
 201          */
 202         if (kmemcheck_show_all() == 0)
 203                 return;
 204
 205         ++data->balance;
 206
 207         /*
 208          * The IF needs to be cleared as well, so that the faulting
 209          * instruction can run "uninterrupted". Otherwise, we might take
 210          * an interrupt and start executing that before we've had a chance
 211          * to hide the page again.
 212          *
 213          * NOTE: In the rare case of multiple faults, we must not override
 214          * the original flags:
 215          */
 216         if (!(regs->flags & X86_EFLAGS_TF))
 217                 data->flags = regs->flags;
 218
 219         regs->flags |= X86_EFLAGS_TF;
 220         regs->flags &= ~X86_EFLAGS_IF;
 221 }
 222
 223 /*
 224  * Called from the #DB handler.
 225  */
 226 void kmemcheck_hide(struct pt_regs *regs)
 227 {
 228         struct kmemcheck_context *data = this_cpu_ptr(&kmemcheck_context);
 229         int n;
 230
 231         BUG_ON(!irqs_disabled());
 232
 233         if (unlikely(data->balance != 1)) {
 234                 kmemcheck_show_all();
 235                 kmemcheck_error_save_bug(regs);
 236                 data->n_addrs = 0;
 237                 data->balance = 0;
 238
 239                 if (!(data->flags & X86_EFLAGS_TF))
 240                         regs->flags &= ~X86_EFLAGS_TF;
 241                 if (data->flags & X86_EFLAGS_IF)
 242                         regs->flags |= X86_EFLAGS_IF;
 243                 return;
 244         }
 245
 246         if (kmemcheck_enabled)
 247                 n = kmemcheck_hide_all();
 248         else
 249                 n = kmemcheck_show_all();
 250
 251         if (n == 0)
 252                 return;
 253
 254         --data->balance;
 255
 256         data->n_addrs = 0;
 257
 258         if (!(data->flags & X86_EFLAGS_TF))
 259                 regs->flags &= ~X86_EFLAGS_TF;
 260         if (data->flags & X86_EFLAGS_IF)
 261                 regs->flags |= X86_EFLAGS_IF;
 262 }
 263
 264 void kmemcheck_show_pages(struct page *p, unsigned int n)
 265 {
 266         unsigned int i;
 267
 268         for (i = 0; i < n; ++i) {
 269                 unsigned long address;
 270                 pte_t *pte;
 271                 unsigned int level;
 272
 273                 address = (unsigned long) page_address(&p[i]);
 274                 pte = lookup_address(address, &level);
 275                 BUG_ON(!pte);
 276                 BUG_ON(level != PG_LEVEL_4K);
 277
 278                 set_pte(pte, __pte(pte_val(*pte) | _PAGE_PRESENT));
 279                 set_pte(pte, __pte(pte_val(*pte) & ~_PAGE_HIDDEN));
 280                 __flush_tlb_one(address);
 281         }
 282 }
 283
 284 bool kmemcheck_page_is_tracked(struct page *p)
 285 {
 286         /* This will also check the "hidden" flag of the PTE. */
 287         return kmemcheck_pte_lookup((unsigned long) page_address(p));
 288 }
 289
 290 void kmemcheck_hide_pages(struct page *p, unsigned int n)
 291 {
 292         unsigned int i;
 293
 294         for (i = 0; i < n; ++i) {
 295                 unsigned long address;
 296                 pte_t *pte;
 297                 unsigned int level;
 298
 299                 address = (unsigned long) page_address(&p[i]);
 300                 pte = lookup_address(address, &level);
 301                 BUG_ON(!pte);
 302                 BUG_ON(level != PG_LEVEL_4K);
 303
 304                 set_pte(pte, __pte(pte_val(*pte) & ~_PAGE_PRESENT));
 305                 set_pte(pte, __pte(pte_val(*pte) | _PAGE_HIDDEN));
 306                 __flush_tlb_one(address);
 307         }
 308 }
 309
 310 /* Access may NOT cross page boundary */
 311 static void kmemcheck_read_strict(struct pt_regs *regs,
 312         unsigned long addr, unsigned int size)
 313 {
 314         void *shadow;
 315         enum kmemcheck_shadow status;
 316
 317         shadow = kmemcheck_shadow_lookup(addr);
 318         if (!shadow)
 319                 return;
 320
 321         kmemcheck_save_addr(addr);
 322         status = kmemcheck_shadow_test(shadow, size);
 323         if (status == KMEMCHECK_SHADOW_INITIALIZED)
 324                 return;
 325
 326         if (kmemcheck_enabled)
 327                 kmemcheck_error_save(status, addr, size, regs);
 328
 329         if (kmemcheck_enabled == 2)
 330                 kmemcheck_enabled = 0;
 331
 332         /* Don't warn about it again. */
 333         kmemcheck_shadow_set(shadow, size);
 334 }
 335
 336 bool kmemcheck_is_obj_initialized(unsigned long addr, size_t size)
 337 {
 338         enum kmemcheck_shadow status;
 339         void *shadow;
 340
 341         shadow = kmemcheck_shadow_lookup(addr);
 342         if (!shadow)
 343                 return true;
 344
 345         status = kmemcheck_shadow_test_all(shadow, size);
 346
 347         return status == KMEMCHECK_SHADOW_INITIALIZED;
 348 }
 349
 350 /* Access may cross page boundary */
 351 static void kmemcheck_read(struct pt_regs *regs,
 352         unsigned long addr, unsigned int size)
 353 {
 354         unsigned long page = addr & PAGE_MASK;
 355         unsigned long next_addr = addr + size - 1;
 356         unsigned long next_page = next_addr & PAGE_MASK;
 357
 358         if (likely(page == next_page)) {
 359                 kmemcheck_read_strict(regs, addr, size);
 360                 return;
 361         }
 362
 363         /*
 364          * What we do is basically to split the access across the
 365          * two pages and handle each part separately. Yes, this means
 366          * that we may now see reads that are 3 + 5 bytes, for
 367          * example (and if both are uninitialized, there will be two
 368          * reports), but it makes the code a lot simpler.
 369          */
 370         kmemcheck_read_strict(regs, addr, next_page - addr);
 371         kmemcheck_read_strict(regs, next_page, next_addr - next_page);
 372 }
 373
 374 static void kmemcheck_write_strict(struct pt_regs *regs,
 375         unsigned long addr, unsigned int size)
 376 {
 377         void *shadow;
 378
 379         shadow = kmemcheck_shadow_lookup(addr);
 380         if (!shadow)
 381                 return;
 382
 383         kmemcheck_save_addr(addr);
 384         kmemcheck_shadow_set(shadow, size);
 385 }
 386
 387 static void kmemcheck_write(struct pt_regs *regs,
 388         unsigned long addr, unsigned int size)
 389 {
 390         unsigned long page = addr & PAGE_MASK;
 391         unsigned long next_addr = addr + size - 1;
 392         unsigned long next_page = next_addr & PAGE_MASK;
 393
 394         if (likely(page == next_page)) {
 395                 kmemcheck_write_strict(regs, addr, size);
 396                 return;
 397         }
 398
 399         /* See comment in kmemcheck_read(). */
 400         kmemcheck_write_strict(regs, addr, next_page - addr);
 401         kmemcheck_write_strict(regs, next_page, next_addr - next_page);
 402 }
 403
 404 /*
 405  * Copying is hard. We have two addresses, each of which may be split across
 406  * a page (and each page will have different shadow addresses).
 407  */
 408 static void kmemcheck_copy(struct pt_regs *regs,
 409         unsigned long src_addr, unsigned long dst_addr, unsigned int size)
 410 {
 411         uint8_t shadow[8];
 412         enum kmemcheck_shadow status;
 413
 414         unsigned long page;
 415         unsigned long next_addr;
 416         unsigned long next_page;
 417
 418         uint8_t *x;
 419         unsigned int i;
 420         unsigned int n;
 421
 422         BUG_ON(size > sizeof(shadow));
 423
 424         page = src_addr & PAGE_MASK;
 425         next_addr = src_addr + size - 1;
 426         next_page = next_addr & PAGE_MASK;
 427
 428         if (likely(page == next_page)) {
 429                 /* Same page */
 430                 x = kmemcheck_shadow_lookup(src_addr);
 431                 if (x) {
 432                         kmemcheck_save_addr(src_addr);
 433                         for (i = 0; i < size; ++i)
 434                                 shadow[i] = x[i];
 435                 } else {
 436                         for (i = 0; i < size; ++i)
 437                                 shadow[i] = KMEMCHECK_SHADOW_INITIALIZED;
 438                 }
 439         } else {
 440                 n = next_page - src_addr;
 441                 BUG_ON(n > sizeof(shadow));
 442
 443                 /* First page */
 444                 x = kmemcheck_shadow_lookup(src_addr);
 445                 if (x) {
 446                         kmemcheck_save_addr(src_addr);
 447                         for (i = 0; i < n; ++i)
 448                                 shadow[i] = x[i];
 449                 } else {
 450                         /* Not tracked */
 451                         for (i = 0; i < n; ++i)
 452                                 shadow[i] = KMEMCHECK_SHADOW_INITIALIZED;
 453                 }
 454
 455                 /* Second page */
 456                 x = kmemcheck_shadow_lookup(next_page);
 457                 if (x) {
 458                         kmemcheck_save_addr(next_page);
 459                         for (i = n; i < size; ++i)
 460                                 shadow[i] = x[i - n];
 461                 } else {
 462                         /* Not tracked */
 463                         for (i = n; i < size; ++i)
 464                                 shadow[i] = KMEMCHECK_SHADOW_INITIALIZED;
 465                 }
 466         }
 467
 468         page = dst_addr & PAGE_MASK;
 469         next_addr = dst_addr + size - 1;
 470         next_page = next_addr & PAGE_MASK;
 471
 472         if (likely(page == next_page)) {
 473                 /* Same page */
 474                 x = kmemcheck_shadow_lookup(dst_addr);
 475                 if (x) {
 476                         kmemcheck_save_addr(dst_addr);
 477                         for (i = 0; i < size; ++i) {
 478                                 x[i] = shadow[i];
 479                                 shadow[i] = KMEMCHECK_SHADOW_INITIALIZED;
 480                         }
 481                 }
 482         } else {
 483                 n = next_page - dst_addr;
 484                 BUG_ON(n > sizeof(shadow));
 485
 486                 /* First page */
 487                 x = kmemcheck_shadow_lookup(dst_addr);
 488                 if (x) {
 489                         kmemcheck_save_addr(dst_addr);
 490                         for (i = 0; i < n; ++i) {
 491                                 x[i] = shadow[i];
 492                                 shadow[i] = KMEMCHECK_SHADOW_INITIALIZED;
 493                         }
 494                 }
 495
 496                 /* Second page */
 497                 x = kmemcheck_shadow_lookup(next_page);
 498                 if (x) {
 499                         kmemcheck_save_addr(next_page);
 500                         for (i = n; i < size; ++i) {
 501                                 x[i - n] = shadow[i];
 502                                 shadow[i] = KMEMCHECK_SHADOW_INITIALIZED;
 503                         }
 504                 }
 505         }
 506
 507         status = kmemcheck_shadow_test(shadow, size);
 508         if (status == KMEMCHECK_SHADOW_INITIALIZED)
 509                 return;
 510
 511         if (kmemcheck_enabled)
 512                 kmemcheck_error_save(status, src_addr, size, regs);
 513
 514         if (kmemcheck_enabled == 2)
 515                 kmemcheck_enabled = 0;
 516 }
 517
 518 enum kmemcheck_method {
 519         KMEMCHECK_READ,
 520         KMEMCHECK_WRITE,
 521 };
 522
 523 static void kmemcheck_access(struct pt_regs *regs,
 524         unsigned long fallback_address, enum kmemcheck_method fallback_method)
 525 {
 526         const uint8_t *insn;
 527         const uint8_t *insn_primary;
 528         unsigned int size;
 529
 530         struct kmemcheck_context *data = this_cpu_ptr(&kmemcheck_context);
 531
 532         /* Recursive fault -- ouch. */
 533         if (data->busy) {
 534                 kmemcheck_show_addr(fallback_address);
 535                 kmemcheck_error_save_bug(regs);
 536                 return;
 537         }
 538
 539         data->busy = true;
 540
 541         insn = (const uint8_t *) regs->ip;
 542         insn_primary = kmemcheck_opcode_get_primary(insn);
 543
 544         kmemcheck_opcode_decode(insn, &size);
 545
 546         switch (insn_primary[0]) {
 547 #ifdef CONFIG_KMEMCHECK_BITOPS_OK
 548                 /* AND, OR, XOR */
 549                 /*
 550                  * Unfortunately, these instructions have to be excluded from
 551                  * our regular checking since they access only some (and not
 552                  * all) bits. This clears out "bogus" bitfield-access warnings.
 553                  */
 554         case 0x80:
 555         case 0x81:
 556         case 0x82:
 557         case 0x83:
 558                 switch ((insn_primary[1] >> 3) & 7) {
 559                         /* OR */
 560                 case 1:
 561                         /* AND */
 562                 case 4:
 563                         /* XOR */
 564                 case 6:
 565                         kmemcheck_write(regs, fallback_address, size);
 566                         goto out;
 567
 568                         /* ADD */
 569                 case 0:
 570                         /* ADC */
 571                 case 2:
 572                         /* SBB */
 573                 case 3:
 574                         /* SUB */
 575                 case 5:
 576                         /* CMP */
 577                 case 7:
 578                         break;
 579                 }
 580                 break;
 581 #endif
 582
 583                 /* MOVS, MOVSB, MOVSW, MOVSD */
 584         case 0xa4:
 585         case 0xa5:
 586                 /*
 587                  * These instructions are special because they take two
 588                  * addresses, but we only get one page fault.
 589                  */
 590                 kmemcheck_copy(regs, regs->si, regs->di, size);
 591                 goto out;
 592
 593                 /* CMPS, CMPSB, CMPSW, CMPSD */
 594         case 0xa6:
 595         case 0xa7:
 596                 kmemcheck_read(regs, regs->si, size);
 597                 kmemcheck_read(regs, regs->di, size);
 598                 goto out;
 599         }
 600
 601         /*
 602          * If the opcode isn't special in any way, we use the data from the
 603          * page fault handler to determine the address and type of memory
 604          * access.
 605          */
 606         switch (fallback_method) {
 607         case KMEMCHECK_READ:
 608                 kmemcheck_read(regs, fallback_address, size);
 609                 goto out;
 610         case KMEMCHECK_WRITE:
 611                 kmemcheck_write(regs, fallback_address, size);
 612                 goto out;
 613         }
 614
 615 out:
 616         data->busy = false;
 617 }
 618
 619 bool kmemcheck_fault(struct pt_regs *regs, unsigned long address,
 620         unsigned long error_code)
 621 {
 622         pte_t *pte;
 623
 624         /*
 625          * XXX: Is it safe to assume that memory accesses from virtual 86
 626          * mode or non-kernel code segments will _never_ access kernel
 627          * memory (e.g. tracked pages)? For now, we need this to avoid
 628          * invoking kmemcheck for PnP BIOS calls.
 629          */
 630         if (regs->flags & X86_VM_MASK)
 631                 return false;
 632         if (regs->cs != __KERNEL_CS)
 633                 return false;
 634
 635         pte = kmemcheck_pte_lookup(address);
 636         if (!pte)
 637                 return false;
 638
 639         WARN_ON_ONCE(in_nmi());
 640
 641         if (error_code & 2)
 642                 kmemcheck_access(regs, address, KMEMCHECK_WRITE);
 643         else
 644                 kmemcheck_access(regs, address, KMEMCHECK_READ);
 645
 646         kmemcheck_show(regs);
 647         return true;
 648 }
 649
 650 bool kmemcheck_trap(struct pt_regs *regs)
 651 {
 652         if (!kmemcheck_active(regs))
 653                 return false;
 654
 655         /* We're done. */
 656         kmemcheck_hide(regs);
 657         return true;
 658 }