scripts/docproc.c

   1 /*
   2  *      docproc is a simple preprocessor for the template files
   3  *      used as placeholders for the kernel internal documentation.
   4  *      docproc is used for documentation-frontend and
   5  *      dependency-generator.
   6  *      The two usages have in common that they require
   7  *      some knowledge of the .tmpl syntax, therefore they
   8  *      are kept together.
   9  *
  10  *      documentation-frontend
  11  *              Scans the template file and call kernel-doc for
  12  *              all occurrences of ![EIF]file
  13  *              Beforehand each referenced file is scanned for
  14  *              any symbols that are exported via these macros:
  15  *                      EXPORT_SYMBOL(), EXPORT_SYMBOL_GPL(), &
  16  *                      EXPORT_SYMBOL_GPL_FUTURE()
  17  *              This is used to create proper -function and
  18  *              -nofunction arguments in calls to kernel-doc.
  19  *              Usage: docproc doc file.tmpl
  20  *
  21  *      dependency-generator:
  22  *              Scans the template file and list all files
  23  *              referenced in a format recognized by make.
  24  *              Usage:  docproc depend file.tmpl
  25  *              Writes dependency information to stdout
  26  *              in the following format:
  27  *              file.tmpl src.c src2.c
  28  *              The filenames are obtained from the following constructs:
  29  *              !Efilename
  30  *              !Ifilename
  31  *              !Dfilename
  32  *              !Ffilename
  33  *              !Pfilename
  34  *
  35  */
  36
  37 #define _GNU_SOURCE
  38 #include <stdio.h>
  39 #include <stdlib.h>
  40 #include <string.h>
  41 #include <ctype.h>
  42 #include <unistd.h>
  43 #include <limits.h>
  44 #include <errno.h>
  45 #include <getopt.h>
  46 #include <sys/types.h>
  47 #include <sys/wait.h>
  48 #include <time.h>
  49
  50 /* exitstatus is used to keep track of any failing calls to kernel-doc,
  51  * but execution continues. */
  52 int exitstatus = 0;
  53
  54 typedef void DFL(char *);
  55 DFL *defaultline;
  56
  57 typedef void FILEONLY(char * file);
  58 FILEONLY *internalfunctions;
  59 FILEONLY *externalfunctions;
  60 FILEONLY *symbolsonly;
  61 FILEONLY *findall;
  62
  63 typedef void FILELINE(char * file, char * line);
  64 FILELINE * singlefunctions;
  65 FILELINE * entity_system;
  66 FILELINE * docsection;
  67
  68 #define MAXLINESZ     2048
  69 #define MAXFILES      250
  70 #define KERNELDOCPATH "scripts/"
  71 #define KERNELDOC     "kernel-doc"
  72 #define DOCBOOK       "-docbook"
  73 #define RST           "-rst"
  74 #define LIST          "-list"
  75 #define FUNCTION      "-function"
  76 #define NOFUNCTION    "-nofunction"
  77 #define NODOCSECTIONS "-no-doc-sections"
  78 #define SHOWNOTFOUND  "-show-not-found"
  79
  80 enum file_format {
  81         FORMAT_AUTO,
  82         FORMAT_DOCBOOK,
  83         FORMAT_RST,
  84 };
  85
  86 static enum file_format file_format = FORMAT_AUTO;
  87
  88 #define KERNELDOC_FORMAT        (file_format == FORMAT_RST ? RST : DOCBOOK)
  89
  90 static char *srctree, *kernsrctree;
  91
  92 static char **all_list = NULL;
  93 static int all_list_len = 0;
  94
  95 static void consume_symbol(const char *sym)
  96 {
  97         int i;
  98
  99         for (i = 0; i < all_list_len; i++) {
 100                 if (!all_list[i])
 101                         continue;
 102                 if (strcmp(sym, all_list[i]))
 103                         continue;
 104                 all_list[i] = NULL;
 105                 break;
 106         }
 107 }
 108
 109 static void usage (void)
 110 {
 111         fprintf(stderr, "Usage: docproc [{--docbook|--rst}] {doc|depend} file\n");
 112         fprintf(stderr, "Input is read from file.tmpl. Output is sent to stdout\n");
 113         fprintf(stderr, "doc: frontend when generating kernel documentation\n");
 114         fprintf(stderr, "depend: generate list of files referenced within file\n");
 115         fprintf(stderr, "Environment variable SRCTREE: absolute path to sources.\n");
 116         fprintf(stderr, "                     KBUILD_SRC: absolute path to kernel source tree.\n");
 117 }
 118
 119 /*
 120  * Execute kernel-doc with parameters given in svec
 121  */
 122 static void exec_kernel_doc(char **svec)
 123 {
 124         pid_t pid;
 125         int ret;
 126         char real_filename[PATH_MAX + 1];
 127         /* Make sure output generated so far are flushed */
 128         fflush(stdout);
 129         switch (pid=fork()) {
 130                 case -1:
 131                         perror("fork");
 132                         exit(1);
 133                 case  0:
 134                         memset(real_filename, 0, sizeof(real_filename));
 135                         strncat(real_filename, kernsrctree, PATH_MAX);
 136                         strncat(real_filename, "/" KERNELDOCPATH KERNELDOC,
 137                                         PATH_MAX - strlen(real_filename));
 138                         execvp(real_filename, svec);
 139                         fprintf(stderr, "exec ");
 140                         perror(real_filename);
 141                         exit(1);
 142                 default:
 143                         waitpid(pid, &ret ,0);
 144         }
 145         if (WIFEXITED(ret))
 146                 exitstatus |= WEXITSTATUS(ret);
 147         else
 148                 exitstatus = 0xff;
 149 }
 150
 151 /* Types used to create list of all exported symbols in a number of files */
 152 struct symbols
 153 {
 154         char *name;
 155 };
 156
 157 struct symfile
 158 {
 159         char *filename;
 160         struct symbols *symbollist;
 161         int symbolcnt;
 162 };
 163
 164 struct symfile symfilelist[MAXFILES];
 165 int symfilecnt = 0;
 166
 167 static void add_new_symbol(struct symfile *sym, char * symname)
 168 {
 169         sym->symbollist =
 170           realloc(sym->symbollist, (sym->symbolcnt + 1) * sizeof(char *));
 171         sym->symbollist[sym->symbolcnt++].name = strdup(symname);
 172 }
 173
 174 /* Add a filename to the list */
 175 static struct symfile * add_new_file(char * filename)
 176 {
 177         symfilelist[symfilecnt++].filename = strdup(filename);
 178         return &symfilelist[symfilecnt - 1];
 179 }
 180
 181 /* Check if file already are present in the list */
 182 static struct symfile * filename_exist(char * filename)
 183 {
 184         int i;
 185         for (i=0; i < symfilecnt; i++)
 186                 if (strcmp(symfilelist[i].filename, filename) == 0)
 187                         return &symfilelist[i];
 188         return NULL;
 189 }
 190
 191 /*
 192  * List all files referenced within the template file.
 193  * Files are separated by tabs.
 194  */
 195 static void adddep(char * file)            { printf("\t%s", file); }
 196 static void adddep2(char * file, char * line)     { line = line; adddep(file); }
 197 static void noaction(char * line)                  { line = line; }
 198 static void noaction2(char * file, char * line)   { file = file; line = line; }
 199
 200 /* Echo the line without further action */
 201 static void printline(char * line)               { printf("%s", line); }
 202
 203 /*
 204  * Find all symbols in filename that are exported with EXPORT_SYMBOL &
 205  * EXPORT_SYMBOL_GPL (& EXPORT_SYMBOL_GPL_FUTURE implicitly).
 206  * All symbols located are stored in symfilelist.
 207  */
 208 static void find_export_symbols(char * filename)
 209 {
 210         FILE * fp;
 211         struct symfile *sym;
 212         char line[MAXLINESZ];
 213         if (filename_exist(filename) == NULL) {
 214                 char real_filename[PATH_MAX + 1];
 215                 memset(real_filename, 0, sizeof(real_filename));
 216                 strncat(real_filename, srctree, PATH_MAX);
 217                 strncat(real_filename, "/", PATH_MAX - strlen(real_filename));
 218                 strncat(real_filename, filename,
 219                                 PATH_MAX - strlen(real_filename));
 220                 sym = add_new_file(filename);
 221                 fp = fopen(real_filename, "r");
 222                 if (fp == NULL) {
 223                         fprintf(stderr, "docproc: ");
 224                         perror(real_filename);
 225                         exit(1);
 226                 }
 227                 while (fgets(line, MAXLINESZ, fp)) {
 228                         char *p;
 229                         char *e;
 230                         if (((p = strstr(line, "EXPORT_SYMBOL_GPL")) != NULL) ||
 231                             ((p = strstr(line, "EXPORT_SYMBOL")) != NULL)) {
 232                                 /* Skip EXPORT_SYMBOL{_GPL} */
 233                                 while (isalnum(*p) || *p == '_')
 234                                         p++;
 235                                 /* Remove parentheses & additional whitespace */
 236                                 while (isspace(*p))
 237                                         p++;
 238                                 if (*p != '(')
 239                                         continue; /* Syntax error? */
 240                                 else
 241                                         p++;
 242                                 while (isspace(*p))
 243                                         p++;
 244                                 e = p;
 245                                 while (isalnum(*e) || *e == '_')
 246                                         e++;
 247                                 *e = '\0';
 248                                 add_new_symbol(sym, p);
 249                         }
 250                 }
 251                 fclose(fp);
 252         }
 253 }
 254
 255 /*
 256  * Document all external or internal functions in a file.
 257  * Call kernel-doc with following parameters:
 258  * kernel-doc [-docbook|-rst] -nofunction function_name1 filename
 259  * Function names are obtained from all the src files
 260  * by find_export_symbols.
 261  * intfunc uses -nofunction
 262  * extfunc uses -function
 263  */
 264 static void docfunctions(char * filename, char * type)
 265 {
 266         int i,j;
 267         int symcnt = 0;
 268         int idx = 0;
 269         char **vec;
 270
 271         for (i=0; i <= symfilecnt; i++)
 272                 symcnt += symfilelist[i].symbolcnt;
 273         vec = malloc((2 + 2 * symcnt + 3) * sizeof(char *));
 274         if (vec == NULL) {
 275                 perror("docproc: ");
 276                 exit(1);
 277         }
 278         vec[idx++] = KERNELDOC;
 279         vec[idx++] = KERNELDOC_FORMAT;
 280         vec[idx++] = NODOCSECTIONS;
 281         for (i=0; i < symfilecnt; i++) {
 282                 struct symfile * sym = &symfilelist[i];
 283                 for (j=0; j < sym->symbolcnt; j++) {
 284                         vec[idx++]     = type;
 285                         consume_symbol(sym->symbollist[j].name);
 286                         vec[idx++] = sym->symbollist[j].name;
 287                 }
 288         }
 289         vec[idx++]     = filename;
 290         vec[idx] = NULL;
 291         if (file_format == FORMAT_RST)
 292                 printf(".. %s\n", filename);
 293         else
 294                 printf("<!-- %s -->\n", filename);
 295         exec_kernel_doc(vec);
 296         fflush(stdout);
 297         free(vec);
 298 }
 299 static void intfunc(char * filename) {  docfunctions(filename, NOFUNCTION); }
 300 static void extfunc(char * filename) { docfunctions(filename, FUNCTION);   }
 301
 302 /*
 303  * Document specific function(s) in a file.
 304  * Call kernel-doc with the following parameters:
 305  * kernel-doc -docbook -function function1 [-function function2]
 306  */
 307 static void singfunc(char * filename, char * line)
 308 {
 309         char *vec[200]; /* Enough for specific functions */
 310         int i, idx = 0;
 311         int startofsym = 1;
 312         vec[idx++] = KERNELDOC;
 313         vec[idx++] = KERNELDOC_FORMAT;
 314         vec[idx++] = SHOWNOTFOUND;
 315
 316         /* Split line up in individual parameters preceded by FUNCTION */
 317         for (i=0; line[i]; i++) {
 318                 if (isspace(line[i])) {
 319                         line[i] = '\0';
 320                         startofsym = 1;
 321                         continue;
 322                 }
 323                 if (startofsym) {
 324                         startofsym = 0;
 325                         vec[idx++] = FUNCTION;
 326                         vec[idx++] = &line[i];
 327                 }
 328         }
 329         for (i = 0; i < idx; i++) {
 330                 if (strcmp(vec[i], FUNCTION))
 331                         continue;
 332                 consume_symbol(vec[i + 1]);
 333         }
 334         vec[idx++] = filename;
 335         vec[idx] = NULL;
 336         exec_kernel_doc(vec);
 337 }
 338
 339 /*
 340  * Insert specific documentation section from a file.
 341  * Call kernel-doc with the following parameters:
 342  * kernel-doc -docbook -function "doc section" filename
 343  */
 344 static void docsect(char *filename, char *line)
 345 {
 346         /* kerneldoc -docbook -show-not-found -function "section" file NULL */
 347         char *vec[7];
 348         char *s;
 349
 350         for (s = line; *s; s++)
 351                 if (*s == '\n')
 352                         *s = '\0';
 353
 354         if (asprintf(&s, "DOC: %s", line) < 0) {
 355                 perror("asprintf");
 356                 exit(1);
 357         }
 358         consume_symbol(s);
 359         free(s);
 360
 361         vec[0] = KERNELDOC;
 362         vec[1] = KERNELDOC_FORMAT;
 363         vec[2] = SHOWNOTFOUND;
 364         vec[3] = FUNCTION;
 365         vec[4] = line;
 366         vec[5] = filename;
 367         vec[6] = NULL;
 368         exec_kernel_doc(vec);
 369 }
 370
 371 static void find_all_symbols(char *filename)
 372 {
 373         char *vec[4]; /* kerneldoc -list file NULL */
 374         pid_t pid;
 375         int ret, i, count, start;
 376         char real_filename[PATH_MAX + 1];
 377         int pipefd[2];
 378         char *data, *str;
 379         size_t data_len = 0;
 380
 381         vec[0] = KERNELDOC;
 382         vec[1] = LIST;
 383         vec[2] = filename;
 384         vec[3] = NULL;
 385
 386         if (pipe(pipefd)) {
 387                 perror("pipe");
 388                 exit(1);
 389         }
 390
 391         switch (pid=fork()) {
 392                 case -1:
 393                         perror("fork");
 394                         exit(1);
 395                 case  0:
 396                         close(pipefd[0]);
 397                         dup2(pipefd[1], 1);
 398                         memset(real_filename, 0, sizeof(real_filename));
 399                         strncat(real_filename, kernsrctree, PATH_MAX);
 400                         strncat(real_filename, "/" KERNELDOCPATH KERNELDOC,
 401                                         PATH_MAX - strlen(real_filename));
 402                         execvp(real_filename, vec);
 403                         fprintf(stderr, "exec ");
 404                         perror(real_filename);
 405                         exit(1);
 406                 default:
 407                         close(pipefd[1]);
 408                         data = malloc(4096);
 409                         do {
 410                                 while ((ret = read(pipefd[0],
 411                                                    data + data_len,
 412                                                    4096)) > 0) {
 413                                         data_len += ret;
 414                                         data = realloc(data, data_len + 4096);
 415                                 }
 416                         } while (ret == -EAGAIN);
 417                         if (ret != 0) {
 418                                 perror("read");
 419                                 exit(1);
 420                         }
 421                         waitpid(pid, &ret ,0);
 422         }
 423         if (WIFEXITED(ret))
 424                 exitstatus |= WEXITSTATUS(ret);
 425         else
 426                 exitstatus = 0xff;
 427
 428         count = 0;
 429         /* poor man's strtok, but with counting */
 430         for (i = 0; i < data_len; i++) {
 431                 if (data[i] == '\n') {
 432                         count++;
 433                         data[i] = '\0';
 434                 }
 435         }
 436         start = all_list_len;
 437         all_list_len += count;
 438         all_list = realloc(all_list, sizeof(char *) * all_list_len);
 439         str = data;
 440         for (i = 0; i < data_len && start != all_list_len; i++) {
 441                 if (data[i] == '\0') {
 442                         all_list[start] = str;
 443                         str = data + i + 1;
 444                         start++;
 445                 }
 446         }
 447 }
 448
 449 /*
 450  * Terminate s at first space, if any. If there was a space, return pointer to
 451  * the character after that. Otherwise, return pointer to the terminating NUL.
 452  */
 453 static char *chomp(char *s)
 454 {
 455         while (*s && !isspace(*s))
 456                 s++;
 457
 458         if (*s)
 459                 *s++ = '\0';
 460
 461         return s;
 462 }
 463
 464 /* Return pointer to directive content, or NULL if not a directive. */
 465 static char *is_directive(char *line)
 466 {
 467         if (file_format == FORMAT_DOCBOOK && line[0] == '!')
 468                 return line + 1;
 469         else if (file_format == FORMAT_RST && !strncmp(line, ".. !", 4))
 470                 return line + 4;
 471
 472         return NULL;
 473 }
 474
 475 /*
 476  * Parse file, calling action specific functions for:
 477  * 1) Lines containing !E
 478  * 2) Lines containing !I
 479  * 3) Lines containing !D
 480  * 4) Lines containing !F
 481  * 5) Lines containing !P
 482  * 6) Lines containing !C
 483  * 7) Default lines - lines not matching the above
 484  */
 485 static void parse_file(FILE *infile)
 486 {
 487         char line[MAXLINESZ];
 488         char *p, *s;
 489         while (fgets(line, MAXLINESZ, infile)) {
 490                 p = is_directive(line);
 491                 if (!p) {
 492                         defaultline(line);
 493                         continue;
 494                 }
 495
 496                 switch (*p++) {
 497                 case 'E':
 498                         chomp(p);
 499                         externalfunctions(p);
 500                         break;
 501                 case 'I':
 502                         chomp(p);
 503                         internalfunctions(p);
 504                         break;
 505                 case 'D':
 506                         chomp(p);
 507                         symbolsonly(p);
 508                         break;
 509                 case 'F':
 510                         /* filename */
 511                         s = chomp(p);
 512                         /* function names */
 513                         while (isspace(*s))
 514                                 s++;
 515                         singlefunctions(p, s);
 516                         break;
 517                 case 'P':
 518                         /* filename */
 519                         s = chomp(p);
 520                         /* DOC: section name */
 521                         while (isspace(*s))
 522                                 s++;
 523                         docsection(p, s);
 524                         break;
 525                 case 'C':
 526                         chomp(p);
 527                         if (findall)
 528                                 findall(p);
 529                         break;
 530                 default:
 531                         defaultline(line);
 532                 }
 533         }
 534         fflush(stdout);
 535 }
 536
 537 /*
 538  * Is this a RestructuredText template?  Answer the question by seeing if its
 539  * name ends in ".rst".
 540  */
 541 static int is_rst(const char *file)
 542 {
 543         char *dot = strrchr(file, '.');
 544
 545         return dot && !strcmp(dot + 1, "rst");
 546 }
 547
 548 enum opts {
 549         OPT_DOCBOOK,
 550         OPT_RST,
 551         OPT_HELP,
 552 };
 553
 554 int main(int argc, char *argv[])
 555 {
 556         const char *subcommand, *filename;
 557         FILE * infile;
 558         int i;
 559
 560         srctree = getenv("SRCTREE");
 561         if (!srctree)
 562                 srctree = getcwd(NULL, 0);
 563         kernsrctree = getenv("KBUILD_SRC");
 564         if (!kernsrctree || !*kernsrctree)
 565                 kernsrctree = srctree;
 566
 567         for (;;) {
 568                 int c;
 569                 struct option opts[] = {
 570                         { "docbook",    no_argument, NULL, OPT_DOCBOOK },
 571                         { "rst",        no_argument, NULL, OPT_RST },
 572                         { "help",       no_argument, NULL, OPT_HELP },
 573                         {}
 574                 };
 575
 576                 c = getopt_long_only(argc, argv, "", opts, NULL);
 577                 if (c == -1)
 578                         break;
 579
 580                 switch (c) {
 581                 case OPT_DOCBOOK:
 582                         file_format = FORMAT_DOCBOOK;
 583                         break;
 584                 case OPT_RST:
 585                         file_format = FORMAT_RST;
 586                         break;
 587                 case OPT_HELP:
 588                         usage();
 589                         return 0;
 590                 default:
 591                 case '?':
 592                         usage();
 593                         return 1;
 594                 }
 595         }
 596
 597         argc -= optind;
 598         argv += optind;
 599
 600         if (argc != 2) {
 601                 usage();
 602                 exit(1);
 603         }
 604
 605         subcommand = argv[0];
 606         filename = argv[1];
 607
 608         if (file_format == FORMAT_AUTO)
 609                 file_format = is_rst(filename) ? FORMAT_RST : FORMAT_DOCBOOK;
 610
 611         /* Open file, exit on error */
 612         infile = fopen(filename, "r");
 613         if (infile == NULL) {
 614                 fprintf(stderr, "docproc: ");
 615                 perror(filename);
 616                 exit(2);
 617         }
 618
 619         if (strcmp("doc", subcommand) == 0) {
 620                 if (file_format == FORMAT_RST) {
 621                         time_t t = time(NULL);
 622                         printf(".. generated from %s by docproc %s\n",
 623                                filename, ctime(&t));
 624                 }
 625
 626                 /* Need to do this in two passes.
 627                  * First pass is used to collect all symbols exported
 628                  * in the various files;
 629                  * Second pass generate the documentation.
 630                  * This is required because some functions are declared
 631                  * and exported in different files :-((
 632                  */
 633                 /* Collect symbols */
 634                 defaultline       = noaction;
 635                 internalfunctions = find_export_symbols;
 636                 externalfunctions = find_export_symbols;
 637                 symbolsonly       = find_export_symbols;
 638                 singlefunctions   = noaction2;
 639                 docsection        = noaction2;
 640                 findall           = find_all_symbols;
 641                 parse_file(infile);
 642
 643                 /* Rewind to start from beginning of file again */
 644                 fseek(infile, 0, SEEK_SET);
 645                 defaultline       = printline;
 646                 internalfunctions = intfunc;
 647                 externalfunctions = extfunc;
 648                 symbolsonly       = printline;
 649                 singlefunctions   = singfunc;
 650                 docsection        = docsect;
 651                 findall           = NULL;
 652
 653                 parse_file(infile);
 654
 655                 for (i = 0; i < all_list_len; i++) {
 656                         if (!all_list[i])
 657                                 continue;
 658                         fprintf(stderr, "Warning: didn't use docs for %s\n",
 659                                 all_list[i]);
 660                 }
 661         } else if (strcmp("depend", subcommand) == 0) {
 662                 /* Create first part of dependency chain
 663                  * file.tmpl */
 664                 printf("%s\t", filename);
 665                 defaultline       = noaction;
 666                 internalfunctions = adddep;
 667                 externalfunctions = adddep;
 668                 symbolsonly       = adddep;
 669                 singlefunctions   = adddep2;
 670                 docsection        = adddep2;
 671                 findall           = adddep;
 672                 parse_file(infile);
 673                 printf("\n");
 674         } else {
 675                 fprintf(stderr, "Unknown option: %s\n", subcommand);
 676                 exit(1);
 677         }
 678         fclose(infile);
 679         fflush(stdout);
 680         return exitstatus;
 681 }