scan-adoc-refs

   1 #!/usr/bin/perl
   2
   3 use strict;
   4 use warnings;
   5 use Getopt::Long;
   6 use IO::File;
   7 use JSON;
   8
   9 use Data::Dumper;
  10
  11 my $generate_depends;
  12
  13 GetOptions("depends=s" => \$generate_depends) or
  14     die("Error in command line arguments\n");
  15
  16
  17 my $environments = {
  18     default => 1,
  19     wiki => 1,
  20     manvolnum => 1,
  21     pvelogo => 0, # ignore
  22 };
  23
  24 my $fileinfo = {};
  25
  26 my $start_env = [];
  27 foreach my $e (keys %$environments) {
  28     push @$start_env, $e if $environments->{$e};
  29 }
  30
  31 my $env_stack = [$start_env];
  32 my $env_name_stack = [];
  33
  34 sub reset_environment_stack {
  35     $env_stack = [$start_env];
  36     $env_name_stack = [];
  37 }
  38
  39 sub push_environment {
  40     my ($env, $not) = @_;
  41
  42     die "undefined environment '$env'\n" if !defined($environments->{$env});
  43
  44     # FIXME: this seems wrong  (nested env?)?
  45     return if !$environments->{$env}; # do not track
  46
  47     if ($not) {
  48         my $new_env = [];
  49         foreach my $e (@{$env_stack->[-1]}) {
  50             if ($e ne $env) {
  51                 push @$new_env, $e;
  52             }
  53         }
  54         die "empty environment" if !scalar($new_env);
  55         push @$env_stack, $new_env;
  56     } else {
  57         push @$env_stack, [$env];
  58     }
  59
  60     push @$env_name_stack, $env;
  61 }
  62
  63 sub pop_environment {
  64     my ($env) = @_;
  65
  66     die "undefined environment '$env'\n" if !defined($environments->{$env});
  67
  68     return if !$environments->{$env}; # do not track
  69
  70     pop @$env_stack;
  71     my $res = pop @$env_name_stack;
  72
  73     die "environment missmatch ($res != $env)\n" if $res ne $env;
  74 }
  75
  76 sub register_include {
  77     my ($filename, $include_filename, $env_list) = @_;
  78
  79     foreach my $e (@$env_list) {
  80         $fileinfo->{include}->{$e}->{$filename}->{$include_filename} = 1;
  81     }
  82 }
  83
  84 sub register_blockid {
  85     my ($filename, $blockid, $reftext, $env_list) = @_;
  86
  87     foreach my $e (@$env_list) {
  88         my $fn = $fileinfo->{blockid}->{$e}->{$blockid};
  89         die "blockid '$blockid' already defined in $fn"
  90             if defined($fn);
  91         $fileinfo->{blockid}->{$e}->{$blockid} = $filename;
  92         $fileinfo->{reftext}->{$e}->{$blockid} = $reftext
  93             if defined($reftext);
  94     }
  95 }
  96
  97 sub register_title {
  98     my ($filename, $env, $doctype, $title, $blockid) = @_;
  99
 100     # fixme: what about other macros?
 101     $title =~ s/\{pve\}/Proxmox VE/g;
 102     $title =~ s!http://\S+\[(.*?)\]!$1!g;
 103
 104     # register document title (onyl once)
 105     if (!defined($fileinfo->{titles}->{$env}->{$filename})) {
 106
 107         $fileinfo->{titles}->{$env}->{$filename} = $title;
 108
 109         if (defined($doctype)) {
 110             $fileinfo->{doctype}->{$env}->{$filename} = $doctype;
 111         } else {
 112             die "unable to change title (no doctype)"
 113                 if !defined($fileinfo->{doctype}->{$env}->{$filename});
 114         }
 115     } elsif (!defined($doctype)) {
 116         # change title via :title: attribute
 117         $fileinfo->{titles}->{$env}->{$filename} = $title;
 118     }
 119
 120     if (defined($doctype) && ($env eq 'manvolnum') && ($doctype == 0)) {
 121         if ($title =~ m/.*\(([1-8])\)\s*$/) {
 122             $fileinfo->{mansection}->{$env}->{$filename} = $1;
 123         }
 124     }
 125
 126     if ($blockid) {
 127         die "internal error"
 128             if !defined($fileinfo->{blockid}->{$env}->{$blockid});
 129         $fileinfo->{reftitle}->{$env}->{$blockid} = $title;
 130     }
 131 }
 132
 133 sub scan_adoc_file {
 134     my ($filename) = @_;
 135
 136     reset_environment_stack();
 137
 138     # print "SCAN $filename\n";
 139
 140     my $fh = IO::File->new("$filename", "r") or
 141         die "unable to open file '$filename' - $!\n";
 142
 143     my $env_last_line = {};
 144     my $env_last_blockid = {};
 145
 146     while (defined (my $line = <$fh>)) {
 147         if ($line =~ m/^if(n?)def::(\S+)\[(.*)\]\s*$/) {
 148             my ($not, $env, $text) = ($1, $2, $3);
 149             die "unsuported ifdef usage - implement me" if $text;
 150             push_environment($env, $not);
 151             next;
 152         } elsif ($line =~ m/^endif::(\S+)\[(.*)\]\s*$/) {
 153             my ($env, $text) = ($1, $2);
 154             die "unsuported ifdef usage - implement me" if $text;
 155             pop_environment($env);
 156             next;
 157         } elsif ($line =~ m/^include::(\S+)\[.*\]\s*$/) {
 158             register_include($filename, $1, $env_stack->[-1]);
 159             next;
 160         }
 161
 162         # try to detect titles
 163         foreach my $e (@{$env_stack->[-1]}) {
 164             if ($line =~ m/^=====+$/) {
 165                 register_title($filename, $e, 0, $env_last_line->{$e},
 166                                $env_last_blockid->{$e});
 167             } elsif ($line =~ m/^-----+$/) {
 168                 register_title($filename, $e, 1, $env_last_line->{$e},
 169                                $env_last_blockid->{$e});
 170             } elsif ($line =~ m/^~~~~~+$/) {
 171                 register_title($filename, $e, 2, $env_last_line->{$e},
 172                                $env_last_blockid->{$e});
 173             } elsif ($line =~ m/^\^\^\^\^\^+$/) {
 174                 register_title($filename, $e, 3, $env_last_line->{$e},
 175                                $env_last_blockid->{$e});
 176             } elsif ($line =~ m/^= +(\S.*?)( +=)?$/) {
 177                 register_title($filename, $e, 0, $1, $env_last_blockid->{$e});
 178             } elsif ($line =~ m/^== +(\S.*?)( +==)?$/) {
 179                 register_title($filename, $e, 1, $1, $env_last_blockid->{$e});
 180             } elsif ($line =~ m/^=== +(\S.*?)( +===)?$/) {
 181                 register_title($filename, $e, 2, $1, $env_last_blockid->{$e});
 182             } elsif ($line =~ m/^==== +(\S.*?)( +====)?$/) {
 183                 register_title($filename, $e, 3, $1, $env_last_blockid->{$e});
 184             }
 185
 186             $env_last_line->{$e} = $line;
 187             chomp $env_last_line->{$e};
 188         }
 189
 190         if ($line =~ m/^:(\S+?):\s*(.*\S)?\s*$/) {
 191             my ($key, $value) = ($1, $2);
 192             if ($key eq 'pve-toplevel') {
 193
 194                 foreach my $e (@{$env_stack->[-1]}) {
 195                     my $title = $fileinfo->{titles}->{$e}->{$filename};
 196                     die "not title for toplevel file '$filename' (env=$e)\n"
 197                         if !defined($title);
 198                     $fileinfo->{toplevel}->{$e}->{$filename} = 1;
 199                 }
 200             } elsif ($key eq 'title') {
 201                 foreach my $e (@{$env_stack->[-1]}) {
 202                     register_title($filename, $e, undef, $value);
 203                 }
 204             }
 205         }
 206
 207         if ($line =~ m/^\[\[(.*)\]\]\s*$/) {
 208             my $blockid = $1;
 209             die "implement me" if $blockid =~m/,/;
 210             my $reftext = '';
 211             register_blockid($filename, $blockid, $reftext, $env_stack->[-1]);
 212             foreach my $e (@{$env_stack->[-1]}) {
 213                 $env_last_blockid->{$e} = $blockid;
 214             }
 215         }
 216
 217         if ($line =~ m/^\s*$/) {
 218             foreach my $e (@{$env_stack->[-1]}) {
 219                 delete $env_last_blockid->{$e};
 220             }
 221         }
 222
 223         # fixme: "anchor:"
 224         # bibliography anchors
 225         if ($line =~ m/\[\[\[([^\]]*)\]\]\]/) {
 226             my $blockid = $1;
 227             die "implement me" if $blockid =~m/,/;
 228             register_blockid($filename, $blockid, "&#91;$blockid&#93;", $env_stack->[-1]);
 229         }
 230     }
 231 }
 232
 233 my $scanned_files = {};
 234 while (my $filename = shift) {
 235     next if $filename !~ m/\.adoc$/; # skip attributes.txt
 236     next if $filename =~ m/-(opts|synopsis)\.adoc$/;
 237     next if $scanned_files->{$filename};
 238
 239     scan_adoc_file($filename);
 240     $scanned_files->{$filename} = 1;
 241 }
 242
 243 sub resolve_link_target {
 244     my ($env, $filename) = @_;
 245
 246     my $include_hash = $fileinfo->{include}->{$env};
 247
 248     my $repeat = 1;
 249
 250     while ($repeat) {
 251         $repeat = 0;
 252         foreach my $fn (keys %$include_hash) {
 253             if ($include_hash->{$fn}->{$filename}) {
 254                 next if ($fn eq 'pve-admin-guide.adoc') &&
 255                     $fileinfo->{outfile}->{$env}->{$filename};
 256                 $filename = $fn;
 257                 $repeat = 1;
 258                 last;
 259             }
 260         }
 261     }
 262
 263     return $filename;
 264 }
 265
 266
 267 # try to generate output file mapping
 268 foreach my $e (@$start_env) {
 269     my $toplevel_hash = $fileinfo->{toplevel}->{$e};
 270     foreach my $fn (sort keys %$toplevel_hash) {
 271         my $mansection = $fileinfo->{mansection}->{manvolnum}->{$fn};
 272         if ($e eq 'wiki') {
 273             my $realfn = $fn;
 274             $realfn =~ s/\.adoc$//;
 275             if (defined($mansection) && ($mansection == 5)) {
 276                 $realfn .= ".$mansection";
 277             }
 278             $realfn = "$realfn-plain.html";
 279             $fileinfo->{outfile}->{$e}->{$fn} = $realfn;
 280         } elsif ($e eq 'manvolnum') {
 281             my $realfn = $fn;
 282             $realfn =~ s/\.adoc$//;
 283             die "toplevel file '$fn' is not marked as manual page!" if !$mansection;
 284             $realfn .= ".$mansection";
 285             $fileinfo->{outfile}->{$e}->{$fn} = $realfn;
 286         } elsif ($e eq 'default') {
 287             my $realfn = $fn;
 288             $realfn =~ s/\.adoc$//;
 289             if (defined($mansection) && ($mansection == 5)) {
 290                 $realfn .= ".$mansection";
 291                 $realfn = "$realfn.html";
 292             } else {
 293                 if (($fn ne 'pve-admin-guide.adoc') &&
 294                     $fileinfo->{doctype}->{$e}->{$fn} == 0) {
 295                     $realfn = "chapter-$realfn.html";
 296                 } else {
 297                     $realfn = "$realfn.html";
 298                 }
 299             }
 300             $fileinfo->{outfile}->{$e}->{$fn} = $realfn;
 301         }
 302     }
 303 }
 304
 305 # now resolve blockids
 306 foreach my $e (@$start_env) {
 307     my $blockid_hash = $fileinfo->{blockid}->{$e};
 308     foreach my $blockid (keys %$blockid_hash) {
 309         my $fn = resolve_link_target($e, $blockid_hash->{$blockid});
 310         if ($e eq 'wiki') {
 311             my $title = $fileinfo->{titles}->{$e}->{$fn};
 312             $title =~ s/\s/_/g;
 313             die "found not title for '$fn' in env '$e'" if !$title;
 314             $fileinfo->{blockid_target}->{$e}->{$blockid} = "link:/wiki/$title#$blockid";
 315
 316             # we do not produce wiki pages for all content
 317             #my $realfn = $fileinfo->{outfile}->{$e}->{$fn};
 318             #warn "no output file mapping for '$fn' ($e)\n" if !$realfn;
 319
 320         } elsif ($e eq 'default') {
 321             my $realfn = $fileinfo->{outfile}->{$e}->{$fn} ||
 322                 die "no output file mapping for '$fn'\n";
 323             $fileinfo->{blockid_target}->{$e}->{$blockid} = "link:/pve-docs/$realfn#$blockid";
 324         } elsif ($e eq 'manvolnum') {
 325             # we do not produce manpages for all content
 326             # my $realfn = $fileinfo->{outfile}->{$e}->{$fn} ||
 327             # warn "no output file mapping for '$fn'\n";
 328             $fileinfo->{blockid_target}->{$e}->{$blockid} = $fn;
 329         }
 330     }
 331 }
 332
 333 my $makevar_hash = {};
 334
 335 sub makevar_define {
 336     my ($varname) = @_;
 337
 338     die "makefile variable '$varname' already defined\n"
 339         if exists($makevar_hash->{$varname});
 340
 341     $makevar_hash->{$varname} = {};
 342 }
 343
 344 sub makevar_list_add {
 345     my ($varname, $value) = @_;
 346
 347     die "makefile variable '$varname' not defined\n"
 348         if !exists($makevar_hash->{$varname});
 349
 350     $makevar_hash->{$varname}->{$value} = 1;
 351 }
 352
 353 sub makevar_dump {
 354     my $txt = '';
 355
 356     foreach my $varname (sort keys %$makevar_hash) {
 357         $txt .= "$varname =";
 358         foreach my $value (sort keys %{$makevar_hash->{$varname}}) {
 359             $txt .= " \\\n\t$value";
 360         }
 361         $txt .= "\n\n";
 362     }
 363
 364     return $txt;
 365 }
 366
 367 if ($generate_depends) {
 368
 369     my $tmpfilename = "${generate_depends}.tmp";
 370     my $outfh = IO::File->new($tmpfilename, "w") ||
 371         die "unable to open temporary file '$tmpfilename'\n";
 372
 373     my $depends = {};
 374     foreach my $e (@$start_env) {
 375         my $env_data = $fileinfo->{include}->{$e};
 376
 377         my $add_depends;
 378
 379         $add_depends = sub {
 380             my ($fn, $dep) = @_;
 381
 382             $depends->{$fn}->{$dep} = 1;
 383             foreach my $nd (keys %{$env_data->{$dep}}) {
 384                 &$add_depends($fn, $nd);
 385             }
 386         };
 387
 388         foreach my $fn (keys %$env_data) {
 389             foreach my $dep (keys %{$env_data->{$fn}}) {
 390                 &$add_depends($fn, $dep);
 391             }
 392         }
 393     }
 394
 395     my $depend_varname_hash = {};
 396
 397     foreach my $fn (sort keys %$depends) {
 398         my $basename = uc($fn);
 399         $basename =~s/\.adoc$//i;
 400         $basename =~s/[^A-Za-z0-9]/_/g;
 401
 402         my $varname1 = "${basename}_ADOCSOURCES";
 403         my $varname2 = "${basename}_ADOCDEPENDS";
 404
 405         makevar_define($varname1);
 406         makevar_define($varname2);
 407
 408         $depend_varname_hash->{$fn} = $varname2;
 409
 410         makevar_list_add($varname1, $fn);
 411         makevar_list_add($varname2, "\$\{$varname1\}");
 412
 413         foreach my $dep (sort keys %{$depends->{$fn}}) {
 414             if ($dep =~ m/-(opts|synopsis).adoc$/) {
 415                 makevar_list_add($varname2, $dep);
 416             } else {
 417                 makevar_list_add($varname1, $dep);
 418             }
 419         }
 420     }
 421
 422     my $man_sources_hash = {};
 423     foreach my $sourcefile (keys %{$fileinfo->{outfile}->{manvolnum}}) {
 424         $man_sources_hash->{$sourcefile} = 1;
 425         my $ihash = $fileinfo->{include}->{manvolnum}->{$sourcefile};
 426         foreach my $include (keys %$ihash) {
 427             if ($include !~ m/-(opts|synopsis).adoc$/) {
 428                 $man_sources_hash->{$include} = 1;
 429             }
 430         }
 431     }
 432
 433     my $varname = "MANUAL_SOURCES";
 434     makevar_define($varname);
 435     foreach my $sourcefile (sort keys %$man_sources_hash) {
 436         makevar_list_add($varname, $sourcefile);
 437     }
 438
 439     $varname = "CHAPTER_LIST";
 440     makevar_define($varname);
 441     my $filelist = $fileinfo->{outfile}->{default};
 442     foreach my $sourcefile (sort keys %$filelist) {
 443         my $target = $filelist->{$sourcefile};
 444         makevar_list_add($varname, $target);
 445     }
 446
 447     $varname = "MANUAL_PAGES";
 448     makevar_define($varname);
 449     $filelist = $fileinfo->{outfile}->{manvolnum};
 450     foreach my $manpage (sort keys %$filelist) {
 451         my $target = $filelist->{$manpage};
 452         makevar_list_add($varname, $target);
 453     }
 454
 455     $varname = "WIKI_IMPORTS";
 456     makevar_define($varname);
 457     $filelist = $fileinfo->{outfile}->{wiki};
 458     foreach my $sourcefile (sort keys %$filelist) {
 459         my $target = $filelist->{$sourcefile};
 460         makevar_list_add($varname, $target);
 461     }
 462
 463     my $res = makevar_dump();
 464
 465     my $make_targets = {};
 466     foreach my $e (@$start_env) {
 467         my $filelist = $fileinfo->{outfile}->{$e};
 468         foreach my $sourcefile (sort keys %$filelist) {
 469             my $varname = $depend_varname_hash->{$sourcefile};
 470             my $target = $filelist->{$sourcefile};
 471             my $dep = "\$\{$varname\}";
 472             $make_targets->{$target} = $dep;
 473             if ($e eq 'manvolnum') {
 474                 $make_targets->{"$target.html"} = $dep;
 475                 $make_targets->{"$target-plain.html"} = $dep;
 476             }
 477         }
 478     }
 479     foreach my $target (sort keys%$make_targets) {
 480         my $dep = $make_targets->{$target};
 481         $res .= "$target: $dep\n\n";
 482     }
 483
 484     print $outfh $res;
 485     close($outfh);
 486
 487     rename($tmpfilename, $generate_depends) ||
 488         die "rename failed - $!";
 489 }
 490
 491 print to_json($fileinfo, { pretty => 1,  canonical => 1 } );