scan-adoc-refs

   1 #!/usr/bin/perl
   2
   3 use strict;
   4 use warnings;
   5 use IO::File;
   6 use JSON;
   7
   8 use Data::Dumper;
   9
  10 my $environments = {
  11     default => 1,
  12     wiki => 1,
  13     manvolnum => 1,
  14     pvelogo => 0, # ignore
  15 };
  16
  17 my $fileinfo = {
  18     outfile => {
  19         default => {
  20             "pve-admin-guide.adoc" => "pve-admin-guide.html",
  21             "datacenter.cfg.adoc" => "datacenter.cfg.5.html",
  22             "ha-manager.adoc" => "chapter-ha-manager.html",
  23             "pct.adoc" => "chapter-pct.html",
  24             "pve-bibliography.adoc" => "chapter-pve-bibliography.html",
  25             "pve-firewall.adoc" => "chapter-pve-firewall.html",
  26             "pve-installation.adoc" => "chapter-pve-installation.html",
  27             "pvecm.adoc" => "chapter-pvecm.html",
  28             "pvesm.adoc" => "chapter-pvesm.html",
  29             "pveum.adoc" => "chapter-pveum.html",
  30             "qm.adoc" => "chapter-qm.html",
  31             "sysadmin.adoc" => "chapter-sysadmin.html",
  32             "vzdump.adoc" => "chapter-vzdump.html",
  33             "pmxcfs.adoc" => "chapter-pmxcfs.html",
  34             "pve-faq.adoc" => "chapter-pve-faq.html",
  35         },
  36         manvolnum => {
  37             "ha-manager.adoc" => "ha-manager.1",
  38             "pct.adoc" => "pct.1",
  39             "pveam.adoc" => "pveam.1",
  40             "pveceph.adoc" => "pveceph.1",
  41             "pvecm.adoc" => "pvecm.1",
  42             "pveperf.adoc" => "pveperf.1",
  43             "pvesm.adoc" => "pvesm.1",
  44             "pvesubscription.adoc" => "pvesubscription.1",
  45             "pveum.adoc" => "pveum.1",
  46             "qm.adoc" => "qm.1",
  47             "qmrestore.adoc" => "qmrestore.1",
  48             "vzdump.adoc" => "vzdump.1",
  49             "datacenter.cfg.adoc" => "datacenter.cfg.5",
  50             "pct.conf.adoc" => "pct.conf.5",
  51             "qm.conf.adoc" => "qm.conf.5",
  52             "pmxcfs.adoc" => "pmxcfs.8",
  53             "pvedaemon.adoc" => "pvedaemon.8",
  54             "pve-firewall.adoc" => "pve-firewall.8",
  55             "pve-ha-crm.adoc" => "pve-ha-crm.8",
  56             "pve-ha-lrm.adoc" => "pve-ha-lrm.8",
  57             "pveproxy.adoc" => "pveproxy.8",
  58             "pvestatd.adoc" => "pvestatd.8",
  59             "spiceproxy.adoc" => "spiceproxy.8",
  60         },
  61         wiki => {
  62             "pve-usbstick.adoc" => "pve-usbstick-plain.html",
  63             "getting-help.adoc" => "getting-help-plain.html",
  64             "pve-system-requirements.adoc" => "pve-system-requirements-plain.html",
  65             "pve-network.adoc" => "pve-network-plain.html",
  66             "pve-package-repos.adoc" => "pve-package-repos-plain.html",
  67             "system-software-updates.adoc" => "system-software-updates-plain.html",
  68             "pve-disk-health-monitoring.adoc" => "pve-disk-health-monitoring-plain.html",
  69             "local-lvm.adoc" => "local-lvm-plain.html",
  70             "local-zfs.adoc" => "local-zfs-plain.html",
  71             "system-timesync.adoc" => "system-timesync-plain.html",
  72             "pve-installation.adoc" => "pve-installation-plain.html",
  73             "sysadmin.adoc" => "sysadmin-plain.html",
  74             "pvecm.adoc" => "pvecm-plain.html",
  75             "pmxcfs.adoc" => "pmxcfs-plain.html",
  76             "pvesm.adoc" => "pvesm-plain.html",
  77             "qm.adoc" => "qm-plain.html",
  78             "pve-firewall.adoc" => "pve-firewall-plain.html",
  79             "pveum.adoc" => "pveum-plain.html",
  80             "pct.adoc" => "pct-plain.html",
  81             "ha-manager.adoc" => "ha-manager-plain.html",
  82             "vzdump.adoc" => "vzdump-plain.html",
  83             "pve-faq.adoc" => "pve-faq-plain.html",
  84             "pve-bibliography.adoc" => "pve-bibliography-plain.html",
  85             "pve-storage-dir.adoc" => "pve-storage-dir-plain.html",
  86             "pve-storage-glusterfs.adoc" => "pve-storage-glusterfs-plain.html",
  87             "pve-storage-iscsi.adoc" => "pve-storage-iscsi-plain.html",
  88             "pve-storage-iscsidirect.adoc" => "pve-storage-iscsidirect-plain.html",
  89             "pve-storage-lvm.adoc" => "pve-storage-lvm-plain.html",
  90             "pve-storage-lvmthin.adoc" => "pve-storage-lvmthin-plain.html",
  91             "pve-storage-nfs.adoc" => "pve-storage-nfs-plain.html",
  92             "pve-storage-rbd.adoc" => "pve-storage-rbd-plain.html",
  93             "pve-storage-zfspool.adoc" => "pve-storage-zfspool-plain.html",
  94         },
  95     },
  96 };
  97
  98 my $start_env = [];
  99 foreach my $e (keys %$environments) {
 100     push @$start_env, $e if $environments->{$e};
 101 }
 102
 103 my $env_stack = [$start_env];
 104 my $env_name_stack = [];
 105
 106 sub reset_environment_stack {
 107     $env_stack = [$start_env];
 108     $env_name_stack = [];
 109 }
 110
 111 sub push_environment {
 112     my ($env, $not) = @_;
 113
 114     die "undefined environment '$env'\n" if !defined($environments->{$env});
 115
 116     # FIXME: this seems wrong  (nested env?)?
 117     return if !$environments->{$env}; # do not track
 118
 119     if ($not) {
 120         my $new_env = [];
 121         foreach my $e (@{$env_stack->[-1]}) {
 122             if ($e ne $env) {
 123                 push @$new_env, $e;
 124             }
 125         }
 126         die "empty environment" if !scalar($new_env);
 127         push @$env_stack, $new_env;
 128     } else {
 129         push @$env_stack, [$env];
 130     }
 131
 132     push @$env_name_stack, $env;
 133 }
 134
 135 sub pop_environment {
 136     my ($env) = @_;
 137
 138     die "undefined environment '$env'\n" if !defined($environments->{$env});
 139
 140     return if !$environments->{$env}; # do not track
 141
 142     pop @$env_stack;
 143     my $res = pop @$env_name_stack;
 144
 145     die "environment missmatch ($res != $env)\n" if $res ne $env;
 146 }
 147
 148 sub register_include {
 149     my ($filename, $include_filename, $env_list) = @_;
 150
 151     return if $include_filename !~ m/\.adoc$/; # skip attributes.txt
 152
 153     foreach my $e (@$env_list) {
 154         $fileinfo->{include}->{$e}->{$filename}->{$include_filename} = 1;
 155     }
 156 }
 157
 158 sub register_blockid {
 159     my ($filename, $blockid, $reftext, $env_list) = @_;
 160
 161     foreach my $e (@$env_list) {
 162         my $fn = $fileinfo->{blockid}->{$e}->{$blockid};
 163         die "blockid '$blockid' already defined in $fn"
 164             if defined($fn);
 165         $fileinfo->{blockid}->{$e}->{$blockid} = $filename;
 166         $fileinfo->{reftext}->{$e}->{$blockid} = $reftext
 167             if defined($reftext);
 168     }
 169 }
 170
 171 sub register_title {
 172     my ($filename, $env, $doctype, $title, $blockid) = @_;
 173
 174     # fixme: what about other macros?
 175     $title =~ s/\{pve\}/Proxmox VE/g;
 176     $title =~ s!http://\S+\[(.*?)\]!$1!g;
 177
 178     # register document title (onyl once)
 179     if (!defined($fileinfo->{titles}->{$env}->{$filename})) {
 180
 181         $fileinfo->{titles}->{$env}->{$filename} = $title;
 182
 183         if (defined($doctype)) {
 184             $fileinfo->{doctype}->{$env}->{$filename} = $doctype;
 185         } else {
 186             die "unable to change title (no doctype)"
 187                 if !defined($fileinfo->{doctype}->{$env}->{$filename});
 188         }
 189
 190         if (defined($doctype) && ($env eq 'manvolnum') && ($doctype == 0)) {
 191             if ($title =~ m/.*\(([1-8])\)\s*$/) {
 192                 $fileinfo->{mansection}->{$env}->{$filename} = $1;
 193             }
 194         }
 195     }
 196
 197     if ($blockid) {
 198         die "internal error"
 199             if !defined($fileinfo->{blockid}->{$env}->{$blockid});
 200         $fileinfo->{reftitle}->{$env}->{$blockid} = $title;
 201     }
 202 }
 203
 204 sub scan_adoc_file {
 205     my ($filename) = @_;
 206
 207     reset_environment_stack();
 208
 209     # print "SCAN $filename\n";
 210
 211     my $fh = IO::File->new("$filename", "r") or
 212         die "unable to open file '$filename' - $!\n";
 213
 214     my $env_last_line = {};
 215     my $env_last_blockid = {};
 216
 217     while (defined (my $line = <$fh>)) {
 218         if ($line =~ m/^if(n?)def::(\S+)\[(.*)\]\s*$/) {
 219             my ($not, $env, $text) = ($1, $2, $3);
 220             die "unsuported ifdef usage - implement me" if $text;
 221             push_environment($env, $not);
 222             next;
 223         } elsif ($line =~ m/^endif::(\S+)\[(.*)\]\s*$/) {
 224             my ($env, $text) = ($1, $2);
 225             die "unsuported ifdef usage - implement me" if $text;
 226             pop_environment($env);
 227             next;
 228         } elsif ($line =~ m/^include::(\S+)\[.*\]\s*$/) {
 229             register_include($filename, $1, $env_stack->[-1]);
 230             next;
 231         }
 232
 233         # try to detect titles
 234         foreach my $e (@{$env_stack->[-1]}) {
 235             if ($line =~ m/^=====+$/) {
 236                 register_title($filename, $e, 0, $env_last_line->{$e},
 237                                $env_last_blockid->{$e});
 238             } elsif ($line =~ m/^-----+$/) {
 239                 register_title($filename, $e, 1, $env_last_line->{$e},
 240                                $env_last_blockid->{$e});
 241             } elsif ($line =~ m/^~~~~~+$/) {
 242                 register_title($filename, $e, 2, $env_last_line->{$e},
 243                                $env_last_blockid->{$e});
 244             } elsif ($line =~ m/^\^\^\^\^\^+$/) {
 245                 register_title($filename, $e, 3, $env_last_line->{$e},
 246                                $env_last_blockid->{$e});
 247             } elsif ($line =~ m/^= +(\S.*?)( +=)?$/) {
 248                 register_title($filename, $e, 0, $1, $env_last_blockid->{$e});
 249             } elsif ($line =~ m/^== +(\S.*?)( +==)?$/) {
 250                 register_title($filename, $e, 1, $1, $env_last_blockid->{$e});
 251             } elsif ($line =~ m/^=== +(\S.*?)( +===)?$/) {
 252                 register_title($filename, $e, 2, $1, $env_last_blockid->{$e});
 253             } elsif ($line =~ m/^==== +(\S.*?)( +====)?$/) {
 254                 register_title($filename, $e, 3, $1, $env_last_blockid->{$e});
 255             }
 256
 257             $env_last_line->{$e} = $line;
 258             chomp $env_last_line->{$e};
 259         }
 260
 261         if ($line =~ m/^:(\S+?):\s*(.*\S)?\s*$/) {
 262             my ($key, $value) = ($1, $2);
 263             if ($key eq 'pve-toplevel') {
 264
 265                 foreach my $e (@{$env_stack->[-1]}) {
 266                     my $title = $fileinfo->{titles}->{$e}->{$filename};
 267                     die "not title for toplevel file '$filename' (env=$e)\n"
 268                         if !defined($title);
 269                     $fileinfo->{toplevel}->{$e}->{$filename} = 1;
 270                 }
 271             } elsif ($key eq 'title') {
 272                 foreach my $e (@{$env_stack->[-1]}) {
 273                     register_title($filename, $e, undef, $value);
 274                 }
 275             }
 276         }
 277
 278         if ($line =~ m/^\[\[(.*)\]\]\s*$/) {
 279             my $blockid = $1;
 280             die "implement me" if $blockid =~m/,/;
 281             my $reftext = '';
 282             register_blockid($filename, $blockid, $reftext, $env_stack->[-1]);
 283             foreach my $e (@{$env_stack->[-1]}) {
 284                 $env_last_blockid->{$e} = $blockid;
 285             }
 286         }
 287
 288         if ($line =~ m/^\s*$/) {
 289             foreach my $e (@{$env_stack->[-1]}) {
 290                 delete $env_last_blockid->{$e};
 291             }
 292         }
 293
 294         # fixme: "anchor:"
 295         # bibliography anchors
 296         if ($line =~ m/\[\[\[([^\]]*)\]\]\]/) {
 297             my $blockid = $1;
 298             die "implement me" if $blockid =~m/,/;
 299             register_blockid($filename, $blockid, "&#91;$blockid&#93;", $env_stack->[-1]);
 300         }
 301     }
 302 }
 303
 304 my $scanned_files = {};
 305 while (my $filename = shift) {
 306     next if $filename !~ m/\.adoc$/; # skip attributes.txt
 307     next if $scanned_files->{$filename};
 308
 309     scan_adoc_file($filename);
 310     $scanned_files->{$filename} = 1;
 311 }
 312
 313 sub resolve_link_target {
 314     my ($env, $filename) = @_;
 315
 316     my $include_hash = $fileinfo->{include}->{$env};
 317
 318     my $repeat = 1;
 319
 320     while ($repeat) {
 321         $repeat = 0;
 322         foreach my $fn (keys %$include_hash) {
 323             if ($include_hash->{$fn}->{$filename}) {
 324                 next if ($fn eq 'pve-admin-guide.adoc') &&
 325                     $fileinfo->{outfile}->{$env}->{$filename};
 326                 $filename = $fn;
 327                 $repeat = 1;
 328                 last;
 329             }
 330         }
 331     }
 332
 333     return $filename;
 334 }
 335
 336 # now resolve blockids
 337 foreach my $e (@$start_env) {
 338     my $blockid_hash = $fileinfo->{blockid}->{$e};
 339     foreach my $blockid (keys %$blockid_hash) {
 340         my $fn = resolve_link_target($e, $blockid_hash->{$blockid});
 341         if ($e eq 'wiki') {
 342             my $title = $fileinfo->{titles}->{$e}->{$fn};
 343             $title =~ s/\s/_/g;
 344             die "found not title for '$fn' in env '$e'" if !$title;
 345             $fileinfo->{blockid_target}->{$e}->{$blockid} = "link:/wiki/$title#$blockid";
 346         } elsif ($e eq 'default') {
 347             my $realfn = $fileinfo->{outfile}->{$e}->{$fn} ||
 348                 die "no output file mapping for '$fn'\n";
 349             $fileinfo->{blockid_target}->{$e}->{$blockid} = "link:/pve-docs/$realfn#$blockid";
 350         } elsif ($e eq 'manvolnum') {
 351             # we do not produce manpages for all content
 352             # my $realfn = $fileinfo->{outfile}->{$e}->{$fn} ||
 353             # warn "no output file mapping for '$fn'\n";
 354             $fileinfo->{blockid_target}->{$e}->{$blockid} = $fn;
 355         }
 356     }
 357 }
 358
 359
 360 print to_json($fileinfo, { pretty => 1,  canonical => 1 } );