551791f2fa7fb47f02a7979b1a02706fc6afb915
[pve-docs.git] / scan-adoc-refs
1 #!/usr/bin/perl
2
3 use strict;
4 use warnings;
5 use IO::File;
6 use JSON;
7
8 use Data::Dumper;
9
10 my $environments = {
11     default => 1,
12     wiki => 1,
13     manvolnum => 1,
14     pvelogo => 0, # ignore
15 };
16
17 my $fileinfo = {
18     outfile => {
19         default => {
20             "pve-admin-guide.adoc" => "pve-admin-guide.html",
21             "datacenter.cfg.adoc" => "datacenter.cfg.5.html",
22             "ha-manager.adoc" => "chapter-ha-manager.html",
23             "pct.adoc" => "chapter-pct.html",
24             "pve-bibliography.adoc" => "chapter-pve-bibliography.html",
25             "pve-firewall.adoc" => "chapter-pve-firewall.html",
26             "pve-installation.adoc" => "chapter-pve-installation.html",
27             "pvecm.adoc" => "chapter-pvecm.html",
28             "pvesm.adoc" => "chapter-pvesm.html",
29             "pveum.adoc" => "chapter-pveum.html",
30             "qm.adoc" => "chapter-qm.html",
31             "sysadmin.adoc" => "chapter-sysadmin.html",
32             "vzdump.adoc" => "chapter-vzdump.html",
33             "pmxcfs.adoc" => "chapter-pmxcfs.html",
34             "pve-faq.adoc" => "chapter-pve-faq.html",
35         },
36         manvolnum => {
37             "ha-manager.adoc" => "ha-manager.1",
38             "pct.adoc" => "pct.1",
39             "pveam.adoc" => "pveam.1",
40             "pveceph.adoc" => "pveceph.1",
41             "pvecm.adoc" => "pvecm.1",
42             "pveperf.adoc" => "pveperf.1",
43             "pvesm.adoc" => "pvesm.1",
44             "pvesubscription.adoc" => "pvesubscription.1",
45             "pveum.adoc" => "pveum.1",
46             "qm.adoc" => "qm.1",
47             "qmrestore.adoc" => "qmrestore.1",
48             "vzdump.adoc" => "vzdump.1",
49             "datacenter.cfg.adoc" => "datacenter.cfg.5",
50             "pct.conf.adoc" => "pct.conf.5",
51             "qm.conf.adoc" => "qm.conf.5",
52             "pmxcfs.adoc" => "pmxcfs.8",
53             "pvedaemon.adoc" => "pvedaemon.8",
54             "pve-firewall.adoc" => "pve-firewall.8",
55             "pve-ha-crm.adoc" => "pve-ha-crm.8",
56             "pve-ha-lrm.adoc" => "pve-ha-lrm.8",
57             "pveproxy.adoc" => "pveproxy.8",
58             "pvestatd.adoc" => "pvestatd.8",
59             "spiceproxy.adoc" => "spiceproxy.8",
60         },
61         wiki => {
62             "pve-usbstick.adoc" => "pve-usbstick-plain.html",
63             "getting-help.adoc" => "getting-help-plain.html",
64             "pve-system-requirements.adoc" => "pve-system-requirements-plain.html",
65             "pve-network.adoc" => "pve-network-plain.html",
66             "pve-package-repos.adoc" => "pve-package-repos-plain.html",
67             "system-software-updates.adoc" => "system-software-updates-plain.html",
68             "pve-disk-health-monitoring.adoc" => "pve-disk-health-monitoring-plain.html",
69             "local-lvm.adoc" => "local-lvm-plain.html",
70             "local-zfs.adoc" => "local-zfs-plain.html",
71             "system-timesync.adoc" => "system-timesync-plain.html",
72             "pve-installation.adoc" => "pve-installation-plain.html",
73             "sysadmin.adoc" => "sysadmin-plain.html",
74             "pvecm.adoc" => "pvecm-plain.html",
75             "pmxcfs.adoc" => "pmxcfs-plain.html",
76             "pvesm.adoc" => "pvesm-plain.html",
77             "qm.adoc" => "qm-plain.html",
78             "pve-firewall.adoc" => "pve-firewall-plain.html",
79             "pveum.adoc" => "pveum-plain.html",
80             "pct.adoc" => "pct-plain.html",
81             "ha-manager.adoc" => "ha-manager-plain.html",
82             "vzdump.adoc" => "vzdump-plain.html",
83             "pve-faq.adoc" => "pve-faq-plain.html",
84             "pve-bibliography.adoc" => "pve-bibliography-plain.html",
85             "pve-storage-dir.adoc" => "pve-storage-dir-plain.html",
86             "pve-storage-glusterfs.adoc" => "pve-storage-glusterfs-plain.html",
87             "pve-storage-iscsi.adoc" => "pve-storage-iscsi-plain.html",
88             "pve-storage-iscsidirect.adoc" => "pve-storage-iscsidirect-plain.html",
89             "pve-storage-lvm.adoc" => "pve-storage-lvm-plain.html",
90             "pve-storage-lvmthin.adoc" => "pve-storage-lvmthin-plain.html",
91             "pve-storage-nfs.adoc" => "pve-storage-nfs-plain.html",
92             "pve-storage-rbd.adoc" => "pve-storage-rbd-plain.html",
93             "pve-storage-zfspool.adoc" => "pve-storage-zfspool-plain.html",
94         },
95     },
96 };
97
98 my $start_env = [];
99 foreach my $e (keys %$environments) {
100     push @$start_env, $e if $environments->{$e};
101 }
102
103 my $env_stack = [$start_env];
104 my $env_name_stack = [];
105
106 sub reset_environment_stack {
107     $env_stack = [$start_env];
108     $env_name_stack = [];
109 }
110
111 sub push_environment {
112     my ($env, $not) = @_;
113
114     die "undefined environment '$env'\n" if !defined($environments->{$env});
115
116     # FIXME: this seems wrong  (nested env?)?
117     return if !$environments->{$env}; # do not track
118
119     if ($not) {
120         my $new_env = [];
121         foreach my $e (@{$env_stack->[-1]}) {
122             if ($e ne $env) {
123                 push @$new_env, $e;
124             }
125         }
126         die "empty environment" if !scalar($new_env);
127         push @$env_stack, $new_env;
128     } else {
129         push @$env_stack, [$env];
130     }
131
132     push @$env_name_stack, $env;
133 }
134
135 sub pop_environment {
136     my ($env) = @_;
137
138     die "undefined environment '$env'\n" if !defined($environments->{$env});
139
140     return if !$environments->{$env}; # do not track
141
142     pop @$env_stack;
143     my $res = pop @$env_name_stack;
144
145     die "environment missmatch ($res != $env)\n" if $res ne $env;
146 }
147
148 sub register_include {
149     my ($filename, $include_filename, $env_list) = @_;
150
151     return if $include_filename !~ m/\.adoc$/; # skip attributes.txt
152
153     foreach my $e (@$env_list) {
154         $fileinfo->{include}->{$e}->{$filename}->{$include_filename} = 1;
155     }
156 }
157
158 sub register_blockid {
159     my ($filename, $blockid, $reftext, $env_list) = @_;
160
161     foreach my $e (@$env_list) {
162         my $fn = $fileinfo->{blockid}->{$e}->{$blockid};
163         die "blockid '$blockid' already defined in $fn"
164             if defined($fn);
165         $fileinfo->{blockid}->{$e}->{$blockid} = $filename;
166         $fileinfo->{reftext}->{$e}->{$blockid} = $reftext
167             if defined($reftext);
168     }
169 }
170
171 sub register_title {
172     my ($filename, $env, $doctype, $title, $blockid) = @_;
173
174     # fixme: what about other macros?
175     $title =~ s/\{pve\}/Proxmox VE/g;
176     $title =~ s!http://\S+\[(.*?)\]!$1!g;
177
178     # register document title (onyl once)
179     if (!defined($fileinfo->{titles}->{$env}->{$filename})) {
180
181         $fileinfo->{titles}->{$env}->{$filename} = $title;
182
183         if (defined($doctype)) {
184             $fileinfo->{doctype}->{$env}->{$filename} = $doctype;
185         } else {
186             die "unable to change title (no doctype)"
187                 if !defined($fileinfo->{doctype}->{$env}->{$filename});
188         }
189
190         if (defined($doctype) && ($env eq 'manvolnum') && ($doctype == 0)) {
191             if ($title =~ m/.*\(([1-8])\)\s*$/) {
192                 $fileinfo->{mansection}->{$env}->{$filename} = $1;
193             }
194         }
195     }
196
197     if ($blockid) {
198         die "internal error"
199             if !defined($fileinfo->{blockid}->{$env}->{$blockid});
200         $fileinfo->{reftitle}->{$env}->{$blockid} = $title;
201     }
202 }
203
204 sub scan_adoc_file {
205     my ($filename) = @_;
206
207     reset_environment_stack();
208
209     # print "SCAN $filename\n";
210
211     my $fh = IO::File->new("$filename", "r") or
212         die "unable to open file '$filename' - $!\n";
213
214     my $env_last_line = {};
215     my $env_last_blockid = {};
216
217     while (defined (my $line = <$fh>)) {
218         if ($line =~ m/^if(n?)def::(\S+)\[(.*)\]\s*$/) {
219             my ($not, $env, $text) = ($1, $2, $3);
220             die "unsuported ifdef usage - implement me" if $text;
221             push_environment($env, $not);
222             next;
223         } elsif ($line =~ m/^endif::(\S+)\[(.*)\]\s*$/) {
224             my ($env, $text) = ($1, $2);
225             die "unsuported ifdef usage - implement me" if $text;
226             pop_environment($env);
227             next;
228         } elsif ($line =~ m/^include::(\S+)\[.*\]\s*$/) {
229             register_include($filename, $1, $env_stack->[-1]);
230             next;
231         }
232
233         # try to detect titles
234         foreach my $e (@{$env_stack->[-1]}) {
235             if ($line =~ m/^=====+$/) {
236                 register_title($filename, $e, 0, $env_last_line->{$e},
237                                $env_last_blockid->{$e});
238             } elsif ($line =~ m/^-----+$/) {
239                 register_title($filename, $e, 1, $env_last_line->{$e},
240                                $env_last_blockid->{$e});
241             } elsif ($line =~ m/^~~~~~+$/) {
242                 register_title($filename, $e, 2, $env_last_line->{$e},
243                                $env_last_blockid->{$e});
244             } elsif ($line =~ m/^\^\^\^\^\^+$/) {
245                 register_title($filename, $e, 3, $env_last_line->{$e},
246                                $env_last_blockid->{$e});
247             } elsif ($line =~ m/^= +(\S.*?)( +=)?$/) {
248                 register_title($filename, $e, 0, $1, $env_last_blockid->{$e});
249             } elsif ($line =~ m/^== +(\S.*?)( +==)?$/) {
250                 register_title($filename, $e, 1, $1, $env_last_blockid->{$e});
251             } elsif ($line =~ m/^=== +(\S.*?)( +===)?$/) {
252                 register_title($filename, $e, 2, $1, $env_last_blockid->{$e});
253             } elsif ($line =~ m/^==== +(\S.*?)( +====)?$/) {
254                 register_title($filename, $e, 3, $1, $env_last_blockid->{$e});
255             }
256
257             $env_last_line->{$e} = $line;
258             chomp $env_last_line->{$e};
259         }
260
261         if ($line =~ m/^:(\S+?):\s*(.*\S)?\s*$/) {
262             my ($key, $value) = ($1, $2);
263             if ($key eq 'pve-toplevel') {
264
265                 foreach my $e (@{$env_stack->[-1]}) {
266                     my $title = $fileinfo->{titles}->{$e}->{$filename};
267                     die "not title for toplevel file '$filename' (env=$e)\n"
268                         if !defined($title);
269                     $fileinfo->{toplevel}->{$e}->{$filename} = 1;
270                 }
271             } elsif ($key eq 'title') {
272                 foreach my $e (@{$env_stack->[-1]}) {
273                     register_title($filename, $e, undef, $value);
274                 }
275             }
276         }
277
278         if ($line =~ m/^\[\[(.*)\]\]\s*$/) {
279             my $blockid = $1;
280             die "implement me" if $blockid =~m/,/;
281             my $reftext = '';
282             register_blockid($filename, $blockid, $reftext, $env_stack->[-1]);
283             foreach my $e (@{$env_stack->[-1]}) {
284                 $env_last_blockid->{$e} = $blockid;
285             }
286         }
287
288         if ($line =~ m/^\s*$/) {
289             foreach my $e (@{$env_stack->[-1]}) {
290                 delete $env_last_blockid->{$e};
291             }
292         }
293
294         # fixme: "anchor:"
295         # bibliography anchors
296         if ($line =~ m/\[\[\[([^\]]*)\]\]\]/) {
297             my $blockid = $1;
298             die "implement me" if $blockid =~m/,/;
299             register_blockid($filename, $blockid, "&#91;$blockid&#93;", $env_stack->[-1]);
300         }
301     }
302 }
303
304 my $scanned_files = {};
305 while (my $filename = shift) {
306     next if $filename !~ m/\.adoc$/; # skip attributes.txt
307     next if $scanned_files->{$filename};
308
309     scan_adoc_file($filename);
310     $scanned_files->{$filename} = 1;
311 }
312
313 sub resolve_link_target {
314     my ($env, $filename) = @_;
315
316     my $include_hash = $fileinfo->{include}->{$env};
317
318     my $repeat = 1;
319
320     while ($repeat) {
321         $repeat = 0;
322         foreach my $fn (keys %$include_hash) {
323             if ($include_hash->{$fn}->{$filename}) {
324                 next if ($fn eq 'pve-admin-guide.adoc') &&
325                     $fileinfo->{outfile}->{$env}->{$filename};
326                 $filename = $fn;
327                 $repeat = 1;
328                 last;
329             }
330         }
331     }
332
333     return $filename;
334 }
335
336 # now resolve blockids
337 foreach my $e (@$start_env) {
338     my $blockid_hash = $fileinfo->{blockid}->{$e};
339     foreach my $blockid (keys %$blockid_hash) {
340         my $fn = resolve_link_target($e, $blockid_hash->{$blockid});
341         if ($e eq 'wiki') {
342             my $title = $fileinfo->{titles}->{$e}->{$fn};
343             $title =~ s/\s/_/g;
344             die "found not title for '$fn' in env '$e'" if !$title;
345             $fileinfo->{blockid_target}->{$e}->{$blockid} = "link:/wiki/$title#$blockid";
346         } elsif ($e eq 'default') {
347             my $realfn = $fileinfo->{outfile}->{$e}->{$fn} ||
348                 die "no output file mapping for '$fn'\n";
349             $fileinfo->{blockid_target}->{$e}->{$blockid} = "link:/pve-docs/$realfn#$blockid";
350         } elsif ($e eq 'manvolnum') {
351             # we do not produce manpages for all content
352             # my $realfn = $fileinfo->{outfile}->{$e}->{$fn} ||
353             # warn "no output file mapping for '$fn'\n";
354             $fileinfo->{blockid_target}->{$e}->{$blockid} = $fn;
355         }
356     }
357 }
358
359
360 print to_json($fileinfo, { pretty => 1,  canonical => 1 } );