]> git.proxmox.com Git - pmg-api.git/blame - PMG/API2/Nodes.pm
node: journal: improve parameter schema
[pmg-api.git] / PMG / API2 / Nodes.pm
CommitLineData
1360e6f0
DM
1package PMG::API2::NodeInfo;
2
3use strict;
4use warnings;
d934c136 5use Time::Local qw(timegm_nocheck);
c1f5acda 6use Filesys::Df;
ba11e2d3 7use Data::Dumper;
d934c136 8
1360e6f0
DM
9use PVE::INotify;
10use PVE::RESTHandler;
11use PVE::JSONSchema qw(get_standard_option);
9d82c6bc 12use PMG::RESTEnvironment;
1360e6f0 13use PVE::SafeSyslog;
c1f5acda 14use PVE::ProcFSTools;
1360e6f0 15
c1f5acda 16use PMG::pmgcfg;
1360e6f0 17use PMG::Ticket;
3d8cdfee 18use PMG::Report;
87952a8d 19use PMG::API2::Subscription;
cc115a48 20use PMG::API2::APT;
fa04fcb4 21use PMG::API2::Tasks;
163fb58f 22use PMG::API2::Services;
2551682a 23use PMG::API2::Network;
d17c5265 24use PMG::API2::ClamAV;
b00d2d73 25use PMG::API2::SpamAssassin;
70805f7d 26use PMG::API2::Postfix;
202c952a 27use PMG::API2::MailTracker;
464f7494 28use PMG::API2::Backup;
1360e6f0
DM
29
30use base qw(PVE::RESTHandler);
31
70805f7d
DM
32__PACKAGE__->register_method ({
33 subclass => "PMG::API2::Postfix",
34 path => 'postfix',
35});
36
d17c5265
DM
37__PACKAGE__->register_method ({
38 subclass => "PMG::API2::ClamAV",
39 path => 'clamav',
40});
41
b00d2d73
DC
42__PACKAGE__->register_method ({
43 subclass => "PMG::API2::SpamAssassin",
44 path => 'spamassassin',
45});
46
2551682a
DM
47__PACKAGE__->register_method ({
48 subclass => "PMG::API2::Network",
49 path => 'network',
50});
51
fa04fcb4
DM
52__PACKAGE__->register_method ({
53 subclass => "PMG::API2::Tasks",
54 path => 'tasks',
55});
56
163fb58f
DM
57__PACKAGE__->register_method ({
58 subclass => "PMG::API2::Services",
59 path => 'services',
60});
61
87952a8d
DM
62__PACKAGE__->register_method ({
63 subclass => "PMG::API2::Subscription",
64 path => 'subscription',
65});
66
cc115a48
DM
67__PACKAGE__->register_method ({
68 subclass => "PMG::API2::APT",
69 path => 'apt',
70});
71
202c952a
DM
72__PACKAGE__->register_method ({
73 subclass => "PMG::API2::MailTracker",
74 path => 'tracker',
75});
76
464f7494
DM
77__PACKAGE__->register_method ({
78 subclass => "PMG::API2::Backup",
79 path => 'backup',
80});
81
1360e6f0
DM
82__PACKAGE__->register_method ({
83 name => 'index',
84 path => '',
85 method => 'GET',
86 permissions => { user => 'all' },
87 description => "Node index.",
88 parameters => {
89 additionalProperties => 0,
90 properties => {
91 node => get_standard_option('pve-node'),
92 },
93 },
94 returns => {
95 type => 'array',
96 items => {
97 type => "object",
98 properties => {},
99 },
100 links => [ { rel => 'child', href => "{name}" } ],
101 },
102 code => sub {
103 my ($param) = @_;
104
105 my $result = [
cc115a48 106 { name => 'apt' },
464f7494 107 { name => 'backup' },
d17c5265 108 { name => 'clamav' },
b00d2d73 109 { name => 'spamassassin' },
70805f7d 110 { name => 'postfix' },
163fb58f 111 { name => 'services' },
7e4d6e51 112 { name => 'syslog' },
8b4a5410 113 { name => 'journal' },
fa04fcb4 114 { name => 'tasks' },
202c952a 115 { name => 'tracker' },
d934c136 116 { name => 'time' },
3d8cdfee 117 { name => 'report' },
c1f5acda 118 { name => 'status' },
87952a8d 119 { name => 'subscription' },
6466cf6d 120 { name => 'termproxy' },
229e2e68 121 { name => 'rrddata' },
1360e6f0
DM
122 ];
123
124 return $result;
125 }});
126
3d8cdfee
DC
127__PACKAGE__->register_method({
128 name => 'report',
129 path => 'report',
130 method => 'GET',
131 protected => 1,
132 proxyto => 'node',
133 permissions => { check => [ 'admin', 'audit' ] },
134 description => "Gather various system information about a node",
135 parameters => {
136 additionalProperties => 0,
137 properties => {
138 node => get_standard_option('pve-node'),
139 },
140 },
141 returns => {
142 type => 'string',
143 },
144 code => sub {
145 return PMG::Report::generate();
146 }});
147
065b2986
DM
148__PACKAGE__->register_method({
149 name => 'rrddata',
150 path => 'rrddata',
151 method => 'GET',
152 protected => 1, # fixme: can we avoid that?
153 proxyto => 'node',
127154c3 154 permissions => { check => [ 'admin', 'audit' ] },
065b2986
DM
155 description => "Read node RRD statistics",
156 parameters => {
157 additionalProperties => 0,
158 properties => {
159 node => get_standard_option('pve-node'),
160 timeframe => {
161 description => "Specify the time frame you are interested in.",
162 type => 'string',
163 enum => [ 'hour', 'day', 'week', 'month', 'year' ],
164 },
165 cf => {
166 description => "The RRD consolidation function",
167 type => 'string',
168 enum => [ 'AVERAGE', 'MAX' ],
169 optional => 1,
170 },
171 },
172 },
173 returns => {
174 type => "array",
175 items => {
176 type => "object",
177 properties => {},
178 },
179 },
180 code => sub {
181 my ($param) = @_;
182
183 return PMG::Utils::create_rrd_data(
184 "pmg-node-v1.rrd", $param->{timeframe}, $param->{cf});
185 }});
186
187
7e4d6e51
DM
188__PACKAGE__->register_method({
189 name => 'syslog',
190 path => 'syslog',
191 method => 'GET',
192 description => "Read system log",
193 proxyto => 'node',
194 protected => 1,
da5777d7 195 permissions => { check => [ 'admin', 'audit' ] },
7e4d6e51
DM
196 parameters => {
197 additionalProperties => 0,
198 properties => {
199 node => get_standard_option('pve-node'),
200 start => {
201 type => 'integer',
202 minimum => 0,
203 optional => 1,
204 },
205 limit => {
206 type => 'integer',
207 minimum => 0,
208 optional => 1,
209 },
210 since => {
cf49e998 211 type => 'string',
7e4d6e51
DM
212 pattern => '^\d{4}-\d{2}-\d{2}( \d{2}:\d{2}(:\d{2})?)?$',
213 description => "Display all log since this date-time string.",
214 optional => 1,
215 },
216 'until' => {
cf49e998 217 type => 'string',
7e4d6e51
DM
218 pattern => '^\d{4}-\d{2}-\d{2}( \d{2}:\d{2}(:\d{2})?)?$',
219 description => "Display all log until this date-time string.",
220 optional => 1,
221 },
cf49e998
DM
222 service => {
223 description => "Service ID",
224 type => 'string',
225 maxLength => 128,
226 optional => 1,
227 },
7e4d6e51
DM
228 },
229 },
230 returns => {
231 type => 'array',
232 items => {
233 type => "object",
234 properties => {
235 n => {
236 description=> "Line number",
237 type=> 'integer',
238 },
239 t => {
240 description=> "Line text",
241 type => 'string',
242 }
243 }
244 }
245 },
246 code => sub {
247 my ($param) = @_;
248
f1c29260 249 my $restenv = PMG::RESTEnvironment->get();
7e4d6e51 250
cf49e998 251 my $service = $param->{service};
ef07e4d0
DM
252 $service = PMG::Utils::lookup_real_service_name($service)
253 if $service;
cf49e998
DM
254
255 my ($count, $lines) = PVE::Tools::dump_journal(
256 $param->{start}, $param->{limit},
257 $param->{since}, $param->{'until'}, $service);
7e4d6e51
DM
258
259 $restenv->set_result_attrib('total', $count);
260
261 return $lines;
262 }});
263
c2e3529a
DC
264__PACKAGE__->register_method({
265 name => 'journal',
266 path => 'journal',
267 method => 'GET',
268 description => "Read Journal",
269 proxyto => 'node',
270 permissions => { check => [ 'admin', 'audit' ] },
271 protected => 1,
272 parameters => {
273 additionalProperties => 0,
274 properties => {
275 node => get_standard_option('pve-node'),
276 since => {
13232555
TL
277 description => "Display all log since this UNIX epoch. Conflicts with 'startcursor'.",
278 type => 'integer',
279 minimum => 0,
c2e3529a
DC
280 optional => 1,
281 },
282 until => {
13232555
TL
283 description => "Display all log until this UNIX epoch. Conflicts with 'endcursor'.",
284 type => 'integer',
285 minimum => 0,
c2e3529a
DC
286 optional => 1,
287 },
288 lastentries => {
13232555 289 description => "Limit to the last X lines. Conflicts with a range.",
c2e3529a 290 type => 'integer',
13232555 291 minimum => 0,
c2e3529a
DC
292 optional => 1,
293 },
294 startcursor => {
13232555 295 description => "Start after the given Cursor. Conflicts with 'since'.",
c2e3529a
DC
296 type => 'string',
297 optional => 1,
298 },
299 endcursor => {
13232555 300 description => "End before the given Cursor. Conflicts with 'until'.",
c2e3529a
DC
301 type => 'string',
302 optional => 1,
303 },
304 },
305 },
306 returns => {
307 type => 'array',
308 items => {
309 type => "string",
310 }
311 },
312 code => sub {
313 my ($param) = @_;
314
315 my $lines = [];
316
317 my $parser = sub {
318 push @$lines, shift;
319 };
320
321 my $cmd = ["/usr/bin/mini-journalreader"];
322 push @$cmd, '-n', $param->{lastentries} if $param->{lastentries};
323 push @$cmd, '-b', $param->{since} if $param->{since};
324 push @$cmd, '-e', $param->{until} if $param->{until};
325 push @$cmd, '-f', $param->{startcursor} if $param->{startcursor};
326 push @$cmd, '-t', $param->{endcursor} if $param->{endcursor};
327
328 PVE::Tools::run_command($cmd, outfunc => $parser);
329
330 return $lines;
331 }});
332
6466cf6d 333
1360e6f0 334__PACKAGE__->register_method ({
6466cf6d
DC
335 name => 'termproxy',
336 path => 'termproxy',
1360e6f0 337 method => 'POST',
aafcc40e 338 permissions => { check => [ 'admin' ] },
fa04fcb4 339 protected => 1,
6466cf6d 340 description => "Creates a Terminal proxy.",
1360e6f0
DM
341 parameters => {
342 additionalProperties => 0,
343 properties => {
344 node => get_standard_option('pve-node'),
5e6e07df
DM
345 upgrade => {
346 type => 'boolean',
347 description => "Run 'apt-get dist-upgrade' instead of normal shell.",
348 optional => 1,
349 default => 0,
350 },
1360e6f0
DM
351 },
352 },
353 returns => {
354 additionalProperties => 0,
355 properties => {
356 user => { type => 'string' },
357 ticket => { type => 'string' },
358 port => { type => 'integer' },
359 upid => { type => 'string' },
360 },
361 },
362 code => sub {
363 my ($param) = @_;
364
365 my $node = $param->{node};
366
5e6e07df 367 if ($node ne PVE::INotify::nodename()) {
6466cf6d 368 die "termproxy to remote node not implemented";
5e6e07df
DM
369 }
370
1360e6f0
DM
371 my $authpath = "/nodes/$node";
372
9d82c6bc 373 my $restenv = PMG::RESTEnvironment->get();
1360e6f0
DM
374 my $user = $restenv->get_user();
375
5e6e07df
DM
376 raise_perm_exc('user != root@pam') if $param->{upgrade} && $user ne 'root@pam';
377
1360e6f0
DM
378 my $ticket = PMG::Ticket::assemble_vnc_ticket($user, $authpath);
379
380 my $family = PVE::Tools::get_host_address_family($node);
381 my $port = PVE::Tools::next_vnc_port($family);
382
8e6893c6
DM
383 my $shcmd;
384
385 if ($user eq 'root@pam') {
5e6e07df
DM
386 if ($param->{upgrade}) {
387 my $upgradecmd = "pmgupgrade --shell";
388 # $upgradecmd = PVE::Tools::shellquote($upgradecmd) if $remip;
389 $shcmd = [ '/bin/bash', '-c', $upgradecmd ];
390 } else {
391 $shcmd = [ '/bin/login', '-f', 'root' ];
392 }
8e6893c6
DM
393 } else {
394 $shcmd = [ '/bin/login' ];
395 }
396
6466cf6d
DC
397 my $cmd = ['/usr/bin/termproxy', $port, '--path', $authpath,
398 '--', @$shcmd];
1360e6f0
DM
399
400 my $realcmd = sub {
401 my $upid = shift;
402
6466cf6d 403 syslog ('info', "starting termproxy $upid\n");
1360e6f0
DM
404
405 my $cmdstr = join (' ', @$cmd);
406 syslog ('info', "launch command: $cmdstr");
407
6466cf6d 408 PVE::Tools::run_command($cmd);
1360e6f0
DM
409
410 return;
411 };
412
6466cf6d 413 my $upid = $restenv->fork_worker('termproxy', "", $user, $realcmd);
1360e6f0
DM
414
415 PVE::Tools::wait_for_vnc_port($port);
416
417 return {
418 user => $user,
419 ticket => $ticket,
420 port => $port,
421 upid => $upid,
422 };
423 }});
424
425__PACKAGE__->register_method({
426 name => 'vncwebsocket',
427 path => 'vncwebsocket',
428 method => 'GET',
aafcc40e 429 permissions => { check => [ 'admin' ] },
1360e6f0
DM
430 description => "Opens a weksocket for VNC traffic.",
431 parameters => {
432 additionalProperties => 0,
433 properties => {
434 node => get_standard_option('pve-node'),
435 vncticket => {
436 description => "Ticket from previous call to vncproxy.",
437 type => 'string',
438 maxLength => 512,
439 },
440 port => {
441 description => "Port number returned by previous vncproxy call.",
442 type => 'integer',
443 minimum => 5900,
444 maximum => 5999,
445 },
446 },
447 },
448 returns => {
449 type => "object",
450 properties => {
451 port => { type => 'string' },
452 },
453 },
454 code => sub {
455 my ($param) = @_;
456
457 my $authpath = "/nodes/$param->{node}";
458
9d82c6bc 459 my $restenv = PMG::RESTEnvironment->get();
1360e6f0
DM
460 my $user = $restenv->get_user();
461
462 PMG::Ticket::verify_vnc_ticket($param->{vncticket}, $user, $authpath);
463
464 my $port = $param->{port};
465
466 return { port => $port };
467 }});
468
bce9f371
DM
469__PACKAGE__->register_method({
470 name => 'dns',
471 path => 'dns',
472 method => 'GET',
473 description => "Read DNS settings.",
474 proxyto => 'node',
aea3488a 475 permissions => { check => [ 'admin', 'audit' ] },
bce9f371
DM
476 parameters => {
477 additionalProperties => 0,
478 properties => {
479 node => get_standard_option('pve-node'),
480 },
481 },
482 returns => {
483 type => "object",
484 additionalProperties => 0,
485 properties => {
486 search => {
487 description => "Search domain for host-name lookup.",
488 type => 'string',
489 optional => 1,
490 },
491 dns1 => {
492 description => 'First name server IP address.',
493 type => 'string',
494 optional => 1,
495 },
496 dns2 => {
497 description => 'Second name server IP address.',
498 type => 'string',
499 optional => 1,
500 },
501 dns3 => {
502 description => 'Third name server IP address.',
503 type => 'string',
504 optional => 1,
505 },
506 },
507 },
508 code => sub {
509 my ($param) = @_;
510
511 my $res = PVE::INotify::read_file('resolvconf');
512
513 return $res;
514 }});
515
516__PACKAGE__->register_method({
517 name => 'update_dns',
518 path => 'dns',
519 method => 'PUT',
520 description => "Write DNS settings.",
521 proxyto => 'node',
522 protected => 1,
523 parameters => {
524 additionalProperties => 0,
525 properties => {
526 node => get_standard_option('pve-node'),
527 search => {
528 description => "Search domain for host-name lookup.",
529 type => 'string',
530 },
531 dns1 => {
532 description => 'First name server IP address.',
533 type => 'string', format => 'ip',
534 optional => 1,
535 },
536 dns2 => {
537 description => 'Second name server IP address.',
538 type => 'string', format => 'ip',
539 optional => 1,
540 },
541 dns3 => {
542 description => 'Third name server IP address.',
543 type => 'string', format => 'ip',
544 optional => 1,
545 },
546 },
547 },
548 returns => { type => "null" },
549 code => sub {
550 my ($param) = @_;
551
552 PVE::INotify::update_file('resolvconf', $param);
553
554 return undef;
555 }});
556
557
d934c136
DM
558__PACKAGE__->register_method({
559 name => 'time',
560 path => 'time',
561 method => 'GET',
562 description => "Read server time and time zone settings.",
563 proxyto => 'node',
aea3488a 564 permissions => { check => [ 'admin', 'audit' ] },
d934c136
DM
565 parameters => {
566 additionalProperties => 0,
567 properties => {
568 node => get_standard_option('pve-node'),
569 },
570 },
571 returns => {
572 type => "object",
573 additionalProperties => 0,
574 properties => {
575 timezone => {
576 description => "Time zone",
577 type => 'string',
578 },
579 time => {
580 description => "Seconds since 1970-01-01 00:00:00 UTC.",
581 type => 'integer',
582 minimum => 1297163644,
583 },
584 localtime => {
585 description => "Seconds since 1970-01-01 00:00:00 (local time)",
586 type => 'integer',
587 minimum => 1297163644,
588 },
589 },
590 },
591 code => sub {
592 my ($param) = @_;
593
594 my $ctime = time();
595 my $ltime = timegm_nocheck(localtime($ctime));
596 my $res = {
597 timezone => PVE::INotify::read_file('timezone'),
598 time => time(),
599 localtime => $ltime,
600 };
601
602 return $res;
603 }});
604
605__PACKAGE__->register_method({
606 name => 'set_timezone',
607 path => 'time',
608 method => 'PUT',
609 description => "Set time zone.",
610 proxyto => 'node',
611 protected => 1,
612 parameters => {
613 additionalProperties => 0,
614 properties => {
615 node => get_standard_option('pve-node'),
616 timezone => {
617 description => "Time zone. The file '/usr/share/zoneinfo/zone.tab' contains the list of valid names.",
618 type => 'string',
619 },
620 },
621 },
622 returns => { type => "null" },
623 code => sub {
624 my ($param) = @_;
625
626 PVE::INotify::write_file('timezone', $param->{timezone});
627
628 return undef;
629 }});
630
c1f5acda
DM
631__PACKAGE__->register_method({
632 name => 'status',
633 path => 'status',
634 method => 'GET',
635 description => "Read server status. This is used by the cluster manager to test the node health.",
636 proxyto => 'node',
e4842eec 637 permissions => { check => [ 'admin', 'qmanager', 'audit' ] },
c1f5acda
DM
638 protected => 1,
639 parameters => {
640 additionalProperties => 0,
641 properties => {
642 node => get_standard_option('pve-node'),
643 },
644 },
645 returns => {
646 type => "object",
647 additionalProperties => 1,
648 properties => {
649 time => {
650 description => "Seconds since 1970-01-01 00:00:00 UTC.",
651 type => 'integer',
652 minimum => 1297163644,
653 },
654 uptime => {
655 description => "The uptime of the system in seconds.",
656 type => 'integer',
657 minimum => 0,
658 },
659 insync => {
660 description => "Database is synced with other nodes.",
661 type => 'boolean',
662 },
663 },
664 },
665 code => sub {
666 my ($param) = @_;
667
668 my $restenv = PMG::RESTEnvironment->get();
669 my $cinfo = $restenv->{cinfo};
670
671 my $ctime = time();
672
673 my $res = { time => $ctime, insync => 1 };
674
675 my $si = PMG::DBTools::cluster_sync_status($cinfo);
676 foreach my $cid (keys %$si) {
677 my $lastsync = $si->{$cid};
678 my $sdiff = $ctime - $lastsync;
679 $sdiff = 0 if $sdiff < 0;
680 $res->{insync} = 0 if $sdiff > (60*3);
681 }
682
683 my ($uptime, $idle) = PVE::ProcFSTools::read_proc_uptime();
684 $res->{uptime} = $uptime;
685
686 my ($avg1, $avg5, $avg15) = PVE::ProcFSTools::read_loadavg();
687 $res->{loadavg} = [ $avg1, $avg5, $avg15];
688
689 my ($sysname, $nodename, $release, $version, $machine) = POSIX::uname();
690
691 $res->{kversion} = "$sysname $release $version";
692
693 $res->{cpuinfo} = PVE::ProcFSTools::read_cpuinfo();
694
695 my $stat = PVE::ProcFSTools::read_proc_stat();
696 $res->{cpu} = $stat->{cpu};
697 $res->{wait} = $stat->{wait};
698
699 my $meminfo = PVE::ProcFSTools::read_meminfo();
700 $res->{memory} = {
701 free => $meminfo->{memfree},
702 total => $meminfo->{memtotal},
703 used => $meminfo->{memused},
704 };
705
706 $res->{swap} = {
707 free => $meminfo->{swapfree},
708 total => $meminfo->{swaptotal},
709 used => $meminfo->{swapused},
710 };
711
712 $res->{pmgversion} = PMG::pmgcfg::package() . "/" .
713 PMG::pmgcfg::version_text();
714
715 my $dinfo = df('/', 1); # output is bytes
716
717 $res->{rootfs} = {
718 total => $dinfo->{blocks},
719 avail => $dinfo->{bavail},
720 used => $dinfo->{used},
d9f4add4 721 free => $dinfo->{blocks} - $dinfo->{used},
c1f5acda
DM
722 };
723
b4df3454
DM
724 if (my $subinfo = PVE::INotify::read_file('subscription')) {
725 if (my $level = $subinfo->{level}) {
726 $res->{level} = $level;
727 }
728 }
729
c1f5acda
DM
730 return $res;
731 }});
732
0a327ff4
DM
733__PACKAGE__->register_method({
734 name => 'node_cmd',
735 path => 'status',
736 method => 'POST',
aafcc40e 737 permissions => { check => [ 'admin' ] },
0a327ff4
DM
738 protected => 1,
739 description => "Reboot or shutdown a node.",
740 proxyto => 'node',
741 parameters => {
742 additionalProperties => 0,
743 properties => {
744 node => get_standard_option('pve-node'),
745 command => {
746 description => "Specify the command.",
747 type => 'string',
748 enum => [qw(reboot shutdown)],
749 },
750 },
751 },
752 returns => { type => "null" },
753 code => sub {
754 my ($param) = @_;
755
756 if ($param->{command} eq 'reboot') {
757 system ("(sleep 2;/sbin/reboot)&");
758 } elsif ($param->{command} eq 'shutdown') {
759 system ("(sleep 2;/sbin/poweroff)&");
760 }
761
762 return undef;
763 }});
1360e6f0
DM
764
765package PMG::API2::Nodes;
766
767use strict;
768use warnings;
769
770use PVE::RESTHandler;
771use PVE::JSONSchema qw(get_standard_option);
772
be512f56
DM
773use PMG::RESTEnvironment;
774
1360e6f0
DM
775use base qw(PVE::RESTHandler);
776
777__PACKAGE__->register_method ({
f6dcba61 778 subclass => "PMG::API2::NodeInfo",
1360e6f0
DM
779 path => '{node}',
780});
781
782__PACKAGE__->register_method ({
783 name => 'index',
784 path => '',
785 method => 'GET',
786 permissions => { user => 'all' },
787 description => "Cluster node index.",
788 parameters => {
789 additionalProperties => 0,
790 properties => {},
791 },
792 returns => {
793 type => 'array',
794 items => {
795 type => "object",
796 properties => {},
797 },
798 links => [ { rel => 'child', href => "{node}" } ],
799 },
800 code => sub {
801 my ($param) = @_;
802
803 my $nodename = PVE::INotify::nodename();
be512f56
DM
804
805 my $res = [ { node => $nodename } ];
806
807 my $done = {};
808
809 $done->{$nodename} = 1;
810
811 my $restenv = PMG::RESTEnvironment->get();
812 my $cinfo = $restenv->{cinfo};
813
814 foreach my $ni (values %{$cinfo->{ids}}) {
815 push @$res, { node => $ni->{name} } if !$done->{$ni->{name}};
816 $done->{$ni->{name}} = 1;
817 }
1360e6f0
DM
818
819 return $res;
820 }});
821
822
8231;