]>
Commit | Line | Data |
---|---|---|
0bc3e510 WL |
1 | #!/usr/bin/perl |
2 | ||
3 | my $PROGNAME = "pve-zsync"; | |
4 | my $CONFIG_PATH = '/var/lib/'.$PROGNAME.'/'; | |
5 | my $CONFIG = "$PROGNAME.cfg"; | |
6 | my $CRONJOBS = '/etc/cron.d/'.$PROGNAME; | |
7 | my $VMCONFIG = '/var/lib/'.$PROGNAME.'/'; | |
8 | my $PATH = "/usr/sbin/"; | |
9 | my $QEMU_CONF = '/etc/pve/local/qemu-server/'; | |
10 | my $DEBUG = 0; | |
11 | ||
12 | use strict; | |
13 | use warnings; | |
14 | use Data::Dumper qw(Dumper); | |
15 | use Fcntl qw(:flock SEEK_END); | |
16 | use Getopt::Long; | |
17 | use Switch; | |
18 | ||
19 | check_bin ('cstream'); | |
20 | check_bin ('zfs'); | |
21 | check_bin ('ssh'); | |
22 | check_bin ('scp'); | |
23 | ||
24 | sub check_bin { | |
25 | my ($bin) = @_; | |
26 | ||
27 | foreach my $p (split (/:/, $ENV{PATH})) { | |
28 | my $fn = "$p/$bin"; | |
29 | if (-x $fn) { | |
30 | return $fn; | |
31 | } | |
32 | } | |
33 | ||
34 | warn "unable to find command '$bin'\n"; | |
35 | } | |
36 | ||
37 | sub cut_to_width { | |
38 | my ($text, $max) = @_; | |
39 | ||
40 | return $text if (length($text) <= $max); | |
41 | my @spl = split('/', $text); | |
42 | ||
43 | my $count = length($spl[@spl-1]); | |
44 | return "..\/".substr($spl[@spl-1],($count-$max)+3 ,$count) if $count > $max; | |
45 | ||
46 | $count += length($spl[0]) if @spl > 1; | |
47 | return substr($spl[0], 0, $max-4-length($spl[@spl-1]))."\/..\/".$spl[@spl-1] if $count > $max; | |
48 | ||
49 | my $rest = 1 ; | |
50 | $rest = $max-$count if ($max-$count > 0); | |
51 | ||
52 | return "$spl[0]".substr($text, length($spl[0]), $rest)."..\/".$spl[@spl-1]; | |
53 | } | |
54 | ||
55 | sub lock { | |
56 | my ($fh) = @_; | |
57 | flock($fh, LOCK_EX) or die "Cannot lock config - $!\n"; | |
58 | ||
59 | seek($fh, 0, SEEK_END) or die "Cannot seek - $!\n"; | |
60 | } | |
61 | ||
62 | sub unlock { | |
63 | my ($fh) = @_; | |
64 | flock($fh, LOCK_UN) or die "Cannot unlock config- $!\n"; | |
65 | } | |
66 | ||
67 | sub check_config { | |
68 | my ($source, $name, $cfg) = @_; | |
69 | ||
70 | if ($source->{vmid} && $cfg->{$source->{vmid}}->{$name}->{locked}){ | |
71 | return "active" if $cfg->{$source->{vmid}}->{$name}->{locked} eq 'yes'; | |
72 | return "exist" if $cfg->{$source->{vmid}}->{$name}->{locked} eq 'no'; | |
73 | } elsif ($cfg->{$source->{abs_path}}->{$name}->{locked}) { | |
74 | return "active" if $cfg->{$source->{abs_path}}->{$name}->{locked} eq 'yes'; | |
75 | return "exist" if $cfg->{$source->{abs_path}}->{$name}->{locked} eq 'no'; | |
76 | } | |
77 | ||
78 | return undef; | |
79 | } | |
80 | ||
81 | ||
82 | sub check_pool_exsits { | |
83 | my ($pool, $ip) = @_; | |
84 | ||
85 | my $cmd = ''; | |
86 | $cmd = "ssh root\@$ip " if $ip; | |
87 | $cmd .= "zfs list $pool -H"; | |
88 | eval { | |
89 | run_cmd($cmd); | |
90 | }; | |
91 | ||
92 | if($@){ | |
93 | return 1; | |
94 | } | |
95 | return undef; | |
96 | } | |
97 | ||
98 | sub write_to_config { | |
99 | my ($cfg) = @_; | |
100 | ||
101 | open(my $fh, ">", "$CONFIG_PATH$CONFIG") | |
102 | or die "cannot open >$CONFIG_PATH$CONFIG: $!\n"; | |
103 | ||
104 | my $text = decode_config($cfg); | |
105 | ||
106 | print($fh $text); | |
107 | ||
108 | close($fh); | |
109 | } | |
110 | ||
111 | sub read_from_config { | |
112 | ||
113 | unless(-e "$CONFIG_PATH$CONFIG") { | |
114 | return undef; | |
115 | } | |
116 | ||
117 | open(my $fh, "<", "$CONFIG_PATH$CONFIG") | |
118 | or die "cannot open > $CONFIG_PATH$CONFIG: $!\n"; | |
119 | ||
120 | $/ = undef; | |
121 | ||
122 | my $text = <$fh>; | |
123 | ||
124 | unlock($fh); | |
125 | ||
126 | my $cfg = encode_config($text); | |
127 | ||
128 | return $cfg; | |
129 | } | |
130 | ||
131 | sub decode_config { | |
132 | my ($cfg) = @_; | |
133 | my $raw = ''; | |
134 | foreach my $source (keys%{$cfg}){ | |
135 | foreach my $sync_name (keys%{$cfg->{$source}}){ | |
136 | $raw .= "$source: $sync_name\n"; | |
137 | foreach my $parameter (keys%{$cfg->{$source}->{$sync_name}}){ | |
138 | $raw .= "\t$parameter: $cfg->{$source}->{$sync_name}->{$parameter}\n"; | |
139 | } | |
140 | } | |
141 | } | |
142 | return $raw; | |
143 | } | |
144 | ||
145 | sub encode_config { | |
146 | my ($raw) = @_; | |
147 | my $cfg = {}; | |
148 | my $source; | |
149 | my $check = 0; | |
150 | my $sync_name; | |
151 | ||
152 | while ($raw && $raw =~ s/^(.*?)(\n|$)//) { | |
153 | my $line = $1; | |
154 | ||
155 | next if $line =~ m/^\#/; | |
156 | next if $line =~ m/^\s*$/; | |
157 | ||
158 | if ($line =~ m/^(\t| )(\w+): (.+)/){ | |
159 | my $par = $2; | |
160 | my $value = $3; | |
161 | ||
162 | if ($par eq 'source_pool') { | |
163 | $cfg->{$source}->{$sync_name}->{$par} = $value; | |
164 | die "error in Config: SourcePool value doubled\n" if ($check & 1); | |
165 | $check += 1; | |
166 | } elsif ($par eq 'source_ip') { | |
167 | $cfg->{$source}->{$sync_name}->{$par} = $value; | |
168 | die "error in Config: SourceIP value doubled\n" if ($check & 2); | |
169 | $check += 2; | |
170 | } elsif ($par eq 'locked') { | |
171 | $cfg->{$source}->{$sync_name}->{$par} = $value; | |
172 | die "error in Config: Locked value doubled\n" if ($check & 4); | |
173 | $check += 4; | |
174 | } elsif ($par eq 'method') { | |
175 | $cfg -> {$source}->{$sync_name}->{$par} = $value; | |
176 | die "error in Config: Method value doubled\n" if ($check & 8); | |
177 | $check += 8; | |
178 | } elsif ($par eq 'interval') { | |
179 | $cfg -> {$source}->{$sync_name}->{$par} = $value; | |
180 | die "error in Config: Iterval value doubled\n" if ($check & 16); | |
181 | $check += 16; | |
182 | } elsif ($par eq 'limit') { | |
183 | $cfg -> {$source}->{$sync_name}->{$par} = $value; | |
184 | die "error in Config: Limit value doubled\n" if ($check & 32); | |
185 | $check += 32; | |
186 | } elsif ($par eq 'dest_pool') { | |
187 | $cfg -> {$source}->{$sync_name}->{$par} = $value; | |
188 | die "error in Config: DestPool value doubled\n" if ($check & 64); | |
189 | $check += 64; | |
190 | } elsif ($par eq 'dest_ip') { | |
191 | $cfg -> {$source}->{$sync_name}->{$par} = $value; | |
192 | die "error in Config: DestIp value doubled\n" if ($check & 128); | |
193 | $check += 128; | |
194 | } elsif ($par eq 'dest_path') { | |
195 | $cfg -> {$source}->{$sync_name}->{$par} = $value; | |
196 | die "error in Config: DestPath value doubled\n" if ($check & 256); | |
197 | $check += 256; | |
198 | } elsif ($par eq 'source_path') { | |
199 | $cfg -> {$source}->{$sync_name}->{$par} = $value; | |
200 | die "error in Config: SourcePath value doubled\n" if ($check & 512); | |
201 | $check += 512; | |
202 | } elsif ($par eq 'vmid') { | |
203 | $cfg -> {$source}->{$sync_name}->{$par} = $value; | |
204 | die "error in Config: Vmid value doubled\n" if ($check & 1024); | |
205 | $check += 1024; | |
206 | } elsif ($par =~ 'lsync') { | |
207 | $cfg->{$source}->{$sync_name}->{$par} = $value; | |
208 | die "error in Config: lsync value doubled\n" if ($check & 2048); | |
209 | $check += 2048; | |
210 | } elsif ($par =~ 'maxsnap') { | |
211 | $cfg->{$source}->{$sync_name}->{$par} = $value; | |
212 | die "error in Config: maxsnap value doubled\n" if ($check & 4096); | |
213 | $check += 4096; | |
214 | } else { | |
215 | die "error in Config\n"; | |
216 | } | |
217 | } elsif ($line =~ m/^((\d+.\d+.\d+.\d+):)?([\w\-\_\/]+): (.+){0,1}/){ | |
218 | $source = $3; | |
219 | $sync_name = $4 ? $4 : 'default' ; | |
220 | $cfg->{$source}->{$sync_name} = undef; | |
221 | $cfg->{$source}->{$sync_name}->{source_ip} = $2 if $2; | |
222 | $check = 0; | |
223 | } | |
224 | } | |
225 | return $cfg; | |
226 | } | |
227 | ||
228 | sub parse_target { | |
229 | my ($text) = @_; | |
230 | ||
231 | if ($text =~ m/^((\d+.\d+.\d+.\d+):)?((\w+)\/?)([\w\/\-\_]*)?$/) { | |
232 | ||
233 | die "Input not valid\n" if !$3; | |
234 | my $tmp = $3; | |
235 | my $target = {}; | |
236 | ||
237 | if ($2) { | |
238 | $target->{ip} = $2 ; | |
239 | } | |
240 | ||
241 | if ($tmp =~ m/^(\d\d\d+)$/){ | |
242 | $target->{vmid} = $tmp; | |
243 | } else { | |
244 | $target->{pool} = $4; | |
245 | my $abs_path = $4; | |
246 | if ($5) { | |
247 | $target->{path} = "\/$5"; | |
248 | $abs_path .= "\/$5"; | |
249 | } | |
250 | $target->{abs_path} = $abs_path; | |
251 | } | |
252 | ||
253 | return $target; | |
254 | } | |
255 | die "Input not valid\n"; | |
256 | } | |
257 | ||
258 | sub list { | |
259 | ||
260 | my $cfg = read_from_config("$CONFIG_PATH$CONFIG"); | |
261 | ||
262 | my $list = sprintf("%-25s%-15s%-7s%-20s%-10s%-5s\n" , "SOURCE", "NAME", "ACTIVE", "LAST SYNC", "INTERVAL", "TYPE"); | |
263 | ||
264 | foreach my $source (keys%{$cfg}){ | |
265 | foreach my $sync_name (keys%{$cfg->{$source}}){ | |
266 | my $source_name = $source; | |
267 | $source_name = $cfg->{$source}->{$sync_name}->{source_ip}.":".$source if $cfg->{$source}->{$sync_name}->{source_ip}; | |
268 | $list .= sprintf("%-25s%-15s", cut_to_width($source_name,25), cut_to_width($sync_name,15)); | |
269 | $list .= sprintf("%-7s",$cfg->{$source}->{$sync_name}->{locked}); | |
270 | $list .= sprintf("%-20s",$cfg->{$source}->{$sync_name}->{lsync}); | |
271 | $list .= sprintf("%-10s",$cfg->{$source}->{$sync_name}->{interval}); | |
272 | $list .= sprintf("%-5s\n",$cfg->{$source}->{$sync_name}->{method}); | |
273 | } | |
274 | } | |
275 | ||
276 | return $list; | |
277 | } | |
278 | ||
279 | sub vm_exists { | |
280 | my ($target) = @_; | |
281 | ||
282 | my $cmd = ""; | |
283 | $cmd = "ssh root\@$target->{ip} " if ($target->{ip}); | |
284 | $cmd .= "qm status $target->{vmid}"; | |
285 | ||
286 | my $res = run_cmd($cmd); | |
287 | ||
288 | return 1 if ($res =~ m/^status.*$/); | |
289 | return undef; | |
290 | } | |
291 | ||
292 | sub init { | |
293 | my ($param) = @_; | |
294 | ||
295 | my $cfg = read_from_config; | |
296 | ||
297 | my $vm = {}; | |
298 | ||
299 | my $name = $param->{name} ? $param->{name} : "default"; | |
300 | my $interval = $param->{interval} ? $param->{interval} : 15; | |
301 | ||
302 | my $source = parse_target($param->{source}); | |
303 | my $dest = parse_target($param->{dest}); | |
304 | ||
305 | $vm->{$name}->{dest_pool} = $dest->{pool}; | |
306 | $vm->{$name}->{dest_ip} = $dest->{ip} if $dest->{ip}; | |
307 | $vm->{$name}->{dest_path} = $dest->{path} if $dest->{path}; | |
308 | ||
309 | $param->{method} = "local" if !$dest->{ip} && !$source->{ip}; | |
310 | $vm->{$name}->{locked} = "no"; | |
311 | $vm->{$name}->{interval} = $interval; | |
312 | $vm->{$name}->{method} = $param->{method} ? $param->{method} : "ssh"; | |
313 | $vm->{$name}->{limit} = $param->{limit} if $param->{limit}; | |
314 | $vm->{$name}->{maxsnap} = $param->{maxsnap} if $param->{maxsnap}; | |
315 | ||
316 | if ( my $ip = $vm->{$name}->{dest_ip} ) { | |
317 | run_cmd("ssh-copy-id -i /root/.ssh/id_rsa.pub root\@$ip"); | |
318 | } | |
319 | ||
320 | if ( my $ip = $source->{ip} ) { | |
321 | run_cmd("ssh-copy-id -i /root/.ssh/id_rsa.pub root\@$ip"); | |
322 | } | |
323 | ||
324 | die "Pool $dest->{abs_path} does not exists\n" if check_pool_exsits($dest->{abs_path}, $dest->{ip}); | |
325 | ||
326 | my $check = check_pool_exsits($source->{abs_path}, $source->{ip}) if !$source->{vmid} && $source->{abs_path}; | |
327 | ||
328 | die "Pool $source->{abs_path} does not exists\n" if undef($check); | |
329 | ||
330 | my $add_job = sub { | |
331 | my ($vm, $name) = @_; | |
332 | my $source = ""; | |
333 | ||
334 | if ($vm->{$name}->{vmid}) { | |
335 | $source = "$vm->{$name}->{source_ip}:" if $vm->{$name}->{source_ip}; | |
336 | $source .= $vm->{$name}->{vmid}; | |
337 | } else { | |
338 | $source = $vm->{$name}->{source_pool}; | |
339 | $source .= $vm->{$name}->{source_path} if $vm->{$name}->{source_path}; | |
340 | } | |
341 | die "Config already exists\n" if $cfg->{$source}->{$name}; | |
342 | ||
343 | cron_add($vm); | |
344 | ||
345 | $cfg->{$source}->{$name} = $vm->{$name}; | |
346 | ||
347 | write_to_config($cfg); | |
348 | }; | |
349 | ||
350 | if ($source->{vmid}) { | |
351 | die "VM $source->{vmid} doesn't exist\n" if !vm_exists($source); | |
352 | my $disks = get_disks($source); | |
353 | $vm->{$name}->{vmid} = $source->{vmid}; | |
354 | $vm->{$name}->{lsync} = 0; | |
355 | $vm->{$name}->{source_ip} = $source->{ip} if $source->{ip}; | |
356 | ||
357 | &$add_job($vm, $name); | |
358 | ||
359 | } else { | |
360 | $vm->{$name}->{source_pool} = $source->{pool}; | |
361 | $vm->{$name}->{source_ip} = $source->{ip} if $source->{ip}; | |
362 | $vm->{$name}->{source_path} = $source->{path} if $source->{path}; | |
363 | $vm->{$name}->{lsync} = 0; | |
364 | ||
365 | &$add_job($vm, $name); | |
366 | } | |
367 | ||
368 | sync($param) if !$param->{skip}; | |
369 | } | |
370 | ||
371 | sub destroy { | |
372 | my ($param) = @_; | |
373 | ||
374 | my $cfg = read_from_config("$CONFIG_PATH$CONFIG"); | |
375 | my $name = $param->{name} ? $param->{name} : "default"; | |
376 | ||
377 | my $source = parse_target($param->{source}); | |
378 | ||
379 | my $delete_cron = sub { | |
380 | my ($path, $name, $cfg) = @_; | |
381 | ||
382 | die "Source does not exist!\n" unless $cfg->{$path} ; | |
383 | ||
384 | die "Sync Name does not exist!\n" unless $cfg->{$path}->{$name}; | |
385 | ||
386 | delete $cfg->{$path}->{$name}; | |
387 | ||
388 | delete $cfg->{$path} if keys%{$cfg->{$path}} == 0; | |
389 | ||
390 | write_to_config($cfg); | |
391 | ||
392 | cron_del($path, $name); | |
393 | }; | |
394 | ||
395 | ||
396 | if ($source->{vmid}) { | |
397 | my $path = $source->{vmid}; | |
398 | ||
399 | &$delete_cron($path, $name, $cfg) | |
400 | ||
401 | } else { | |
402 | ||
403 | my $path = $source->{pool}; | |
404 | $path .= $source->{path} if $source->{path}; | |
405 | ||
406 | &$delete_cron($path, $name, $cfg); | |
407 | } | |
408 | } | |
409 | ||
410 | sub sync { | |
411 | my ($param) = @_; | |
412 | ||
413 | my $cfg = read_from_config("$CONFIG_PATH$CONFIG"); | |
414 | ||
415 | my $name = $param->{name} ? $param->{name} : "default"; | |
416 | my $max_snap = $param->{maxsnap} ? $param->{maxsnap} : 1; | |
417 | my $method = $param->{method} ? $param->{method} : "ssh"; | |
418 | ||
419 | my $dest = parse_target($param->{dest}); | |
420 | my $source = parse_target($param->{source}); | |
421 | ||
422 | my $sync_path = sub { | |
423 | my ($source, $name, $cfg, $max_snap, $dest, $method) = @_; | |
424 | ||
425 | ($source->{old_snap},$source->{last_snap}) = snapshot_get($source, $dest, $max_snap, $name); | |
426 | ||
427 | my $job_status = check_config($source, $name, $cfg) if $cfg; | |
428 | die "VM syncing at the moment!\n" if ($job_status && $job_status eq "active"); | |
429 | ||
430 | if ($job_status && $job_status eq "exist") { | |
431 | my $conf_name = $source->{abs_path}; | |
432 | $conf_name = $source->{vmid} if $source->{vmid}; | |
433 | $cfg->{$conf_name}->{$name}->{locked} = "yes"; | |
434 | write_to_config($cfg); | |
435 | } | |
436 | ||
437 | my $date = snapshot_add($source, $dest, $name); | |
438 | ||
439 | send_image($source, $dest, $method, $param->{verbose}, $param->{limit}); | |
440 | ||
441 | snapshot_destroy($source, $dest, $method, $source->{old_snap}) if ($source->{destroy} && $source->{old_snap}); | |
442 | ||
443 | if ($job_status && $job_status eq "exist") { | |
444 | my $conf_name = $source->{abs_path}; | |
445 | $conf_name = $source->{vmid} if $source->{vmid}; | |
446 | $cfg->{$conf_name}->{$name}->{locked} = "no"; | |
447 | $cfg->{$conf_name}->{$name}->{lsync} = $date; | |
448 | write_to_config($cfg); | |
449 | } | |
450 | }; | |
451 | ||
452 | $param->{method} = "ssh" if !$param->{method}; | |
453 | ||
454 | if ($source->{vmid}) { | |
455 | die "VM $source->{vmid} doesn't exist\n" if !vm_exists($source); | |
456 | my $disks = get_disks($source); | |
457 | ||
458 | foreach my $disk (keys %{$disks}) { | |
459 | $source->{abs_path} = $disks->{$disk}->{pool}; | |
460 | $source->{abs_path} .= "\/$disks->{$disk}->{path}" if $disks->{$disk}->{path}; | |
461 | ||
462 | $source->{pool} = $disks->{$disk}->{pool}; | |
463 | $source->{path} = "\/$disks->{$disk}->{path}"; | |
464 | ||
465 | &$sync_path($source, $name, $cfg, $max_snap, $dest, $method); | |
466 | } | |
467 | } else { | |
468 | &$sync_path($source, $name, $cfg, $max_snap, $dest, $method); | |
469 | } | |
470 | } | |
471 | ||
472 | sub snapshot_get{ | |
473 | my ($source, $dest, $max_snap, $name) = @_; | |
474 | ||
475 | my $cmd = "zfs list -r -t snapshot -Ho name, -S creation "; | |
476 | ||
477 | $cmd .= $source->{abs_path}; | |
478 | $cmd = "ssh root\@$source->{ip} ".$cmd if $source->{ip}; | |
479 | ||
480 | my $raw = run_cmd($cmd); | |
481 | my $index = 1; | |
482 | my $line = ""; | |
483 | my $last_snap = undef; | |
484 | ||
485 | while ($raw && $raw =~ s/^(.*?)(\n|$)//) { | |
486 | $line = $1; | |
487 | $last_snap = $line if $index == 1; | |
488 | if ($index == $max_snap) { | |
489 | $source->{destroy} = 1; | |
490 | last; | |
491 | }; | |
492 | $index++; | |
493 | } | |
494 | ||
495 | $line =~ m/^(.+)\@(rep_$name\_.+)(\n|$)/; | |
496 | return ($2, $last_snap) if $2; | |
497 | ||
498 | return undef; | |
499 | } | |
500 | ||
501 | sub snapshot_add { | |
502 | my ($source, $dest, $name) = @_; | |
503 | ||
504 | my $date = get_date(); | |
505 | ||
506 | my $snap_name = "rep_$name\_".$date; | |
507 | ||
508 | $source->{new_snap} = $snap_name; | |
509 | ||
510 | my $path = $source->{abs_path}."\@".$snap_name; | |
511 | ||
512 | my $cmd = "zfs snapshot $path"; | |
513 | $cmd = "ssh root\@$source->{ip} ".$cmd if $source->{ip}; | |
514 | ||
515 | eval{ | |
516 | run_cmd($cmd); | |
517 | }; | |
518 | ||
519 | if (my $err = $@){ | |
520 | snapshot_destroy($source, $dest, 'ssh', $snap_name); | |
521 | die "$err\n"; | |
522 | } | |
523 | return $date; | |
524 | } | |
525 | ||
526 | sub cron_add { | |
527 | my ($vm) = @_; | |
528 | ||
529 | open(my $fh, '>>', "$CRONJOBS") | |
530 | or die "Could not open file: $!\n"; | |
531 | ||
532 | foreach my $name (keys%{$vm}){ | |
533 | my $text = "*/$vm->{$name}->{interval} * * * * root "; | |
534 | $text .= "$PATH$PROGNAME sync"; | |
535 | $text .= " -source "; | |
536 | if ($vm->{$name}->{vmid}) { | |
537 | $text .= "$vm->{$name}->{source_ip}:" if $vm->{$name}->{source_ip}; | |
538 | $text .= "$vm->{$name}->{vmid} "; | |
539 | } else { | |
540 | $text .= "$vm->{$name}->{source_ip}:" if $vm->{$name}->{source_ip}; | |
541 | $text .= "$vm->{$name}->{source_pool}"; | |
542 | $text .= "$vm->{$name}->{source_path}" if $vm->{$name}->{source_path}; | |
543 | } | |
544 | $text .= " -dest "; | |
545 | $text .= "$vm->{$name}->{dest_ip}:" if $vm->{$name}->{dest_ip}; | |
546 | $text .= "$vm->{$name}->{dest_pool}"; | |
547 | $text .= "$vm->{$name}->{dest_path}" if $vm->{$name}->{dest_path}; | |
548 | $text .= " -name $name "; | |
549 | $text .= " -limit $vm->{$name}->{limit}" if $vm->{$name}->{limit}; | |
550 | $text .= " -maxsnap $vm->{$name}->{maxsnap}" if $vm->{$name}->{maxsnap}; | |
551 | $text .= "\n"; | |
552 | print($fh $text); | |
553 | } | |
554 | close($fh); | |
555 | } | |
556 | ||
557 | sub cron_del { | |
558 | my ($source, $name) = @_; | |
559 | ||
560 | open(my $fh, '<', "$CRONJOBS") | |
561 | or die "Could not open file: $!\n"; | |
562 | ||
563 | $/ = undef; | |
564 | ||
565 | my $text = <$fh>; | |
566 | my $buffer = ""; | |
567 | close($fh); | |
568 | while ($text && $text =~ s/^(.*?)(\n|$)//) { | |
569 | my $line = $1.$2; | |
570 | if ($line !~ m/.*$PROGNAME.*$source.*$name.*/){ | |
571 | $buffer .= $line; | |
572 | } | |
573 | } | |
574 | open($fh, '>', "$CRONJOBS") | |
575 | or die "Could not open file: $!\n"; | |
576 | print($fh $buffer); | |
577 | close($fh); | |
578 | } | |
579 | ||
580 | sub get_disks { | |
581 | my ($target) = @_; | |
582 | ||
583 | my $cmd = ""; | |
584 | $cmd = "ssh root\@$target->{ip} " if $target->{ip}; | |
585 | $cmd .= "qm config $target->{vmid}"; | |
586 | ||
587 | my $res = run_cmd($cmd); | |
588 | ||
589 | my $disks = parse_disks($res, $target->{ip}); | |
590 | ||
591 | return $disks; | |
592 | } | |
593 | ||
594 | sub run_cmd { | |
595 | my ($cmd) = @_; | |
596 | print "Start CMD\n" if $DEBUG; | |
597 | print Dumper $cmd if $DEBUG; | |
598 | my $output = `$cmd 2>&1`; | |
599 | ||
600 | die $output if 0 != $?; | |
601 | ||
602 | chomp($output); | |
603 | print Dumper $output if $DEBUG; | |
604 | print "END CMD\n" if $DEBUG; | |
605 | return $output; | |
606 | } | |
607 | ||
608 | sub parse_disks { | |
609 | my ($text, $ip) = @_; | |
610 | ||
611 | my $disks; | |
612 | ||
613 | my $num = 0; | |
614 | my $cmd = ""; | |
615 | $cmd .= "ssh root\@$ip " if $ip; | |
616 | $cmd .= "pvesm zfsscan"; | |
617 | my $zfs_pools = run_cmd($cmd); | |
618 | while ($text && $text =~ s/^(.*?)(\n|$)//) { | |
619 | my $line = $1; | |
620 | my $disk = undef; | |
621 | my $stor = undef; | |
622 | if($line =~ m/^(virtio\d+: )(.+:)([A-Za-z0-9\-]+),(.*)$/) { | |
623 | $disk = $3; | |
624 | $stor = $2; | |
625 | } elsif($line =~ m/^(ide\d+: )(.+:)([A-Za-z0-9\-]+),(.*)$/) { | |
626 | $disk = $3; | |
627 | $stor = $2; | |
628 | } elsif($line =~ m/^(scsi\d+: )(.+:)([A-Za-z0-9\-]+),(.*)$/) { | |
629 | $disk = $3; | |
630 | $stor = $2; | |
631 | } elsif($line =~ m/^(sata\d+: )(.+:)([A-Za-z0-9\-]+),(.*)$/) { | |
632 | $disk = $3; | |
633 | $stor = $2; | |
634 | } | |
635 | ||
636 | if($disk && $disk ne "none" && $disk !~ m/cdrom/ ) { | |
637 | my $cmd = ""; | |
638 | $cmd .= "ssh root\@$ip " if $ip; | |
639 | $cmd .= "pvesm path $stor$disk"; | |
640 | my $path = run_cmd($cmd); | |
641 | ||
642 | if ($path =~ m/^\/dev\/zvol\/(\w+).*(\/$disk)$/){ | |
643 | ||
644 | $disks->{$num}->{pool} = $1; | |
645 | $disks->{$num}->{path} = $disk; | |
646 | $num++; | |
647 | ||
648 | } else { | |
649 | die "ERROR: in path\n"; | |
650 | } | |
651 | } | |
652 | } | |
653 | die "disk is not on ZFS Storage\n" if $num == 0; | |
654 | return $disks; | |
655 | } | |
656 | ||
657 | sub snapshot_destroy { | |
658 | my ($source, $dest, $method, $snap) = @_; | |
659 | ||
660 | my $zfscmd = "zfs destroy "; | |
661 | my $name = "$source->{path}\@$snap"; | |
662 | ||
663 | eval { | |
664 | if($source->{ip} && $method eq 'ssh'){ | |
665 | run_cmd("ssh root\@$source->{ip} $zfscmd $source->{pool}$name"); | |
666 | } else { | |
667 | run_cmd("$zfscmd $source->{pool}$name"); | |
668 | } | |
669 | }; | |
670 | if (my $erro = $@) { | |
671 | warn "WARN: $erro"; | |
672 | } | |
673 | if ($dest){ | |
674 | my $ssh = $dest->{ip} ? "ssh root\@$dest->{ip}" : ""; | |
675 | ||
676 | my $path = ""; | |
677 | $path ="$dest->{path}" if $dest->{path}; | |
678 | ||
679 | my @dir = split(/\//, $source->{path}); | |
680 | eval { | |
681 | run_cmd("$ssh $zfscmd $dest->{pool}$path\/$dir[@dir-1]\@$snap "); | |
682 | }; | |
683 | if (my $erro = $@) { | |
684 | warn "WARN: $erro"; | |
685 | } | |
686 | } | |
687 | } | |
688 | ||
689 | sub snapshot_exist { | |
690 | my ($source ,$dest, $method) = @_; | |
691 | ||
692 | my $cmd = ""; | |
693 | $cmd = "ssh root\@$dest->{ip} " if $dest->{ip}; | |
694 | $cmd .= "zfs list -rt snapshot -Ho name $dest->{pool}"; | |
695 | $cmd .= "$dest->{path}" if $dest->{path}; | |
696 | my @dir = split(/\//, $source->{path}); | |
697 | $cmd .= "\/$dir[@dir-1]\@$source->{old_snap}"; | |
698 | ||
699 | my $text = ""; | |
700 | eval {$text =run_cmd($cmd);}; | |
701 | if (my $erro = $@) { | |
702 | warn "WARN: $erro"; | |
703 | return undef; | |
704 | } | |
705 | ||
706 | while ($text && $text =~ s/^(.*?)(\n|$)//) { | |
707 | my $line = $1; | |
708 | return 1 if $line =~ m/^.*$source->{old_snap}$/; | |
709 | } | |
710 | } | |
711 | ||
712 | sub send_image { | |
713 | my ($source, $dest, $method, $verbose, $limit) = @_; | |
714 | ||
715 | my $cmd = ""; | |
716 | ||
717 | $cmd .= "ssh root\@$source->{ip} " if $source->{ip}; | |
718 | $cmd .= "zfs send "; | |
719 | $cmd .= "-v " if $verbose; | |
720 | ||
721 | if($source->{last_snap} && snapshot_exist($source ,$dest, $method)) { | |
722 | $cmd .= "-i $source->{abs_path}\@$source->{old_snap} $source->{abs_path}\@$source->{new_snap} "; | |
723 | } else { | |
724 | $cmd .= "$source->{abs_path}\@$source->{new_snap} "; | |
725 | } | |
726 | ||
727 | if ($limit){ | |
728 | my $bwl = $limit*1024; | |
729 | $cmd .= "| cstream -t $bwl"; | |
730 | } | |
731 | $cmd .= "| "; | |
732 | $cmd .= "ssh root\@$dest->{ip} " if $dest->{ip}; | |
733 | $cmd .= "zfs recv $dest->{pool}"; | |
734 | $cmd .= "$dest->{path}" if $dest->{path}; | |
735 | ||
736 | my @dir = split(/\//,$source->{path}); | |
737 | $cmd .= "\/$dir[@dir-1]\@$source->{new_snap}"; | |
738 | ||
739 | eval { | |
740 | run_cmd($cmd) | |
741 | }; | |
742 | ||
743 | if (my $erro = $@) { | |
744 | snapshot_destroy($source, undef, $method, $source->{new_snap}); | |
745 | die $erro; | |
746 | }; | |
747 | ||
748 | if ($source->{vmid}) { | |
749 | if ($method eq "ssh") { | |
750 | send_config($source, $dest,'ssh'); | |
751 | } | |
752 | } | |
753 | } | |
754 | ||
755 | ||
756 | sub send_config{ | |
757 | my ($source, $dest, $method) = @_; | |
758 | ||
759 | if ($method eq 'ssh'){ | |
760 | if ($dest->{ip} && $source->{ip}) { | |
761 | run_cmd("ssh root\@$dest->{ip} mkdir $VMCONFIG -p"); | |
762 | run_cmd("scp root\@$source->{ip}:$QEMU_CONF$source->{vmid}.conf root\@$dest->{ip}:$VMCONFIG$source->{vmid}.conf.$source->{new_snap}"); | |
763 | } elsif ($dest->{ip}) { | |
764 | run_cmd("ssh root\@$dest->{ip} mkdir $VMCONFIG -p"); | |
765 | run_cmd("scp $QEMU_CONF$source->{vmid}.conf root\@$dest->{ip}:$VMCONFIG$source->{vmid}.conf.$source->{new_snap}"); | |
766 | } elsif ($source->{ip}) { | |
767 | run_cmd("mkdir $VMCONFIG -p"); | |
768 | run_cmd("scp root\@$source->{ip}:$QEMU_CONF$source->{vmid}.conf $VMCONFIG$source->{vmid}.conf.$source->{new_snap}"); | |
769 | } | |
770 | ||
771 | if ($source->{destroy}){ | |
772 | if($dest->{ip}){ | |
773 | run_cmd("ssh root\@$dest->{ip} rm -f $VMCONFIG$source->{vmid}.conf.$source->{old_snap}"); | |
774 | } else { | |
775 | run_cmd("rm -f $VMCONFIG$source->{vmid}.conf.$source->{old_snap}"); | |
776 | } | |
777 | } | |
778 | } | |
779 | } | |
780 | ||
781 | sub get_date { | |
782 | my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time); | |
783 | my $datestamp = sprintf ( "%04d-%02d-%02d_%02d:%02d:%02d",$year+1900,$mon+1,$mday,$hour,$min,$sec); | |
784 | ||
785 | return $datestamp; | |
786 | } | |
787 | ||
788 | sub status { | |
789 | my $cfg = read_from_config("$CONFIG_PATH$CONFIG"); | |
790 | ||
791 | my $status_list = sprintf("%-25s%-15s%-10s\n","SOURCE","NAME","STATUS"); | |
792 | ||
793 | foreach my $source (keys%{$cfg}){ | |
794 | foreach my $sync_name (keys%{$cfg->{$source}}){ | |
795 | my $status; | |
796 | ||
797 | my $source_name = $source; | |
798 | ||
799 | $source_name = $cfg->{$source}->{$sync_name}->{source_ip}.":".$source if $cfg->{$source}->{$sync_name}->{source_ip}; | |
800 | ||
801 | if ($cfg->{$source}->{$sync_name}->{locked} eq 'no'){ | |
802 | $status = sprintf("%-10s","OK"); | |
803 | } elsif ($cfg->{$source}->{$sync_name}->{locked} eq 'yes' && | |
804 | $cfg->{$source}->{$sync_name}->{failure}) { | |
805 | $status = sprintf("%-10s","sync error"); | |
806 | } else { | |
807 | $status = sprintf("%-10s","syncing"); | |
808 | } | |
809 | ||
810 | $status_list .= sprintf("%-25s%-15s", cut_to_width($source_name,25), cut_to_width($sync_name,15)); | |
811 | $status_list .= "$status\n"; | |
812 | } | |
813 | } | |
814 | ||
815 | return $status_list; | |
816 | } | |
817 | ||
818 | ||
819 | my $command = $ARGV[0]; | |
820 | ||
821 | my $commands = {'destroy' => 1, | |
822 | 'create' => 1, | |
823 | 'sync' => 1, | |
824 | 'list' => 1, | |
825 | 'status' => 1, | |
826 | 'help' => 1}; | |
827 | ||
828 | if (!$command || !$commands->{$command}) { | |
829 | usage(); | |
830 | die "\n"; | |
831 | } | |
832 | ||
833 | my $dest = ''; | |
834 | my $source = ''; | |
835 | my $verbose = ''; | |
836 | my $interval = ''; | |
837 | my $limit = ''; | |
838 | my $maxsnap = ''; | |
839 | my $name = ''; | |
840 | my $skip = ''; | |
841 | ||
842 | my $help_sync = "zfs-zsync sync -dest <string> -source <string> [OPTIONS]\n | |
843 | \twill sync one time\n | |
844 | \t-dest\tstring\n | |
845 | \t\tthe destination target is like [IP:]<Pool>[/Path]\n | |
846 | \t-limit\tinteger\n | |
847 | \t\tmax sync speed in kBytes/s, default unlimited\n | |
848 | \t-maxsnap\tinteger\n | |
849 | \t\thow much snapshots will be kept before get erased, default 1/n | |
850 | \t-name\tstring\n | |
851 | \t\tname of the sync job, if not set it is default. | |
852 | \tIt is only necessary if scheduler allready contains this source.\n | |
853 | \t-source\tstring\n | |
854 | \t\tthe source can be an <VMID> or [IP:]<ZFSPool>[/Path]\n"; | |
855 | ||
856 | my $help_create = "zfs-zsync create -dest <string> -source <string> [OPTIONS]/n | |
857 | \tCreate a sync Job\n | |
858 | \t-dest\tstringn\n | |
859 | \t\tthe destination target is like [IP]:<Pool>[/Path]\n | |
860 | \t-interval\tinteger\n | |
861 | \t\tthe interval in min in witch the zfs will sync, | |
862 | \t\tdefault is 15 min\n | |
863 | \t-limit\tinteger\n | |
864 | \t\tmax sync speed, default unlimited\n | |
865 | \t-maxsnap\tstring\n | |
866 | \t\thow much snapshots will be kept before get erased, default 1\n | |
867 | \t-name\tstring\n | |
868 | \t\tname of the sync job, if not set it is default\n | |
869 | \t-skip\tboolean\n | |
870 | \t\tif this flag is set it will skip the first sync\n | |
871 | \t-source\tstring\n | |
872 | \t\tthe source can be an <VMID> or [IP:]<ZFSPool>[/Path]\n"; | |
873 | ||
874 | my $help_destroy = "zfs-zsync destroy -source <string> [OPTIONS]\n | |
875 | \tremove a sync Job from the scheduler\n | |
876 | \t-name\tstring\n | |
877 | \t\tname of the sync job, if not set it is default\n | |
878 | \t-source\tstring\n | |
879 | \t\tthe source can be an <VMID> or [IP:]<ZFSPool>[/Path]\n"; | |
880 | ||
881 | my $help_help = "zfs-zsync help <cmd> [OPTIONS]\n | |
882 | \tGet help about specified command.\n | |
883 | \t<cmd>\tstring\n | |
884 | \t\tCommand name\n | |
885 | \t-verbose\tboolean\n | |
886 | \t\tVerbose output format.\n"; | |
887 | ||
888 | my $help_list = "zfs-zsync list\n | |
889 | \tGet a List of all scheduled Sync Jobs\n"; | |
890 | ||
891 | my $help_status = "zfs-zsync status\n | |
892 | \tGet the status of all scheduled Sync Jobs\n"; | |
893 | ||
894 | sub help{ | |
895 | my ($command) = @_; | |
896 | ||
897 | switch($command){ | |
898 | case 'help' | |
899 | { | |
900 | die "$help_help\n"; | |
901 | } | |
902 | case 'sync' | |
903 | { | |
904 | die "$help_sync\n"; | |
905 | } | |
906 | case 'destroy' | |
907 | { | |
908 | die "$help_destroy\n"; | |
909 | } | |
910 | case 'create' | |
911 | { | |
912 | die "$help_create\n"; | |
913 | } | |
914 | case 'list' | |
915 | { | |
916 | die "$help_list\n"; | |
917 | } | |
918 | case 'status' | |
919 | { | |
920 | die "$help_status\n"; | |
921 | } | |
922 | } | |
923 | ||
924 | } | |
925 | ||
926 | my $err = GetOptions ('dest=s' => \$dest, | |
927 | 'source=s' => \$source, | |
928 | 'verbose' => \$verbose, | |
929 | 'interval=i' => \$interval, | |
930 | 'limit=i' => \$limit, | |
931 | 'maxsnap=i' => \$maxsnap, | |
932 | 'name=s' => \$name, | |
933 | 'skip' => \$skip); | |
934 | ||
935 | if ($err == 0) { | |
936 | die "can't parse options\n"; | |
937 | } | |
938 | ||
939 | my $param; | |
940 | $param->{dest} = $dest; | |
941 | $param->{source} = $source; | |
942 | $param->{verbose} = $verbose; | |
943 | $param->{interval} = $interval; | |
944 | $param->{limit} = $limit; | |
945 | $param->{maxsnap} = $maxsnap; | |
946 | $param->{name} = $name; | |
947 | $param->{skip} = $skip; | |
948 | ||
949 | switch($command){ | |
950 | case "destroy" | |
951 | { | |
952 | die "$help_destroy\n" if !$source; | |
953 | check_target($source); | |
954 | destroy($param); | |
955 | } | |
956 | case "sync" | |
957 | { | |
958 | die "$help_sync\n" if !$source || !$dest; | |
959 | check_target($source); | |
960 | check_target($dest); | |
961 | sync($param); | |
962 | } | |
963 | case "create" | |
964 | { | |
965 | die "$help_create\n" if !$source || !$dest; | |
966 | check_target($source); | |
967 | check_target($dest); | |
968 | init($param); | |
969 | } | |
970 | case "status" | |
971 | { | |
972 | print status(); | |
973 | } | |
974 | case "list" | |
975 | { | |
976 | print list(); | |
977 | } | |
978 | case "help" | |
979 | { | |
980 | my $help_command = $ARGV[1]; | |
981 | if ($help_command && $commands->{$help_command}) { | |
982 | print help($help_command); | |
983 | } | |
984 | if ($verbose){ | |
985 | exec("man $PROGNAME"); | |
986 | } else { | |
987 | usage(1); | |
988 | } | |
989 | } | |
990 | } | |
991 | ||
992 | sub usage{ | |
993 | my ($help) = @_; | |
994 | ||
995 | print("ERROR:\tno command specified\n") if !$help; | |
996 | print("USAGE:\t$PROGNAME <COMMAND> [ARGS] [OPTIONS]\n"); | |
997 | print("\tpve-zsync help [<cmd>] [OPTIONS]\n\n"); | |
998 | print("\tpve-zsync create -dest <string> -source <string> [OPTIONS]\n"); | |
999 | print("\tpve-zsync destroy -source <string> [OPTIONS]\n"); | |
1000 | print("\tpve-zsync list\n"); | |
1001 | print("\tpve-zsync status\n"); | |
1002 | print("\tpve-zsync sync -dest <string> -source <string> [OPTIONS]\n"); | |
1003 | } | |
1004 | ||
1005 | sub check_target{ | |
1006 | my ($target) = @_; | |
1007 | ||
1008 | chomp($target); | |
1009 | ||
1010 | if($target !~ m/(\d+.\d+.\d+.\d+:)?([\w\-\_\/]+)(\/.+)?/){ | |
1011 | print("ERROR:\t$target is not valid.\n\tUse [IP:]<ZFSPool>[/Path]!\n"); | |
1012 | return 1; | |
1013 | } | |
1014 | return undef; | |
1015 | } | |
1016 | ||
1017 | __END__ | |
1018 | ||
1019 | =head1 NAME | |
1020 | ||
1021 | pve-zsync - PVE ZFS Replication Manager | |
1022 | ||
1023 | =head1 SYNOPSIS | |
1024 | ||
1025 | zfs-zsync <COMMAND> [ARGS] [OPTIONS] | |
1026 | ||
1027 | zfs-zsync help <cmd> [OPTIONS] | |
1028 | ||
1029 | Get help about specified command. | |
1030 | ||
1031 | <cmd> string | |
1032 | ||
1033 | Command name | |
1034 | ||
1035 | -verbose boolean | |
1036 | ||
1037 | Verbose output format. | |
1038 | ||
1039 | zfs-zsync create -dest <string> -source <string> [OPTIONS] | |
1040 | ||
1041 | Create a sync Job | |
1042 | ||
1043 | -dest string | |
1044 | ||
1045 | the destination target is like [IP]:<Pool>[/Path] | |
1046 | ||
1047 | -interval integer | |
1048 | ||
1049 | the interval in min in witch the zfs will sync, default is 15 min | |
1050 | ||
1051 | -limit integer | |
1052 | ||
1053 | max sync speed, default unlimited | |
1054 | ||
1055 | -maxsnap string | |
1056 | ||
1057 | how much snapshots will be kept before get erased, default 1 | |
1058 | ||
1059 | -name string | |
1060 | ||
1061 | name of the sync job, if not set it is default | |
1062 | ||
1063 | -skip boolean | |
1064 | ||
1065 | if this flag is set it will skip the first sync | |
1066 | ||
1067 | -source string | |
1068 | ||
1069 | the source can be an <VMID> or [IP:]<ZFSPool>[/Path] | |
1070 | ||
1071 | zfs-zsync destroy -source <string> [OPTIONS] | |
1072 | ||
1073 | remove a sync Job from the scheduler | |
1074 | ||
1075 | -name string | |
1076 | ||
1077 | name of the sync job, if not set it is default | |
1078 | ||
1079 | -source string | |
1080 | ||
1081 | the source can be an <VMID> or [IP:]<ZFSPool>[/Path] | |
1082 | ||
1083 | zfs-zsync list | |
1084 | ||
1085 | Get a List of all scheduled Sync Jobs | |
1086 | ||
1087 | zfs-zsync status | |
1088 | ||
1089 | Get the status of all scheduled Sync Jobs | |
1090 | ||
1091 | zfs-zsync sync -dest <string> -source <string> [OPTIONS] | |
1092 | ||
1093 | will sync one time | |
1094 | ||
1095 | -dest string | |
1096 | ||
1097 | the destination target is like [IP:]<Pool>[/Path] | |
1098 | ||
1099 | -limit integer | |
1100 | ||
1101 | max sync speed in kBytes/s, default unlimited | |
1102 | ||
1103 | -maxsnap integer | |
1104 | ||
1105 | how much snapshots will be kept before get erased, default 1 | |
1106 | ||
1107 | -name string | |
1108 | ||
1109 | name of the sync job, if not set it is default. | |
1110 | It is only necessary if scheduler allready contains this source. | |
1111 | ||
1112 | -source string | |
1113 | ||
1114 | the source can be an <VMID> or [IP:]<ZFSPool>[/Path] | |
1115 | ||
1116 | =head1 DESCRIPTION | |
1117 | ||
1118 | This Tool helps you to sync your VM or directory which stored on ZFS between 2 servers. | |
1119 | This tool also has the capability to add jobs to cron so the sync will be automatically done. | |
1120 | ||
1121 | =head2 PVE ZFS Storage sync Tool | |
1122 | ||
1123 | This Tool can get remote pool on other PVE or send Pool to others ZFS machines | |
1124 | ||
1125 | =head1 EXAMPLES | |
1126 | ||
1127 | add sync job from local VM to remote ZFS Server | |
1128 | zfs-zsync -source=100 -dest=192.168.1.2:zfspool | |
1129 | ||
1130 | =head1 IMPORTANT FILES | |
1131 | ||
1132 | Where the cron jobs are stored /etc/cron.d/pve-zsync | |
1133 | Where the VM config get copied on the destination machine /var/pve-zsync | |
1134 | Where the config is stored /var/pve-zsync | |
1135 | ||
1136 | Copyright (C) 2007-2015 Proxmox Server Solutions GmbH | |
1137 | ||
1138 | This program is free software: you can redistribute it and/or modify it | |
1139 | under the terms of the GNU Affero General Public License as published | |
1140 | by the Free Software Foundation, either version 3 of the License, or | |
1141 | (at your option) any later version. | |
1142 | ||
1143 | This program is distributed in the hope that it will be useful, but | |
1144 | WITHOUT ANY WARRANTY; without even the implied warranty of | |
1145 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
1146 | Affero General Public License for more details. | |
1147 | ||
1148 | You should have received a copy of the GNU Affero General Public | |
1149 | License along with this program. If not, see | |
1150 | <http://www.gnu.org/licenses/>. |