]> git.proxmox.com Git - pmg-api.git/blob - PMG/LDAPCache.pm
pmgconfig ldapsync: improver error logging
[pmg-api.git] / PMG / LDAPCache.pm
1 package PMG::LDAPCache;
2
3 use strict;
4 use warnings;
5 use File::Path;
6 use LockFile::Simple;
7 use Data::Dumper;
8 use Net::LDAP;
9 use Net::LDAP::Control::Paged;
10 use Net::LDAP::Constant qw (LDAP_CONTROL_PAGED);
11 use DB_File;
12
13 use PVE::SafeSyslog;
14
15 use PMG::Utils;
16
17 $DB_HASH->{'cachesize'} = 10000;
18 $DB_RECNO->{'cachesize'} = 10000;
19 $DB_BTREE->{'cachesize'} = 10000;
20 $DB_BTREE->{'flags'} = R_DUP ;
21
22 my $cachedir = '/var/lib/pmg';
23
24 my $last_atime = {};
25 my $ldapcache = {};
26
27 # DB Description
28 #
29 # users (hash): UID -> pmail, account, DN
30 # dnames (hash): DN -> UID
31 # accounts (hash): account -> UID
32 # mail (hash): mail -> UID
33 # groups (hash): group -> GID
34 # memberof (btree): UID -> GID
35 #
36 my @dbs = ('users', 'dnames', 'groups', 'mails', 'accounts', 'memberof');
37
38 sub new {
39 my ($self, %args) = @_;
40
41 my $type = ref($self) || $self;
42
43 die "undefined ldap id" if !$args{id};
44
45 my $id = $args{id};
46
47 if ($ldapcache->{$id}) {
48 $self = $ldapcache->{$id};
49 } else {
50 $ldapcache->{$id} = $self = bless {}, $type;
51 $self->{id} = $id;
52 }
53
54 if (!$args{mailattr}) {
55 $args{mailattr} = "mail, userPrincipalName, proxyAddresses, othermailbox";
56 }
57 $args{mailattr} =~ s/[\,\;]/ /g;
58 $args{mailattr} =~ s/\s+/,/g;
59
60 if ($args{mode} && ($args{mode} eq 'ldap' || $args{mode} eq 'ldaps')) {
61 $self->{mode} = $args{mode};
62 } else {
63 $self->{mode} = 'ldap';
64 }
65
66 $self->{accountattr} = $args{accountattr} || 'sAMAccountName';
67 @{$self->{mailattr}} = split(/,/, $args{mailattr});
68 $self->{server1} = $args{server1};
69 $self->{server2} = $args{server2};
70 $self->{binddn} = $args{binddn};
71 $self->{bindpw} = $args{bindpw};
72 $self->{basedn} = $args{basedn};
73 $self->{port} = $args{port};
74 $self->{groupbasedn} = $args{groupbasedn};
75 $self->{filter} = $args{filter};
76
77 if ($args{syncmode} == 1) {
78 # read local data only
79 $self->{errors} = '';
80 $self->loadcache();
81 return $self;
82 }
83
84 return $self if !($args{server1});
85
86 if ($args{syncmode} == 2) {
87 # force sync
88 $self->loaddata(1);
89 } else {
90 $self->loaddata();
91 }
92
93 return $self;
94 }
95
96 sub lockdir {
97 my ($id) = @_;
98
99 my $dir = "$cachedir/ldapdb_$id";
100 my $scheme = LockFile::Simple->make(
101 -warn => 0, -stale => 1, -autoclean => 1);
102 my $lock = $scheme->lock($dir);
103
104 return $lock;
105 }
106
107 sub delete {
108 my ($class, $id) = @_;
109
110 if (my $lock = lockdir($id)) {
111 delete $ldapcache->{$id};
112 delete $last_atime->{$id};
113 my $dir = "$cachedir/ldapdb_$id";
114 rmtree $dir;
115 $lock->release;
116 } else {
117 syslog('err' , "can't lock ldap database '$id'");
118 }
119 }
120
121 sub update {
122 my ($self, $syncmode) = @_;
123
124 if ($syncmode == 1) {
125 # read local data only
126 $self->{errors} = '';
127 $self->loadcache();
128 } elsif ($syncmode == 2) {
129 # force sync
130 $self->loaddata(1);
131 } else {
132 $self->loaddata();
133 }
134 }
135
136 sub queryusers {
137 my ($self, $ldap) = @_;
138
139 my $filter = '(|';
140 foreach my $attr (@{$self->{mailattr}}) {
141 $filter .= "($attr=*)";
142 }
143 $filter .= ')';
144
145 if ($self->{filter}) {
146 my $tmp = $self->{filter};
147 $tmp = "($tmp)" if $tmp !~ m/^\(.*\)$/;
148
149 $filter = "(&${filter}${tmp})";
150 }
151
152 my $page = Net::LDAP::Control::Paged->new(size => 900);
153
154 my @args = (
155 base => $self->{basedn},
156 scope => "subtree",
157 filter => $filter,
158 control => [ $page ],
159 attrs => [ @{$self->{mailattr}}, $self->{accountattr}, 'memberOf' ]
160 );
161
162 my $cookie;
163
164 while(1) {
165
166 my $mesg = $ldap->search(@args);
167
168 # stop on error
169 if ($mesg->code) {
170 my $err = "ldap user search error: " . $mesg->error;
171 $self->{errors} .= "$err\n";
172 syslog('err', $err);
173 last;
174 }
175
176 #foreach my $entry ($mesg->entries) { $entry->dump; }
177 foreach my $entry ($mesg->entries) {
178 my $dn = $entry->dn;
179
180 my $umails = {};
181 my $pmail;
182
183 foreach my $attr (@{$self->{mailattr}}) {
184 foreach my $mail ($entry->get_value($attr)) {
185 $mail = lc($mail);
186 # Test if the Line starts with one of the following lines:
187 # proxyAddresses: [smtp|SMTP]:
188 # and also discard this starting string, so that $mail is only the
189 # address without any other characters...
190
191 $mail =~ s/^(smtp|SMTP)[\:\$]//gs;
192
193 if ($mail !~ m/[\{\}\\\/]/ && $mail =~ m/^\S+\@\S+$/) {
194 $umails->{$mail} = 1;
195 $pmail = $mail if !$pmail;
196 }
197 }
198 }
199 my $addresses = [ keys %$umails ];
200
201 next if !$pmail; # account has no email addresses
202
203 my $cuid;
204 $self->{dbstat}->{dnames}->{dbh}->get($dn, $cuid);
205 if (!$cuid) {
206 $cuid = ++$self->{dbstat}->{dnames}->{idcount};
207 $self->{dbstat}->{dnames}->{dbh}->put($dn, $cuid);
208 }
209
210 my $account = $entry->get_value($self->{accountattr});
211 if ($account && ($account =~ m/^\S+$/s)) {
212 $account = lc($account);
213 $self->{dbstat}->{accounts}->{dbh}->put($account, $cuid);
214 my $data = pack('n/a* n/a* n/a*', $pmail, $account, $dn);
215 $self->{dbstat}->{users}->{dbh}->put($cuid, $data);
216 }
217
218 foreach my $mail (@$addresses) {
219 $self->{dbstat}->{mails}->{dbh}->put($mail, $cuid);
220 }
221
222 if (!$self->{groupbasedn}) {
223 my @groups = $entry->get_value('memberOf');
224 foreach my $group (@groups) {
225 my $cgid;
226 $self->{dbstat}->{groups}->{dbh}->get($group, $cgid);
227 if (!$cgid) {
228 $cgid = ++$self->{dbstat}->{groups}->{idcount};
229 $self->{dbstat}->{groups}->{dbh}->put($group, $cgid);
230 }
231 $self->{dbstat}->{memberof}->{dbh}->put($cuid, $cgid);
232 }
233 }
234 }
235
236 # Get cookie from paged control
237 my ($resp) = $mesg->control(LDAP_CONTROL_PAGED) or last;
238 $cookie = $resp->cookie or last;
239
240 # Set cookie in paged control
241 $page->cookie($cookie);
242 }
243
244
245 if ($cookie) {
246 # We had an abnormal exit, so let the server know we do not want any more
247 $page->cookie($cookie);
248 $page->size(0);
249 $ldap->search(@args);
250 my $err = "LDAP user query unsuccessful";
251 $self->{errors} .= "$err\n";
252 syslog('err', $err);
253 }
254 }
255
256 sub querygroups {
257 my ($self, $ldap) = @_;
258
259 return undef if !$self->{groupbasedn};
260
261 my $filter = "(|(objectclass=group)(objectclass=univentionGroup))";
262
263 my $page = Net::LDAP::Control::Paged->new(size => 100);
264
265 my @args = ( base => $self->{groupbasedn},
266 scope => "subtree",
267 filter => $filter,
268 control => [ $page ],
269 attrs => [ 'member', 'uniqueMember' ],
270 );
271
272 my $cookie;
273 while(1) {
274
275 my $mesg = $ldap->search(@args);
276
277 # stop on error
278 if ($mesg->code) {
279 my $err = "ldap group search error: " . $mesg->error;
280 $self->{errors} .= "$err\n";
281 syslog('err', $err);
282 last;
283 }
284
285 foreach my $entry ( $mesg->entries ) {
286 my $group = $entry->dn;
287 my @members = $entry->get_value('member');
288 if (!scalar(@members)) {
289 @members = $entry->get_value('uniqueMember');
290 }
291 my $cgid;
292 $self->{dbstat}->{groups}->{dbh}->get($group, $cgid);
293 if (!$cgid) {
294 $cgid = ++$self->{dbstat}->{groups}->{idcount};
295 $self->{dbstat}->{groups}->{dbh}->put($group, $cgid);
296 }
297
298 foreach my $m (@members) {
299
300 my $cuid;
301 $self->{dbstat}->{dnames}->{dbh}->get($m, $cuid);
302 if (!$cuid) {
303 $cuid = ++$self->{dbstat}->{dnames}->{idcount};
304 $self->{dbstat}->{dnames}->{dbh}->put($m, $cuid);
305 }
306
307 $self->{dbstat}->{memberof}->{dbh}->put($cuid, $cgid);
308 }
309 }
310
311 # Get cookie from paged control
312 my ($resp) = $mesg->control(LDAP_CONTROL_PAGED) or last;
313 $cookie = $resp->cookie or last;
314
315 # Set cookie in paged control
316 $page->cookie($cookie);
317 }
318
319 if ($cookie) {
320 # We had an abnormal exit, so let the server know we do not want any more
321 $page->cookie($cookie);
322 $page->size(0);
323 $ldap->search(@args);
324 my $err = "LDAP group query unsuccessful";
325 $self->{errors} .= "$err\n";
326 syslog('err', $err);
327 }
328 }
329
330 sub ldap_connect {
331 my ($self) = @_;
332
333 my $hosts = [ $self->{server1} ];
334
335 push @$hosts, $self->{server2} if $self->{server2};
336
337 my $opts = { timeout => 10, onerror => 'die' };
338
339 $opts->{port} = $self->{port} if $self->{port};
340 $opts->{schema} = $self->{mode};
341
342 return Net::LDAP->new($hosts, %$opts);
343 }
344
345 sub ldap_connect_and_bind {
346 my ($self) = @_;
347
348 my $ldap = $self->ldap_connect() ||
349 die "Can't bind to ldap server '$self->{id}': $!\n";
350
351 my $mesg;
352
353 if ($self->{binddn}) {
354 $mesg = $ldap->bind($self->{binddn}, password => $self->{bindpw});
355 } else {
356 $mesg = $ldap->bind(); # anonymous bind
357 }
358
359 die "ldap bind failed: " . $mesg->error . "\n" if $mesg->code;
360
361 if (!$self->{basedn}) {
362 my $root = $ldap->root_dse(attrs => [ 'defaultNamingContext' ]);
363 $self->{basedn} = $root->get_value('defaultNamingContext');
364 }
365
366 return $ldap;
367 }
368
369 sub sync_database {
370 my ($self) = @_;
371
372 my $dir = "ldapdb_" . $self->{id};
373 mkdir "$cachedir/$dir";
374
375 # open ldap connection
376
377 my $ldap;
378
379 eval { $ldap = $self->ldap_connect_and_bind(); };
380 if (my $err = $@) {
381 $self->{errors} .= "$err\n";
382 syslog('err', $err);
383 return;
384 }
385
386 # open temporary database files
387
388 my $olddbh = {};
389
390 foreach my $db (@dbs) {
391 $self->{dbstat}->{$db}->{tmpfilename} = "$cachedir/$dir/${db}_tmp$$.db";
392 $olddbh->{$db} = $self->{dbstat}->{$db}->{dbh};
393 }
394
395 my $error_cleanup = sub {
396 # close and delete all files
397 foreach my $db (@dbs) {
398 undef $self->{dbstat}->{$db}->{dbh};
399 unlink $self->{dbstat}->{$db}->{tmpfilename};
400 $self->{dbstat}->{$db}->{dbh} = $olddbh->{$db};
401 }
402 };
403
404 eval {
405 foreach my $db (@dbs) {
406 my $filename = $self->{dbstat}->{$db}->{tmpfilename};
407 $self->{dbstat}->{$db}->{idcount} = 0;
408 unlink $filename;
409
410 if ($db eq 'memberof') {
411 $self->{dbstat}->{$db}->{dbh} =
412 tie (my %h, 'DB_File', $filename,
413 O_CREAT|O_RDWR, 0666, $DB_BTREE);
414 } else {
415 $self->{dbstat}->{$db}->{dbh} =
416 tie (my %h, 'DB_File', $filename,
417 O_CREAT|O_RDWR, 0666, $DB_HASH);
418 }
419
420 die "unable to open database file '$filename': $!\n"
421 if !$self->{dbstat}->{$db}->{dbh};
422 }
423 };
424 if (my $err = $@) {
425 $error_cleanup->();
426 $self->{errors} .= $err;
427 syslog('err', $err);
428 return;
429 }
430
431 $self->querygroups ($ldap) if $self->{groupbasedn};
432
433 $self->queryusers($ldap) if !$self->{errors};
434
435 $ldap->unbind;
436
437 if ($self->{errors}) {
438 $error_cleanup->();
439 return;
440 }
441
442 my $lock = lockdir($self->{id});
443
444 if (!$lock) {
445 my $err = "unable to get database lock for ldap database '$self->{id}'";
446 $self->{errors} .= "$err\n";
447 syslog('err', $err);
448 $error_cleanup->();
449 return;
450 }
451
452 foreach my $db (@dbs) {
453 my $filename = $self->{dbstat}->{$db}->{filename} =
454 "$cachedir/$dir/${db}.db";
455 $self->{dbstat}->{$db}->{dbh}->sync(); # flush everything
456 rename $self->{dbstat}->{$db}->{tmpfilename}, $filename;
457 }
458
459 $lock->release;
460
461 $last_atime->{$self->{id}} = time();
462
463 $self->{gcount} = $self->{dbstat}->{groups}->{idcount};
464 $self->{ucount} = __count_entries($self->{dbstat}->{accounts}->{dbh});
465 $self->{mcount} = __count_entries($self->{dbstat}->{mails}->{dbh});
466 }
467
468 sub __count_entries {
469 my ($dbh) = @_;
470
471 return 0 if !$dbh;
472
473 my $key = 0 ;
474 my $value = "" ;
475 my $count = 0;
476 my $status = $dbh->seq($key, $value, R_FIRST());
477
478 while ($status == 0) {
479 $count++;
480 $status = $dbh->seq($key, $value, R_NEXT());
481 }
482
483 return $count;
484 }
485
486 sub loadcache {
487 my ($self, $try) = @_;
488
489 my $dir = "ldapdb_" . $self->{id};
490 mkdir "$cachedir/$dir";
491
492 my $filename = "$cachedir/$dir/mails.db";
493
494 return if $last_atime->{$self->{id}} &&
495 PMG::Utils::file_older_than ($filename, $last_atime->{$self->{id}});
496
497 eval {
498 foreach my $db (@dbs) {
499 my $filename = $self->{dbstat}->{$db}->{filename} =
500 "$cachedir/$dir/${db}.db";
501 $self->{dbstat}->{$db}->{idcount} = 0;
502 if ($db eq 'memberof') {
503 $self->{dbstat}->{$db}->{dbh} =
504 tie (my %h, 'DB_File', $filename,
505 O_RDONLY, 0666, $DB_BTREE);
506 } else {
507 $self->{dbstat}->{$db}->{dbh} =
508 tie (my %h, 'DB_File', $filename,
509 O_RDONLY, 0666, $DB_HASH);
510 }
511
512 if (!$self->{dbstat}->{$db}->{dbh} && !$try) {
513 my $err = "ldap error - unable to open database file '$filename': $!";
514 $self->{errors} .= "$err\n";
515 syslog('err', $err) if !$self->{dbstat}->{$db}->{dbh};
516 }
517 }
518 };
519
520 $last_atime->{$self->{id}} = time();
521
522 $self->{gcount} = __count_entries($self->{dbstat}->{groups}->{dbh});
523 $self->{ucount} = __count_entries($self->{dbstat}->{accounts}->{dbh});
524 $self->{mcount} = __count_entries($self->{dbstat}->{mails}->{dbh});
525 }
526
527 sub loaddata {
528 my ($self, $force) = @_;
529
530 $self->{errors} = '';
531
532 if (!$force) {
533 # only sync if file is older than 1 hour
534
535 my $dir = "ldapdb_" . $self->{id};
536 mkdir "$cachedir/$dir";
537 my $filename = "$cachedir/$dir/mails.db";
538
539 if (-e $filename &&
540 !PMG::Utils::file_older_than($filename, time() - 3600)) {
541 $self->loadcache();
542 return;
543 }
544 }
545
546 $self->sync_database();
547
548 if ($self->{errors}) {
549 $self->loadcache(1);
550 }
551 }
552
553 sub list_groups {
554 my ($self) = @_;
555
556 my $res = [];
557
558 my $dbh = $self->{dbstat}->{groups}->{dbh};
559
560 return $res if !$dbh;
561
562 my $key = 0 ;
563 my $value = "" ;
564 my $status = $dbh->seq($key, $value, R_FIRST());
565 my $keys;
566
567 while ($status == 0) {
568 push @$res, {
569 dn => $key,
570 };
571 $status = $dbh->seq($key, $value, R_NEXT());
572 }
573
574 return $res;
575 }
576
577 sub list_users {
578 my ($self) = @_;
579
580 my $res = [];
581
582 my $dbh = $self->{dbstat}->{users}->{dbh};
583
584 return $res if !$dbh;
585
586 my $key = 0 ;
587 my $value = "" ;
588 my $status = $dbh->seq($key, $value, R_FIRST());
589 my $keys;
590
591 while ($status == 0) {
592 my ($pmail, $account, $dn) = unpack('n/a* n/a* n/a*', $value);
593 push @$res, {
594 pmail => $pmail,
595 account => $account,
596 dn => $dn,
597 };
598 $status = $dbh->seq($key, $value, R_NEXT());
599 }
600
601 return $res;
602 }
603
604 sub list_addresses {
605 my ($self, $mail) = @_;
606
607 my $dbhmails = $self->{dbstat}->{mails}->{dbh};
608 my $dbhusers = $self->{dbstat}->{users}->{dbh};
609
610 return undef if !$dbhmails || !$dbhusers;
611
612 $mail = lc($mail);
613
614 my $res = [];
615
616 my $cuid;
617 $dbhmails->get($mail, $cuid);
618 return undef if !$cuid;
619
620 my $rdata;
621 $dbhusers->get($cuid, $rdata);
622 return undef if !$rdata;
623
624 my ($pmail, $account, $dn) = unpack('n/a* n/a* n/a*', $rdata);
625
626 push @$res, { primary => 1, email => $pmail };
627
628 my $key = 0 ;
629 my $value = "" ;
630 my $status = $dbhmails->seq($key, $value, R_FIRST());
631
632 while ($status == 0) {
633 if ($value == $cuid && $key ne $pmail) {
634 push @$res, { primary => 0, email => $key };
635 }
636 $status = $dbhmails->seq($key, $value, R_NEXT());
637 }
638
639 return $res;
640 }
641
642 sub mail_exists {
643 my ($self, $mail) = @_;
644
645 my $dbh = $self->{dbstat}->{mails}->{dbh};
646 return 0 if !$dbh;
647
648 $mail = lc($mail);
649
650 my $res;
651 $dbh->get($mail, $res);
652 return $res;
653 }
654
655 sub account_exists {
656 my ($self, $account) = @_;
657
658 my $dbh = $self->{dbstat}->{accounts}->{dbh};
659 return 0 if !$dbh;
660
661 $account = lc($account);
662
663 my $res;
664 $dbh->get($account, $res);
665 return $res;
666 }
667
668 sub group_exists {
669 my ($self, $group) = @_;
670
671 my $dbh = $self->{dbstat}->{groups}->{dbh};
672 return 0 if !$dbh;
673
674 my $res;
675 $dbh->get($group, $res);
676 return $res;
677 }
678
679 sub account_has_address {
680 my ($self, $account, $mail) = @_;
681
682 my $dbhmails = $self->{dbstat}->{mails}->{dbh};
683 my $dbhaccounts = $self->{dbstat}->{accounts}->{dbh};
684 return 0 if !$dbhmails || !$dbhaccounts;
685
686 $account = lc($account);
687 $mail = lc($mail);
688
689 my $accid;
690 $dbhaccounts->get($account, $accid);
691 return 0 if !$accid;
692
693 my $mailid;
694 $dbhmails->get($mail, $mailid);
695 return 0 if !$mailid;
696
697 return ($accid == $mailid);
698 }
699
700 sub user_in_group {
701 my ($self, $mail, $group) = @_;
702
703 my $dbhmails = $self->{dbstat}->{mails}->{dbh};
704 my $dbhgroups = $self->{dbstat}->{groups}->{dbh};
705 my $dbhmemberof = $self->{dbstat}->{memberof}->{dbh};
706
707 return 0 if !$dbhmails || !$dbhgroups || !$dbhmemberof;
708
709 $mail = lc($mail);
710
711 my $cuid;
712 $dbhmails->get($mail, $cuid);
713 return 0 if !$cuid;
714
715 my $groupid;
716 $dbhgroups->get($group, $groupid);
717 return 0 if !$groupid;
718
719 my @gida = $dbhmemberof->get_dup($cuid);
720
721 return grep { $_ eq $groupid } @gida;
722 }
723
724 sub account_info {
725 my ($self, $mail, $scan) = @_;
726
727 my $dbhmails = $self->{dbstat}->{mails}->{dbh};
728 my $dbhusers = $self->{dbstat}->{users}->{dbh};
729
730 return undef if !$dbhmails || !$dbhusers;
731
732 $mail = lc($mail);
733
734 my $res = {};
735
736 my $cuid;
737 $dbhmails->get($mail, $cuid);
738 return undef if !$cuid;
739
740 my $rdata;
741 $dbhusers->get($cuid, $rdata);
742 return undef if !$rdata;
743
744 my ($pmail, $account, $dn) = unpack('n/a* n/a* n/a*', $rdata);
745
746 $res->{dn} = $dn;
747 $res->{account} = $account;
748 $res->{pmail} = $pmail;
749
750 if ($scan) {
751 my $key = 0 ;
752 my $value = "" ;
753 my $status = $dbhmails->seq($key, $value, R_FIRST());
754 my $mails;
755
756 while ($status == 0) {
757 push @$mails, $key if $value == $cuid;
758 $status = $dbhmails->seq($key, $value, R_NEXT());
759 }
760 $res->{mails} = $mails;
761 }
762
763 return $res;
764 }
765
766 1;