PMG/LDAPCache.pm

   1 package PMG::LDAPCache;
   2
   3 use strict;
   4 use warnings;
   5 use File::Path;
   6 use LockFile::Simple;
   7 use Net::LDAP;
   8 use Net::LDAP::Control::Paged;
   9 use Net::LDAP::Constant qw (LDAP_CONTROL_PAGED);
  10 use DB_File;
  11
  12 use PVE::SafeSyslog;
  13
  14 use PMG::Utils;
  15
  16 $DB_HASH->{'cachesize'} = 10000;
  17 $DB_RECNO->{'cachesize'} = 10000;
  18 $DB_BTREE->{'cachesize'} = 10000;
  19 $DB_BTREE->{'flags'} = R_DUP ;
  20
  21 my $cachedir = '/var/lib/pmg';
  22
  23 my $last_atime = {};
  24 my $ldapcache = {};
  25
  26 # DB Description
  27 #
  28 # users      (hash): UID -> pmail, account, DN
  29 # dnames     (hash): DN -> UID
  30 # accounts   (hash): account -> UID
  31 # mail       (hash): mail -> UID
  32 # groups     (hash): group -> GID
  33 # memberof  (btree): UID -> GID
  34 #
  35 my @dbs = ('users', 'dnames', 'groups', 'mails', 'accounts', 'memberof');
  36
  37 sub new {
  38     my ($self, %args) = @_;
  39
  40     my $type   = ref($self) || $self;
  41
  42     die "undefined ldap id" if !$args{id};
  43
  44     my $id = $args{id};
  45
  46     if ($ldapcache->{$id}) {
  47         $self = $ldapcache->{$id};
  48     } else {
  49         $ldapcache->{$id} = $self = bless {}, $type;
  50         $self->{id} = $id;
  51     }
  52
  53     if (!$args{mailattr}) {
  54         $args{mailattr} = "mail, userPrincipalName, proxyAddresses, othermailbox";
  55     }
  56     $args{mailattr} =~ s/[\,\;]/ /g;
  57     $args{mailattr} =~ s/\s+/,/g;
  58
  59     if ($args{mode} && ($args{mode} eq 'ldap' ||  $args{mode} eq 'ldaps')) {
  60         $self->{mode} = $args{mode};
  61     } else {
  62         $self->{mode} = 'ldap';
  63     }
  64
  65     $self->{accountattr} = $args{accountattr} || 'sAMAccountName';
  66     @{$self->{mailattr}} = split(/,/, $args{mailattr});
  67     $self->{server1} = $args{server1};
  68     $self->{server2} = $args{server2};
  69     $self->{binddn} = $args{binddn};
  70     $self->{bindpw} = $args{bindpw};
  71     $self->{basedn} = $args{basedn};
  72     $self->{port} = $args{port};
  73     $self->{groupbasedn} = $args{groupbasedn};
  74     $self->{filter} = $args{filter};
  75
  76     if ($args{syncmode} == 1) {
  77         # read local data only
  78         $self->{errors} = '';
  79         $self->loadcache();
  80         return $self;
  81     }
  82
  83     return $self if !($args{server1});
  84
  85     if ($args{syncmode} == 2) {
  86         # force sync
  87         $self->loaddata(1);
  88     } else {
  89         $self->loaddata();
  90     }
  91
  92     return $self;
  93 }
  94
  95 sub lockdir {
  96     my ($id) = @_;
  97
  98     my $dir = "$cachedir/ldapdb_$id";
  99     my $scheme = LockFile::Simple->make(
 100         -warn => 0, -stale => 1, -autoclean => 1);
 101     my $lock = $scheme->lock($dir);
 102
 103     return $lock;
 104 }
 105
 106 sub delete {
 107     my ($class, $id) = @_;
 108
 109     if (my $lock = lockdir($id)) {
 110         delete $ldapcache->{$id};
 111         delete $last_atime->{$id};
 112         my $dir = "$cachedir/ldapdb_$id";
 113         rmtree $dir;
 114         $lock->release;
 115     } else {
 116         syslog('err' , "can't lock ldap database '$id'");
 117     }
 118 }
 119
 120 sub update {
 121     my ($self, $syncmode) = @_;
 122
 123     if ($syncmode == 1) {
 124         # read local data only
 125         $self->{errors} = '';
 126         $self->loadcache();
 127     } elsif ($syncmode == 2) {
 128         # force sync
 129         $self->loaddata(1);
 130     } else {
 131         $self->loaddata();
 132     }
 133 }
 134
 135 sub queryusers {
 136     my ($self, $ldap) = @_;
 137
 138     my $filter = '(|';
 139     foreach my $attr (@{$self->{mailattr}}) {
 140         $filter .= "($attr=*)";
 141     }
 142     $filter .= ')';
 143
 144     if ($self->{filter}) {
 145         my $tmp = $self->{filter};
 146         $tmp = "($tmp)" if $tmp !~ m/^\(.*\)$/;
 147
 148         $filter = "(&${filter}${tmp})";
 149     }
 150
 151     my $page = Net::LDAP::Control::Paged->new(size => 900);
 152
 153     my @args = (
 154         base     => $self->{basedn},
 155         scope    => "subtree",
 156         filter   => $filter,
 157         control  => [ $page ],
 158         attrs  => [ @{$self->{mailattr}}, $self->{accountattr}, 'memberOf' ]
 159         );
 160
 161     my $cookie;
 162
 163     while(1) {
 164
 165         my $mesg = $ldap->search(@args);
 166
 167         # stop on error
 168         if ($mesg->code)  {
 169             my $err = "ldap user search error: " . $mesg->error;
 170             $self->{errors} .= "$err\n";
 171             syslog('err', $err);
 172             last;
 173         }
 174
 175         #foreach my $entry ($mesg->entries) { $entry->dump; }
 176         foreach my $entry ($mesg->entries) {
 177             my $dn = $entry->dn;
 178
 179             my $umails = {};
 180             my $pmail;
 181
 182             foreach my $attr (@{$self->{mailattr}}) {
 183                 foreach my $mail ($entry->get_value($attr)) {
 184                     $mail = lc($mail);
 185                     # Test if the Line starts with one of the following lines:
 186                     # proxyAddresses: [smtp|SMTP]:
 187                     # and also discard this starting string, so that $mail is only the
 188                     # address without any other characters...
 189
 190                     $mail =~ s/^(smtp|SMTP)[\:\$]//gs;
 191
 192                     if ($mail !~ m/[\{\}\\\/]/ && $mail =~ m/^\S+\@\S+$/) {
 193                         $umails->{$mail} = 1;
 194                         $pmail = $mail if !$pmail;
 195                     }
 196                 }
 197             }
 198             my $addresses = [ keys %$umails ];
 199
 200             next if !$pmail; # account has no email addresses
 201
 202             my $cuid;
 203             $self->{dbstat}->{dnames}->{dbh}->get($dn, $cuid);
 204             if (!$cuid) {
 205                 $cuid = ++$self->{dbstat}->{dnames}->{idcount};
 206                 $self->{dbstat}->{dnames}->{dbh}->put($dn, $cuid);
 207             }
 208
 209             my $account = $entry->get_value($self->{accountattr});
 210             if ($account && ($account =~ m/^\S+$/s)) {
 211                 $account = lc($account);
 212                 $self->{dbstat}->{accounts}->{dbh}->put($account, $cuid);
 213             } else {
 214                 $account = '';
 215             }
 216
 217             my $data = pack('n/a* n/a* n/a*', $pmail, $account, $dn);
 218             $self->{dbstat}->{users}->{dbh}->put($cuid, $data);
 219
 220             foreach my $mail (@$addresses) {
 221                 $self->{dbstat}->{mails}->{dbh}->put($mail, $cuid);
 222             }
 223
 224             if (!$self->{groupbasedn}) {
 225                 my @groups = $entry->get_value('memberOf');
 226                 foreach my $group (@groups) {
 227                     my $cgid;
 228                     $self->{dbstat}->{groups}->{dbh}->get($group, $cgid);
 229                     if (!$cgid) {
 230                         $cgid = ++$self->{dbstat}->{groups}->{idcount};
 231                         $self->{dbstat}->{groups}->{dbh}->put($group, $cgid);
 232                     }
 233                     $self->{dbstat}->{memberof}->{dbh}->put($cuid, $cgid);
 234                 }
 235             }
 236         }
 237
 238         # Get cookie from paged control
 239         my ($resp) = $mesg->control(LDAP_CONTROL_PAGED) or last;
 240         $cookie = $resp->cookie or last;
 241
 242         # Set cookie in paged control
 243         $page->cookie($cookie);
 244     }
 245
 246
 247     if ($cookie) {
 248         # We had an abnormal exit, so let the server know we do not want any more
 249         $page->cookie($cookie);
 250         $page->size(0);
 251         $ldap->search(@args);
 252         my $err = "LDAP user query unsuccessful";
 253         $self->{errors} .= "$err\n";
 254         syslog('err', $err);
 255     }
 256 }
 257
 258 sub querygroups {
 259     my ($self, $ldap) = @_;
 260
 261     return undef if !$self->{groupbasedn};
 262
 263     my $filter = "(|(objectclass=group)(objectclass=univentionGroup))";
 264
 265     my $page = Net::LDAP::Control::Paged->new(size => 100);
 266
 267     my @args = ( base     => $self->{groupbasedn},
 268                  scope    => "subtree",
 269                  filter   => $filter,
 270                  control  => [ $page ],
 271                  attrs  => [ 'member', 'uniqueMember' ],
 272                  );
 273
 274     my $cookie;
 275     while(1) {
 276
 277         my $mesg = $ldap->search(@args);
 278
 279         # stop on error
 280         if ($mesg->code)  {
 281             my $err = "ldap group search error: " . $mesg->error;
 282             $self->{errors} .= "$err\n";
 283             syslog('err', $err);
 284             last;
 285         }
 286
 287         foreach my $entry ( $mesg->entries ) {
 288             my $group = $entry->dn;
 289             my @members = $entry->get_value('member');
 290             if (!scalar(@members)) {
 291                 @members = $entry->get_value('uniqueMember');
 292             }
 293             my $cgid;
 294             $self->{dbstat}->{groups}->{dbh}->get($group, $cgid);
 295             if (!$cgid) {
 296                 $cgid = ++$self->{dbstat}->{groups}->{idcount};
 297                 $self->{dbstat}->{groups}->{dbh}->put($group, $cgid);
 298             }
 299
 300             foreach my $m (@members) {
 301
 302                 my $cuid;
 303                 $self->{dbstat}->{dnames}->{dbh}->get($m, $cuid);
 304                 if (!$cuid) {
 305                     $cuid = ++$self->{dbstat}->{dnames}->{idcount};
 306                     $self->{dbstat}->{dnames}->{dbh}->put($m, $cuid);
 307                 }
 308
 309                 $self->{dbstat}->{memberof}->{dbh}->put($cuid, $cgid);
 310             }
 311         }
 312
 313         # Get cookie from paged control
 314         my ($resp) = $mesg->control(LDAP_CONTROL_PAGED) or last;
 315         $cookie = $resp->cookie or last;
 316
 317         # Set cookie in paged control
 318         $page->cookie($cookie);
 319     }
 320
 321     if ($cookie) {
 322         # We had an abnormal exit, so let the server know we do not want any more
 323         $page->cookie($cookie);
 324         $page->size(0);
 325         $ldap->search(@args);
 326         my $err = "LDAP group query unsuccessful";
 327         $self->{errors} .= "$err\n";
 328         syslog('err', $err);
 329     }
 330 }
 331
 332 sub ldap_connect {
 333     my ($self) = @_;
 334
 335     my $mode = $self->{mode};
 336     my $portstr = '';
 337     $portstr = ':' . $self->{port} if $self->{port};
 338
 339     my $serverstr = "$mode://$self->{server1}${portstr}/";
 340     my $ldap = Net::LDAP->new($serverstr);
 341     if(!$ldap && $self->{server2} && $self->{server2} ne '127.0.0.1')  {
 342         $serverstr = "$mode://$self->{server2}${portstr}/";
 343         $ldap = Net::LDAP->new($serverstr);
 344     }
 345
 346     return $ldap;
 347 }
 348
 349 sub ldap_connect_and_bind {
 350      my ($self) = @_;
 351
 352      my $ldap = $self->ldap_connect() ||
 353          die "Can't bind to ldap server '$self->{id}': $!\n";
 354
 355      my $mesg;
 356
 357      if ($self->{binddn}) {
 358          $mesg = $ldap->bind($self->{binddn}, password => $self->{bindpw});
 359      } else {
 360          $mesg = $ldap->bind(); # anonymous bind
 361      }
 362
 363      die "ldap bind failed: " . $mesg->error . "\n" if $mesg->code;
 364
 365      if (!$self->{basedn}) {
 366          my $root = $ldap->root_dse(attrs => [ 'defaultNamingContext' ]);
 367          $self->{basedn} = $root->get_value('defaultNamingContext');
 368      }
 369
 370      return $ldap;
 371 }
 372
 373 sub sync_database {
 374     my ($self) = @_;
 375
 376     my $dir = "ldapdb_" .  $self->{id};
 377     mkdir "$cachedir/$dir";
 378
 379     # open ldap connection
 380
 381     syslog('info', "syncing ldap database '$self->{id}'");
 382
 383     my $ldap;
 384
 385     eval { $ldap = $self->ldap_connect_and_bind(); };
 386     if (my $err = $@) {
 387         $self->{errors} .= "$err\n";
 388         syslog('err', $err);
 389         return;
 390     }
 391
 392     # open temporary database files
 393
 394     my $olddbh = {};
 395
 396     foreach my $db (@dbs) {
 397         $self->{dbstat}->{$db}->{tmpfilename} = "$cachedir/$dir/${db}_tmp$$.db";
 398         $olddbh->{$db} = $self->{dbstat}->{$db}->{dbh};
 399     }
 400
 401     my $error_cleanup = sub {
 402         # close and delete all files
 403         foreach my $db (@dbs) {
 404             undef $self->{dbstat}->{$db}->{dbh};
 405             unlink $self->{dbstat}->{$db}->{tmpfilename};
 406             $self->{dbstat}->{$db}->{dbh} = $olddbh->{$db};
 407         }
 408     };
 409
 410     eval {
 411         foreach my $db (@dbs) {
 412             my $filename = $self->{dbstat}->{$db}->{tmpfilename};
 413             $self->{dbstat}->{$db}->{idcount} = 0;
 414             unlink $filename;
 415
 416             if ($db eq 'memberof') {
 417                 $self->{dbstat}->{$db}->{dbh} =
 418                     tie (my %h,  'DB_File', $filename,
 419                          O_CREAT|O_RDWR, 0666, $DB_BTREE);
 420             } else {
 421                 $self->{dbstat}->{$db}->{dbh} =
 422                     tie (my %h,  'DB_File', $filename,
 423                          O_CREAT|O_RDWR, 0666, $DB_HASH);
 424             }
 425
 426             die "unable to open database file '$filename': $!\n"
 427                 if !$self->{dbstat}->{$db}->{dbh};
 428         }
 429     };
 430     if (my $err = $@) {
 431         $error_cleanup->();
 432         $self->{errors} .= $err;
 433         syslog('err', $err);
 434         return;
 435     }
 436
 437     $self->querygroups ($ldap) if $self->{groupbasedn};
 438
 439     $self->queryusers($ldap) if !$self->{errors};
 440
 441     $ldap->unbind;
 442
 443     if ($self->{errors}) {
 444         $error_cleanup->();
 445         return;
 446     }
 447
 448     my $lock = lockdir($self->{id});
 449
 450     if (!$lock) {
 451         my $err = "unable to get database lock for ldap database '$self->{id}'";
 452         $self->{errors} .= "$err\n";
 453         syslog('err', $err);
 454         $error_cleanup->();
 455         return;
 456     }
 457
 458     foreach my $db (@dbs) {
 459         my $filename = $self->{dbstat}->{$db}->{filename} =
 460             "$cachedir/$dir/${db}.db";
 461         $self->{dbstat}->{$db}->{dbh}->sync(); # flush everything
 462         rename $self->{dbstat}->{$db}->{tmpfilename}, $filename;
 463     }
 464
 465     $lock->release;
 466
 467     $last_atime->{$self->{id}} = time();
 468
 469     $self->{gcount} = $self->{dbstat}->{groups}->{idcount};
 470     $self->{ucount} = __count_entries($self->{dbstat}->{accounts}->{dbh});
 471     $self->{mcount} = __count_entries($self->{dbstat}->{mails}->{dbh});
 472
 473     syslog('info', "ldap sync '$self->{id}' successful ($self->{mcount})");
 474 }
 475
 476 sub __count_entries {
 477     my ($dbh) = @_;
 478
 479     return 0 if !$dbh;
 480
 481     my $key = 0 ;
 482     my $value = "" ;
 483     my $count = 0;
 484     my $status = $dbh->seq($key, $value, R_FIRST());
 485
 486     while ($status == 0) {
 487         $count++;
 488         $status = $dbh->seq($key, $value, R_NEXT());
 489     }
 490
 491     return $count;
 492 }
 493
 494 sub loadcache {
 495     my ($self, $try) = @_;
 496
 497     my $dir = "ldapdb_" .  $self->{id};
 498     mkdir "$cachedir/$dir";
 499
 500     my $filename = "$cachedir/$dir/mails.db";
 501
 502     return if $last_atime->{$self->{id}} &&
 503         PMG::Utils::file_older_than ($filename, $last_atime->{$self->{id}});
 504
 505     eval {
 506         foreach my $db (@dbs) {
 507             my $filename = $self->{dbstat}->{$db}->{filename} =
 508                 "$cachedir/$dir/${db}.db";
 509             $self->{dbstat}->{$db}->{idcount} = 0;
 510             if ($db eq 'memberof') {
 511                 $self->{dbstat}->{$db}->{dbh} =
 512                     tie (my %h,  'DB_File', $filename,
 513                          O_RDONLY, 0666, $DB_BTREE);
 514             } else {
 515                 $self->{dbstat}->{$db}->{dbh} =
 516                     tie (my %h,  'DB_File', $filename,
 517                          O_RDONLY, 0666, $DB_HASH);
 518             }
 519
 520             if (!$self->{dbstat}->{$db}->{dbh} && !$try) {
 521                 my $err = "ldap error - unable to open database file '$filename': $!";
 522                 $self->{errors} .= "$err\n";
 523                 syslog('err', $err) if !$self->{dbstat}->{$db}->{dbh};
 524             }
 525         }
 526     };
 527
 528     $last_atime->{$self->{id}} = time();
 529
 530     $self->{gcount} = __count_entries($self->{dbstat}->{groups}->{dbh});
 531     $self->{ucount} = __count_entries($self->{dbstat}->{accounts}->{dbh});
 532     $self->{mcount} = __count_entries($self->{dbstat}->{mails}->{dbh});
 533 }
 534
 535 sub loaddata {
 536     my ($self, $force) = @_;
 537
 538     $self->{errors} = '';
 539
 540     if (!$force) {
 541         # only sync if file is older than 1 hour
 542
 543         my $dir = "ldapdb_" .  $self->{id};
 544         mkdir "$cachedir/$dir";
 545         my $filename = "$cachedir/$dir/mails.db";
 546
 547         if (-e $filename &&
 548             !PMG::Utils::file_older_than($filename, time() - 3600)) {
 549             $self->loadcache();
 550             return;
 551         }
 552     }
 553
 554     $self->sync_database();
 555
 556     if ($self->{errors}) {
 557         $self->loadcache(1);
 558     }
 559 }
 560
 561 sub groups {
 562     my ($self) = @_;
 563
 564     my $dbh = $self->{dbstat}->{groups}->{dbh};
 565     return [] if !$dbh;
 566
 567     my $key = 0 ;
 568     my $value = "" ;
 569     my $status = $dbh->seq($key, $value, R_FIRST());
 570     my $keys;
 571
 572     while ($status == 0) {
 573         push @$keys, $key;
 574         $status = $dbh->seq($key, $value, R_NEXT());
 575     }
 576
 577     return $keys;
 578 }
 579
 580 sub mail_exists {
 581     my ($self, $mail) = @_;
 582
 583     my $dbh = $self->{dbstat}->{mails}->{dbh};
 584     return 0 if !$dbh;
 585
 586     $mail = lc($mail);
 587
 588     my $res;
 589     $dbh->get($mail, $res);
 590     return $res;
 591 }
 592
 593 sub account_exists {
 594     my ($self, $account) = @_;
 595
 596     my $dbh = $self->{dbstat}->{accounts}->{dbh};
 597     return 0 if !$dbh;
 598
 599     $account = lc($account);
 600
 601     my $res;
 602     $dbh->get($account, $res);
 603     return $res;
 604 }
 605
 606 sub account_has_address {
 607     my ($self, $account, $mail) = @_;
 608
 609     my $dbhmails = $self->{dbstat}->{mails}->{dbh};
 610     my $dbhaccounts = $self->{dbstat}->{accounts}->{dbh};
 611     return 0 if !$dbhmails || !$dbhaccounts;
 612
 613     $account = lc($account);
 614     $mail = lc($mail);
 615
 616     my $accid;
 617     $dbhaccounts->get($account, $accid);
 618     return 0 if !$accid;
 619
 620     my $mailid;
 621     $dbhmails->get($mail, $mailid);
 622     return 0 if !$mailid;
 623
 624     return ($accid == $mailid);
 625 }
 626
 627 sub user_in_group {
 628     my ($self, $mail, $group) = @_;
 629
 630     my $dbhmails = $self->{dbstat}->{mails}->{dbh};
 631     my $dbhgroups = $self->{dbstat}->{groups}->{dbh};
 632     my $dbhmemberof = $self->{dbstat}->{memberof}->{dbh};
 633
 634     return 0 if !$dbhmails || !$dbhgroups || !$dbhmemberof;
 635
 636     $mail = lc($mail);
 637
 638     my $cuid;
 639     $dbhmails->get($mail, $cuid);
 640     return 0 if !$cuid;
 641
 642     my $groupid;
 643     $dbhgroups->get($group, $groupid);
 644     return 0 if !$groupid;
 645
 646     my @gida  = $dbhmemberof->get_dup($cuid);
 647
 648     return grep { $_ eq $groupid } @gida;
 649 }
 650
 651 sub account_info {
 652     my ($self, $mail, $scan) = @_;
 653
 654     my $dbhmails = $self->{dbstat}->{mails}->{dbh};
 655     my $dbhusers = $self->{dbstat}->{users}->{dbh};
 656
 657     return undef if !$dbhmails || !$dbhusers;
 658
 659     $mail = lc($mail);
 660
 661     my $res = {};
 662
 663     my $cuid;
 664     $dbhmails->get($mail, $cuid);
 665     return undef if !$cuid;
 666
 667     my $rdata;
 668     $dbhusers->get($cuid, $rdata);
 669     return undef if !$rdata;
 670
 671     my ($pmail, $account, $dn) = unpack('n/a* n/a* n/a*', $rdata);
 672
 673     $res->{dn} = $dn;
 674     $res->{account} = $account;
 675     $res->{pmail} = $pmail;
 676
 677     if ($scan) {
 678         my $key = 0 ;
 679         my $value = "" ;
 680         my $status = $dbhmails->seq($key, $value, R_FIRST());
 681         my $mails;
 682
 683         while ($status == 0) {
 684             push @$mails, $key if $value == $cuid;
 685             $status = $dbhmails->seq($key, $value, R_NEXT());
 686         }
 687         $res->{mails} = $mails;
 688     }
 689
 690     return $res;
 691 }
 692
 693 1;