PMG/LDAPCache.pm

   1 package PMG::LDAPCache;
   2
   3 use strict;
   4 use warnings;
   5 use Carp;
   6 use File::Path;
   7 use LockFile::Simple;
   8 use Net::LDAP;
   9 use Net::LDAP::Control::Paged;
  10 use Net::LDAP::Constant qw (LDAP_CONTROL_PAGED);
  11 use DB_File;
  12
  13 use PVE::SafeSyslog;
  14
  15 use PMG::Utils;
  16
  17 $DB_HASH->{'cachesize'} = 10000;
  18 $DB_RECNO->{'cachesize'} = 10000;
  19 $DB_BTREE->{'cachesize'} = 10000;
  20 $DB_BTREE->{'flags'} = R_DUP ;
  21
  22 my $cachedir = '/var/lib/proxmox-mailgateway';
  23
  24 my $last_atime = {};
  25 my $ldapcache = {};
  26
  27 # DB Description
  28 #
  29 # users      (hash): UID -> pmail, account, DN
  30 # dnames     (hash): DN -> UID
  31 # accounts   (hash): account -> UID
  32 # mail       (hash): mail -> UID
  33 # groups     (hash): group -> GID
  34 # memberof  (btree): UID -> GID
  35 #
  36 my @dbs = ('users', 'dnames', 'groups', 'mails', 'accounts', 'memberof');
  37
  38 sub new {
  39     my ($self, %args) = @_;
  40
  41     my $type   = ref($self) || $self;
  42
  43     die "undefined ldap id" if !$args{id};
  44
  45     my $id = $args{id};
  46
  47     if ($ldapcache->{$id}) {
  48         $self = $ldapcache->{$id};
  49     } else {
  50         $ldapcache->{$id} = $self = bless {}, $type;
  51         $self->{id} = $id;
  52     }
  53
  54     if (!$args{mailattr}) {
  55         $args{mailattr} = "mail, userPrincipalName, proxyAddresses, othermailbox";
  56     }
  57     $args{mailattr} =~ s/[\,\;]/ /g;
  58     $args{mailattr} =~ s/\s+/,/g;
  59
  60     if ($args{mode} && ($args{mode} eq 'ldap' ||  $args{mode} eq 'ldaps')) {
  61         $self->{mode} = $args{mode};
  62     } else {
  63         $self->{mode} = 'ldap';
  64     }
  65
  66     $self->{accountattr} = $args{accountattr} || 'sAMAccountName';
  67     @{$self->{mailattr}} = split(/,/, $args{mailattr});
  68     $self->{server1} = $args{server1};
  69     $self->{server2} = $args{server2};
  70     $self->{binddn} = $args{binddn};
  71     $self->{bindpw} = $args{bindpw};
  72     $self->{basedn} = $args{basedn};
  73     $self->{port} = $args{port};
  74     $self->{groupbasedn} = $args{groupbasedn};
  75     $self->{filter} = $args{filter};
  76
  77     if ($args{syncmode} == 1) {
  78         # read local data only
  79         $self->{errors} = '';
  80         $self->loadcache();
  81         return $self;
  82     }
  83
  84     return $self if !($args{server1});
  85
  86     if ($args{syncmode} == 2) {
  87         # force sync
  88         $self->loaddata(1);
  89     } else {
  90         $self->loaddata();
  91     }
  92
  93     return $self;
  94 }
  95
  96 sub lockdir {
  97     my ($id) = @_;
  98
  99     my $dir = "$cachedir/ldapdb_$id";
 100     my $scheme = LockFile::Simple->make(
 101         -warn => 0, -stale => 1, -autoclean => 1);
 102     my $lock = $scheme->lock($dir);
 103
 104     return $lock;
 105 }
 106
 107 sub delete {
 108     my ($class, $id) = @_;
 109
 110     if (my $lock = lockdir($id)) {
 111         delete $ldapcache->{$id};
 112         delete $last_atime->{$id};
 113         my $dir = "$cachedir/ldapdb_$id";
 114         rmtree $dir;
 115         $lock->release;
 116     } else {
 117         syslog('err' , "can't lock ldap database '$id'");
 118     }
 119 }
 120
 121 sub update {
 122     my ($self, $syncmode) = @_;
 123
 124     if ($syncmode == 1) {
 125         # read local data only
 126         $self->{errors} = '';
 127         $self->loadcache();
 128     } elsif ($syncmode == 2) {
 129         # force sync
 130         $self->loaddata(1);
 131     } else {
 132         $self->loaddata();
 133     }
 134 }
 135
 136 sub queryusers {
 137     my ($self, $ldap) = @_;
 138
 139     my $filter = '(|';
 140     foreach my $attr (@{$self->{mailattr}}) {
 141         $filter .= "($attr=*)";
 142     }
 143     $filter .= ')';
 144
 145     if ($self->{filter}) {
 146         my $tmp = $self->{filter};
 147         $tmp = "($tmp)" if $tmp !~ m/^\(.*\)$/;
 148
 149         $filter = "(&${filter}${tmp})";
 150     }
 151
 152     my $page = Net::LDAP::Control::Paged->new(size => 900);
 153
 154     my @args = (
 155         base     => $self->{basedn},
 156         scope    => "subtree",
 157         filter   => $filter,
 158         control  => [ $page ],
 159         attrs  => [ @{$self->{mailattr}}, $self->{accountattr}, 'memberOf' ]
 160         );
 161
 162     my $cookie;
 163
 164     while(1) {
 165
 166         my $mesg = $ldap->search(@args);
 167
 168         # stop on error
 169         if ($mesg->code)  {
 170             my $err = "ldap user search error: " . $mesg->error;
 171             $self->{errors} .= "$err\n";
 172             syslog('err', $err);
 173             last;
 174         }
 175
 176         #foreach my $entry ($mesg->entries) { $entry->dump; }
 177         foreach my $entry ($mesg->entries) {
 178             my $dn = $entry->dn;
 179
 180             my $umails = {};
 181             my $pmail;
 182
 183             foreach my $attr (@{$self->{mailattr}}) {
 184                 foreach my $mail ($entry->get_value($attr)) {
 185                     $mail = lc($mail);
 186                     # Test if the Line starts with one of the following lines:
 187                     # proxyAddresses: [smtp|SMTP]:
 188                     # and also discard this starting string, so that $mail is only the
 189                     # address without any other characters...
 190
 191                     $mail =~ s/^(smtp|SMTP)[\:\$]//gs;
 192
 193                     if ($mail !~ m/[\{\}\\\/]/ && $mail =~ m/^\S+\@\S+$/) {
 194                         $umails->{$mail} = 1;
 195                         $pmail = $mail if !$pmail;
 196                     }
 197                 }
 198             }
 199             my $addresses = [ keys %$umails ];
 200
 201             next if !$pmail; # account has no email addresses
 202
 203             my $cuid;
 204             $self->{dbstat}->{dnames}->{dbh}->get($dn, $cuid);
 205             if (!$cuid) {
 206                 $cuid = ++$self->{dbstat}->{dnames}->{idcount};
 207                 $self->{dbstat}->{dnames}->{dbh}->put($dn, $cuid);
 208             }
 209
 210             my $account = $entry->get_value($self->{accountattr});
 211             if ($account && ($account =~ m/^\S+$/s)) {
 212                 $account = lc($account);
 213                 $self->{dbstat}->{accounts}->{dbh}->put($account, $cuid);
 214             } else {
 215                 $account = '';
 216             }
 217
 218             my $data = pack('n/a* n/a* n/a*', $pmail, $account, $dn);
 219             $self->{dbstat}->{users}->{dbh}->put($cuid, $data);
 220
 221             foreach my $mail (@$addresses) {
 222                 $self->{dbstat}->{mails}->{dbh}->put($mail, $cuid);
 223             }
 224
 225             if (!$self->{groupbasedn}) {
 226                 my @groups = $entry->get_value('memberOf');
 227                 foreach my $group (@groups) {
 228                     my $cgid;
 229                     $self->{dbstat}->{groups}->{dbh}->get($group, $cgid);
 230                     if (!$cgid) {
 231                         $cgid = ++$self->{dbstat}->{groups}->{idcount};
 232                         $self->{dbstat}->{groups}->{dbh}->put($group, $cgid);
 233                     }
 234                     $self->{dbstat}->{memberof}->{dbh}->put($cuid, $cgid);
 235                 }
 236             }
 237         }
 238
 239         # Get cookie from paged control
 240         my ($resp) = $mesg->control(LDAP_CONTROL_PAGED) or last;
 241         $cookie = $resp->cookie or last;
 242
 243         # Set cookie in paged control
 244         $page->cookie($cookie);
 245     }
 246
 247
 248     if ($cookie) {
 249         # We had an abnormal exit, so let the server know we do not want any more
 250         $page->cookie($cookie);
 251         $page->size(0);
 252         $ldap->search(@args);
 253         my $err = "LDAP user query unsuccessful";
 254         $self->{errors} .= "$err\n";
 255         syslog('err', $err);
 256     }
 257 }
 258
 259 sub querygroups {
 260     my ($self, $ldap) = @_;
 261
 262     return undef if !$self->{groupbasedn};
 263
 264     my $filter = "(objectclass=group)";
 265
 266     my $page = Net::LDAP::Control::Paged->new(size => 100);
 267
 268     my @args = ( base     => $self->{groupbasedn},
 269                  scope    => "subtree",
 270                  filter   => $filter,
 271                  control  => [ $page ],
 272                  attrs  => [ 'member' ]
 273                  );
 274
 275     my $cookie;
 276     while(1) {
 277
 278         my $mesg = $ldap->search(@args);
 279
 280         # stop on error
 281         if ($mesg->code)  {
 282             my $err = "ldap group search error: " . $mesg->error;
 283             $self->{errors} .= "$err\n";
 284             syslog('err', $err);
 285             last;
 286         }
 287
 288         foreach my $entry ( $mesg->entries ) {
 289             my $group = $entry->dn;
 290             my @members = $entry->get_value('member');
 291
 292             my $cgid;
 293             $self->{dbstat}->{groups}->{dbh}->get($group, $cgid);
 294             if (!$cgid) {
 295                 $cgid = ++$self->{dbstat}->{groups}->{idcount};
 296                 $self->{dbstat}->{groups}->{dbh}->put($group, $cgid);
 297             }
 298
 299             foreach my $m (@members) {
 300
 301                 my $cuid;
 302                 $self->{dbstat}->{dnames}->{dbh}->get($m, $cuid);
 303                 if (!$cuid) {
 304                     $cuid = ++$self->{dbstat}->{dnames}->{idcount};
 305                     $self->{dbstat}->{dnames}->{dbh}->put($m, $cuid);
 306                 }
 307
 308                 $self->{dbstat}->{memberof}->{dbh}->put($cuid, $cgid);
 309             }
 310         }
 311
 312         # Get cookie from paged control
 313         my ($resp) = $mesg->control(LDAP_CONTROL_PAGED) or last;
 314         $cookie = $resp->cookie or last;
 315
 316         # Set cookie in paged control
 317         $page->cookie($cookie);
 318     }
 319
 320     if ($cookie) {
 321         # We had an abnormal exit, so let the server know we do not want any more
 322         $page->cookie($cookie);
 323         $page->size(0);
 324         $ldap->search(@args);
 325         my $err = "LDAP group query unsuccessful";
 326         $self->{errors} .= "$err\n";
 327         syslog('err', $err);
 328     }
 329 }
 330
 331 sub ldap_connect {
 332     my ($self) = @_;
 333
 334     my $mode = $self->{mode};
 335     my $portstr = '';
 336     $portstr = ':' . $self->{port} if $self->{port};
 337
 338     my $serverstr = "$mode://$self->{server1}${portstr}/";
 339     my $ldap = Net::LDAP->new($serverstr);
 340     if(!$ldap && $self->{server2} && $self->{server2} ne '127.0.0.1')  {
 341         $serverstr = "$mode://$self->{server2}${portstr}/";
 342         $ldap = Net::LDAP->new($serverstr);
 343     }
 344
 345     return $ldap;
 346 }
 347
 348 sub sync_database {
 349     my ($self) = @_;
 350
 351     my $dir = "ldapdb_" .  $self->{id};
 352     mkdir "$cachedir/$dir";
 353
 354     # open ldap connection
 355
 356     syslog('info', "syncing ldap database '$self->{id}'");
 357
 358     my $ldap = $self->ldap_connect();
 359
 360     if (!$ldap) {
 361         my $err = "Can't bind to ldap server '$self->{id}': $!";
 362         $self->{errors} .= "$err\n";
 363         syslog('err', $err);
 364         return;
 365     }
 366
 367     my $mesg;
 368
 369     if ($self->{binddn}) {
 370         $mesg = $ldap->bind($self->{binddn}, password => $self->{bindpw});
 371     } else {
 372         $mesg = $ldap->bind(); # anonymous bind
 373     }
 374
 375     if ($mesg->code) {
 376         my $err = "ldap bind failed: " . $mesg->error;
 377         $self->{errors} .= "$err\n";
 378         syslog('err', $err);
 379         return;
 380     }
 381
 382     if (!$self->{basedn}) {
 383         my $root = $ldap->root_dse(attrs => [ 'defaultNamingContext' ]);
 384         $self->{basedn} = $root->get_value('defaultNamingContext');
 385     }
 386
 387     # open temporary database files
 388
 389     my $olddbh = {};
 390
 391     foreach my $db (@dbs) {
 392         $self->{dbstat}->{$db}->{tmpfilename} = "$cachedir/$dir/${db}_tmp$$.db";
 393         $olddbh->{$db} = $self->{dbstat}->{$db}->{dbh};
 394     }
 395
 396     eval {
 397         foreach my $db (@dbs) {
 398             my $filename = $self->{dbstat}->{$db}->{tmpfilename};
 399             $self->{dbstat}->{$db}->{idcount} = 0;
 400             unlink $filename;
 401
 402             if ($db eq 'memberof') {
 403                 $self->{dbstat}->{$db}->{dbh} =
 404                     tie (my %h,  'DB_File', $filename,
 405                          O_CREAT|O_RDWR, 0666, $DB_BTREE);
 406             } else {
 407                 $self->{dbstat}->{$db}->{dbh} =
 408                     tie (my %h,  'DB_File', $filename,
 409                          O_CREAT|O_RDWR, 0666, $DB_HASH);
 410             }
 411
 412             die "unable to open database file '$filename': $!\n"
 413                 if !$self->{dbstat}->{$db}->{dbh};
 414         }
 415     };
 416
 417     my $err = $@;
 418
 419     if ($err) {
 420         # close and delete all files
 421         foreach my $db (@dbs) {
 422             undef $self->{dbstat}->{$db}->{dbh};
 423             unlink $self->{dbstat}->{$db}->{tmpfilename};
 424             $self->{dbstat}->{$db}->{dbh} = $olddbh->{$db};
 425         }
 426         $self->{errors} .= $err;
 427         syslog('err', $err);
 428
 429         return;
 430     }
 431
 432     $self->querygroups ($ldap) if $self->{groupbasedn};
 433
 434     if (!$self->{errors}) {
 435         $self->queryusers($ldap);
 436     }
 437
 438     $ldap->unbind;
 439
 440     if ($self->{errors}) {
 441         # close and delete all files
 442         foreach my $db (@dbs) {
 443             undef $self->{dbstat}->{$db}->{dbh};
 444             unlink $self->{dbstat}->{$db}->{tmpfilename};
 445             $self->{dbstat}->{$db}->{dbh} = $olddbh->{$db};
 446         }
 447     } else {
 448
 449         my $lock = lockdir($self->{id});
 450
 451         if (!$lock) {
 452             my $err = "unable to get database lock for ldap database '$self->{id}'";
 453             $self->{errors} .= "$err\n";
 454             syslog('err', $err);
 455
 456             # close and delete all files
 457             foreach my $db (@dbs) {
 458                 undef $self->{dbstat}->{$db}->{dbh};
 459                 unlink $self->{dbstat}->{$db}->{tmpfilename};
 460                 $self->{dbstat}->{$db}->{dbh} = $olddbh->{$db};
 461             }
 462         } else {
 463             foreach my $db (@dbs) {
 464                 my $filename = $self->{dbstat}->{$db}->{filename} =
 465                     "$cachedir/$dir/${db}.db";
 466                 $self->{dbstat}->{$db}->{dbh}->sync(); # flush everything
 467                 rename $self->{dbstat}->{$db}->{tmpfilename}, $filename;
 468             }
 469
 470             $lock->release;
 471
 472             $last_atime->{$self->{id}} = time();
 473
 474             $self->{gcount} = $self->{dbstat}->{groups}->{idcount};
 475             $self->{ucount} = __count_entries($self->{dbstat}->{accounts}->{dbh});
 476             $self->{mcount} = __count_entries($self->{dbstat}->{mails}->{dbh});
 477         }
 478     }
 479 }
 480
 481 sub __count_entries {
 482     my ($dbh) = @_;
 483
 484     return 0 if !$dbh;
 485
 486     my $key = 0 ;
 487     my $value = "" ;
 488     my $count = 0;
 489     my $status = $dbh->seq($key, $value, R_FIRST());
 490
 491     while ($status == 0) {
 492         $count++;
 493         $status = $dbh->seq($key, $value, R_NEXT());
 494     }
 495
 496     return $count;
 497 }
 498
 499 sub loadcache {
 500     my ($self, $try) = @_;
 501
 502     my $dir = "ldapdb_" .  $self->{id};
 503     mkdir "$cachedir/$dir";
 504
 505     my $filename = "$cachedir/$dir/mails.db";
 506
 507     return if $last_atime->{$self->{id}} &&
 508         PMG::Utils::file_older_than ($filename, $last_atime->{$self->{id}});
 509
 510     eval {
 511         foreach my $db (@dbs) {
 512             my $filename = $self->{dbstat}->{$db}->{filename} =
 513                 "$cachedir/$dir/${db}.db";
 514             $self->{dbstat}->{$db}->{idcount} = 0;
 515             if ($db eq 'memberof') {
 516                 $self->{dbstat}->{$db}->{dbh} =
 517                     tie (my %h,  'DB_File', $filename,
 518                          O_RDONLY, 0666, $DB_BTREE);
 519             } else {
 520                 $self->{dbstat}->{$db}->{dbh} =
 521                     tie (my %h,  'DB_File', $filename,
 522                          O_RDONLY, 0666, $DB_HASH);
 523             }
 524
 525             if (!$self->{dbstat}->{$db}->{dbh} && !$try) {
 526                 my $err = "ldap error - unable to open database file '$filename': $!";
 527                 $self->{errors} .= "$err\n";
 528                 syslog('err', $err) if !$self->{dbstat}->{$db}->{dbh};
 529             }
 530         }
 531     };
 532
 533     $last_atime->{$self->{id}} = time();
 534
 535     $self->{gcount} = __count_entries($self->{dbstat}->{groups}->{dbh});
 536     $self->{ucount} = __count_entries($self->{dbstat}->{accounts}->{dbh});
 537     $self->{mcount} = __count_entries($self->{dbstat}->{mails}->{dbh});
 538 }
 539
 540 sub loaddata {
 541     my ($self, $force) = @_;
 542
 543     $self->{errors} = '';
 544
 545     if (!$force) {
 546         # only sync if file is older than 1 hour
 547
 548         my $dir = "ldapdb_" .  $self->{id};
 549         mkdir "$cachedir/$dir";
 550         my $filename = "$cachedir/$dir/mails.db";
 551
 552         if (-e $filename &&
 553             !PMG::Utils::file_older_than($filename, time() - 3600)) {
 554             $self->loadcache();
 555             return;
 556         }
 557     }
 558
 559     $self->sync_database();
 560
 561     if ($self->{errors}) {
 562         $self->loadcache(1);
 563     }
 564 }
 565
 566 sub groups {
 567     my ($self) = @_;
 568
 569     my $dbh = $self->{dbstat}->{groups}->{dbh};
 570     return [] if !$dbh;
 571
 572     my $key = 0 ;
 573     my $value = "" ;
 574     my $status = $dbh->seq($key, $value, R_FIRST());
 575     my $keys;
 576
 577     while ($status == 0) {
 578         push @$keys, $key;
 579         $status = $dbh->seq($key, $value, R_NEXT());
 580     }
 581
 582     return $keys;
 583 }
 584
 585 sub mail_exists {
 586     my ($self, $mail) = @_;
 587
 588     my $dbh = $self->{dbstat}->{mails}->{dbh};
 589     return 0 if !$dbh;
 590
 591     $mail = lc($mail);
 592
 593     my $res;
 594     $dbh->get($mail, $res);
 595     return $res;
 596 }
 597
 598 sub account_exists {
 599     my ($self, $account) = @_;
 600
 601     my $dbh = $self->{dbstat}->{accounts}->{dbh};
 602     return 0 if !$dbh;
 603
 604     $account = lc($account);
 605
 606     my $res;
 607     $dbh->get($account, $res);
 608     return $res;
 609 }
 610
 611 sub account_has_address {
 612     my ($self, $account, $mail) = @_;
 613
 614     my $dbhmails = $self->{dbstat}->{mails}->{dbh};
 615     my $dbhaccounts = $self->{dbstat}->{accounts}->{dbh};
 616     return 0 if !$dbhmails || !$dbhaccounts;
 617
 618     $account = lc($account);
 619     $mail = lc($mail);
 620
 621     my $accid;
 622     $dbhaccounts->get($account, $accid);
 623     return 0 if !$accid;
 624
 625     my $mailid;
 626     $dbhmails->get($mail, $mailid);
 627     return 0 if !$mailid;
 628
 629     return ($accid == $mailid);
 630 }
 631
 632 sub user_in_group {
 633     my ($self, $mail, $group) = @_;
 634
 635     my $dbhmails = $self->{dbstat}->{mails}->{dbh};
 636     my $dbhgroups = $self->{dbstat}->{groups}->{dbh};
 637     my $dbhmemberof = $self->{dbstat}->{memberof}->{dbh};
 638
 639     return 0 if !$dbhmails || !$dbhgroups || !$dbhmemberof;
 640
 641     $mail = lc($mail);
 642
 643     my $cuid;
 644     $dbhmails->get($mail, $cuid);
 645     return 0 if !$cuid;
 646
 647     my $groupid;
 648     $dbhgroups->get($group, $groupid);
 649     return 0 if !$groupid;
 650
 651     my @gida  = $dbhmemberof->get_dup($cuid);
 652
 653     return grep { $_ eq $groupid } @gida;
 654 }
 655
 656 sub account_info {
 657     my ($self, $mail, $scan) = @_;
 658
 659     my $dbhmails = $self->{dbstat}->{mails}->{dbh};
 660     my $dbhusers = $self->{dbstat}->{users}->{dbh};
 661
 662     return undef if !$dbhmails || !$dbhusers;
 663
 664     $mail = lc($mail);
 665
 666     my $res = {};
 667
 668     my $cuid;
 669     $dbhmails->get($mail, $cuid);
 670     return undef if !$cuid;
 671
 672     my $rdata;
 673     $dbhusers->get($cuid, $rdata);
 674     return undef if !$rdata;
 675
 676     my ($pmail, $account, $dn) = unpack('n/a* n/a* n/a*', $rdata);
 677
 678     $res->{dn} = $dn;
 679     $res->{account} = $account;
 680     $res->{pmail} = $pmail;
 681
 682     if ($scan) {
 683         my $key = 0 ;
 684         my $value = "" ;
 685         my $status = $dbhmails->seq($key, $value, R_FIRST());
 686         my $mails;
 687
 688         while ($status == 0) {
 689             push @$mails, $key if $value == $cuid;
 690             $status = $dbhmails->seq($key, $value, R_NEXT());
 691         }
 692         $res->{mails} = $mails;
 693     }
 694
 695     return $res;
 696 }
 697
 698 1;