PMG/LDAPCache.pm

   1 package PMG::LDAPCache;
   2
   3 use strict;
   4 use warnings;
   5 use Carp;
   6 use File::Path;
   7 use LockFile::Simple;
   8 use Net::LDAP;
   9 use Net::LDAP::Control::Paged;
  10 use Net::LDAP::Constant qw (LDAP_CONTROL_PAGED);
  11 use DB_File;
  12
  13 use PVE::SafeSyslog;
  14
  15 use PMG::Utils;
  16
  17 $DB_HASH->{'cachesize'} = 10000;
  18 $DB_RECNO->{'cachesize'} = 10000;
  19 $DB_BTREE->{'cachesize'} = 10000;
  20 $DB_BTREE->{'flags'} = R_DUP ;
  21
  22 my $cachedir = '/var/lib/proxmox-mailgateway';
  23
  24 my $last_atime = {};
  25 my $ldapcache = {};
  26
  27 # DB Description
  28 #
  29 # users      (hash): UID -> pmail, account, DN
  30 # dnames     (hash): DN -> UID
  31 # accounts   (hash): account -> UID
  32 # mail       (hash): mail -> UID
  33 # groups     (hash): group -> GID
  34 # memberof  (btree): UID -> GID
  35 #
  36 my @dbs = ('users', 'dnames', 'groups', 'mails', 'accounts', 'memberof');
  37
  38 sub new {
  39     my ($self) = @_;
  40
  41     my $type   = ref($self) || $self;
  42
  43     my %args = @_;
  44
  45     die "undefine ldap id" if !$args{id};
  46
  47     my $id = $args{id};
  48     $id =~ s/^server_//;
  49
  50     if ($ldapcache->{$id}) {
  51         $self = $ldapcache->{$id};
  52     } else {
  53         $ldapcache->{$id} = $self = bless {}, $type;
  54         $self->{id} = $id;
  55     }
  56
  57     if (!$args{mailattr}) {
  58         $args{mailattr} = "mail, userPrincipalName, proxyAddresses, othermailbox";
  59     }
  60     $args{mailattr} =~ s/[\,\;]/ /g;
  61     $args{mailattr} =~ s/\s+/,/g;
  62
  63     if ($args{mode} && ($args{mode} eq 'ldap' ||  $args{mode} eq 'ldaps')) {
  64         $self->{mode} = $args{mode};
  65     } else {
  66         $self->{mode} = 'ldap';
  67     }
  68
  69     $self->{accountattr} = $args{accountattr} || 'sAMAccountName';
  70     @{$self->{mailattr}} = split(/,/, $args{mailattr});
  71     $self->{server1} = $args{server1};
  72     $self->{server2} = $args{server2};
  73     $self->{binddn} = $args{binddn};
  74     $self->{bindpw} = $args{bindpw};
  75     $self->{basedn} = $args{basedn};
  76     $self->{port} = $args{port};
  77     $self->{groupbasedn} = $args{groupbasedn};
  78     $self->{filter} = $args{filter};
  79
  80     if ($args{syncmode} == 1) {
  81         # read local data only
  82         $self->{errors} = '';
  83         $self->loadcache();
  84         return $self;
  85     }
  86
  87     return $self if !($args{server1});
  88
  89     if ($args{syncmode} == 2) {
  90         # force sync
  91         $self->loaddata(1);
  92     } else {
  93         $self->loaddata();
  94     }
  95
  96     return $self;
  97 }
  98
  99 sub lockdir {
 100     my ($id) = @_;
 101
 102     my $dir = "$cachedir/ldapdb_$id";
 103     my $scheme = LockFile::Simple->make(
 104         -warn => 0, -stale => 1, -autoclean => 1);
 105     my $lock = $scheme->lock($dir);
 106
 107     return $lock;
 108 }
 109
 110 sub delete {
 111     my ($class, $id) = @_;
 112
 113     $id =~ s/^server_//;
 114
 115     if (my $lock = lockdir($id)) {
 116         delete $ldapcache->{$id};
 117         delete $last_atime->{$id};
 118         my $dir = "$cachedir/ldapdb_$id";
 119         rmtree $dir;
 120         $lock->release;
 121     } else {
 122         syslog('err' , "can't lock ldap database '$id'");
 123     }
 124 }
 125
 126 sub update {
 127     my ($self, $syncmode) = @_;
 128
 129     if ($syncmode == 1) {
 130         # read local data only
 131         $self->{errors} = '';
 132         $self->loadcache();
 133     } elsif ($syncmode == 2) {
 134         # force sync
 135         $self->loaddata(1);
 136     } else {
 137         $self->loaddata();
 138     }
 139 }
 140
 141 sub queryusers {
 142     my ($self, $ldap) = @_;
 143
 144     my $filter = '(|';
 145     foreach my $attr (@{$self->{mailattr}}) {
 146         $filter .= "($attr=*)";
 147     }
 148     $filter .= ')';
 149
 150     if ($self->{filter}) {
 151         my $tmp = $self->{filter};
 152         $tmp = "($tmp)" if $tmp !~ m/^\(.*\)$/;
 153
 154         $filter = "(&${filter}${tmp})";
 155     }
 156
 157     my $page = Net::LDAP::Control::Paged->new(size => 900);
 158
 159     my @args = (
 160         base     => $self->{basedn},
 161         scope    => "subtree",
 162         filter   => $filter,
 163         control  => [ $page ],
 164         attrs  => [ @{$self->{mailattr}}, $self->{accountattr}, 'memberOf' ]
 165         );
 166
 167     my $cookie;
 168
 169     while(1) {
 170
 171         my $mesg = $ldap->search(@args);
 172
 173         # stop on error
 174         if ($mesg->code)  {
 175             my $err = "ldap user search error: " . $mesg->error;
 176             $self->{errors} .= "$err\n";
 177             syslog('err', $err);
 178             last;
 179         }
 180
 181         #foreach my $entry ($mesg->entries) { $entry->dump; }
 182         foreach my $entry ($mesg->entries) {
 183             my $dn = $entry->dn;
 184
 185             my $umails = {};
 186             my $pmail;
 187
 188             foreach my $attr (@{$self->{mailattr}}) {
 189                 foreach my $mail ($entry->get_value($attr)) {
 190                     $mail = lc($mail);
 191                     # Test if the Line starts with one of the following lines:
 192                     # proxyAddresses: [smtp|SMTP]:
 193                     # and also discard this starting string, so that $mail is only the
 194                     # address without any other characters...
 195
 196                     $mail =~ s/^(smtp|SMTP)[\:\$]//gs;
 197
 198                     if ($mail !~ m/[\{\}\\\/]/ && $mail =~ m/^\S+\@\S+$/) {
 199                         $umails->{$mail} = 1;
 200                         $pmail = $mail if !$pmail;
 201                     }
 202                 }
 203             }
 204             my $addresses = [ keys %$umails ];
 205
 206             next if !$pmail; # account has no email addresses
 207
 208             my $cuid;
 209             $self->{dbstat}->{dnames}->{dbh}->get($dn, $cuid);
 210             if (!$cuid) {
 211                 $cuid = ++$self->{dbstat}->{dnames}->{idcount};
 212                 $self->{dbstat}->{dnames}->{dbh}->put($dn, $cuid);
 213             }
 214
 215             my $account = $entry->get_value($self->{accountattr});
 216             if ($account && ($account =~ m/^\S+$/s)) {
 217                 $account = lc($account);
 218                 $self->{dbstat}->{accounts}->{dbh}->put($account, $cuid);
 219             } else {
 220                 $account = '';
 221             }
 222
 223             my $data = pack('n/a* n/a* n/a*', $pmail, $account, $dn);
 224             $self->{dbstat}->{users}->{dbh}->put($cuid, $data);
 225
 226             foreach my $mail (@$addresses) {
 227                 $self->{dbstat}->{mails}->{dbh}->put($mail, $cuid);
 228             }
 229
 230             if (!$self->{groupbasedn}) {
 231                 my @groups = $entry->get_value('memberOf');
 232                 foreach my $group (@groups) {
 233                     my $cgid;
 234                     $self->{dbstat}->{groups}->{dbh}->get($group, $cgid);
 235                     if (!$cgid) {
 236                         $cgid = ++$self->{dbstat}->{groups}->{idcount};
 237                         $self->{dbstat}->{groups}->{dbh}->put($group, $cgid);
 238                     }
 239                     $self->{dbstat}->{memberof}->{dbh}->put($cuid, $cgid);
 240                 }
 241             }
 242         }
 243
 244         # Get cookie from paged control
 245         my ($resp) = $mesg->control(LDAP_CONTROL_PAGED) or last;
 246         $cookie = $resp->cookie or last;
 247
 248         # Set cookie in paged control
 249         $page->cookie($cookie);
 250     }
 251
 252
 253     if ($cookie) {
 254         # We had an abnormal exit, so let the server know we do not want any more
 255         $page->cookie($cookie);
 256         $page->size(0);
 257         $ldap->search(@args);
 258         my $err = "LDAP user query unsuccessful";
 259         $self->{errors} .= "$err\n";
 260         syslog('err', $err);
 261     }
 262 }
 263
 264 sub querygroups {
 265     my ($self, $ldap) = @_;
 266
 267     return undef if !$self->{groupbasedn};
 268
 269     my $filter = "(objectclass=group)";
 270
 271     my $page = Net::LDAP::Control::Paged->new(size => 100);
 272
 273     my @args = ( base     => $self->{groupbasedn},
 274                  scope    => "subtree",
 275                  filter   => $filter,
 276                  control  => [ $page ],
 277                  attrs  => [ 'member' ]
 278                  );
 279
 280     my $cookie;
 281     while(1) {
 282
 283         my $mesg = $ldap->search(@args);
 284
 285         # stop on error
 286         if ($mesg->code)  {
 287             my $err = "ldap group search error: " . $mesg->error;
 288             $self->{errors} .= "$err\n";
 289             syslog('err', $err);
 290             last;
 291         }
 292
 293         foreach my $entry ( $mesg->entries ) {
 294             my $group = $entry->dn;
 295             my @members = $entry->get_value('member');
 296
 297             my $cgid;
 298             $self->{dbstat}->{groups}->{dbh}->get($group, $cgid);
 299             if (!$cgid) {
 300                 $cgid = ++$self->{dbstat}->{groups}->{idcount};
 301                 $self->{dbstat}->{groups}->{dbh}->put($group, $cgid);
 302             }
 303
 304             foreach my $m (@members) {
 305
 306                 my $cuid;
 307                 $self->{dbstat}->{dnames}->{dbh}->get($m, $cuid);
 308                 if (!$cuid) {
 309                     $cuid = ++$self->{dbstat}->{dnames}->{idcount};
 310                     $self->{dbstat}->{dnames}->{dbh}->put($m, $cuid);
 311                 }
 312
 313                 $self->{dbstat}->{memberof}->{dbh}->put($cuid, $cgid);
 314             }
 315         }
 316
 317         # Get cookie from paged control
 318         my ($resp) = $mesg->control(LDAP_CONTROL_PAGED) or last;
 319         $cookie = $resp->cookie or last;
 320
 321         # Set cookie in paged control
 322         $page->cookie($cookie);
 323     }
 324
 325     if ($cookie) {
 326         # We had an abnormal exit, so let the server know we do not want any more
 327         $page->cookie($cookie);
 328         $page->size(0);
 329         $ldap->search(@args);
 330         my $err = "LDAP group query unsuccessful";
 331         $self->{errors} .= "$err\n";
 332         syslog('err', $err);
 333     }
 334 }
 335
 336 sub ldap_connect {
 337     my ($self) = @_;
 338
 339     my $mode = $self->{mode};
 340     my $portstr = '';
 341     $portstr = ':' . $self->{port} if $self->{port};
 342
 343     my $serverstr = "$mode://$self->{server1}${portstr}/";
 344     my $ldap = Net::LDAP->new($serverstr);
 345     if(!$ldap && $self->{server2} && $self->{server2} ne '127.0.0.1')  {
 346         $serverstr = "$mode://$self->{server2}${portstr}/";
 347         $ldap = Net::LDAP->new($serverstr);
 348     }
 349
 350     return $ldap;
 351 }
 352
 353 sub sync_database {
 354     my ($self) = @_;
 355
 356     my $dir = "ldapdb_" .  $self->{id};
 357     mkdir "$cachedir/$dir";
 358
 359     # open ldap connection
 360
 361     syslog('info', "syncing ldap database '$self->{id}'");
 362
 363     my $ldap = $self->ldap_connect();
 364
 365     if (!$ldap) {
 366         my $err = "Can't bind to ldap server '$self->{id}': $!";
 367         $self->{errors} .= "$err\n";
 368         syslog('err', $err);
 369         return;
 370     }
 371
 372     my $mesg;
 373
 374     if ($self->{binddn}) {
 375         $mesg = $ldap->bind($self->{binddn}, password => $self->{bindpw});
 376     } else {
 377         $mesg = $ldap->bind(); # anonymous bind
 378     }
 379
 380     if ($mesg->code) {
 381         my $err = "ldap bind failed: " . $mesg->error;
 382         $self->{errors} .= "$err\n";
 383         syslog('err', $err);
 384         return;
 385     }
 386
 387     if (!$self->{basedn}) {
 388         my $root = $ldap->root_dse(attrs => [ 'defaultNamingContext' ]);
 389         $self->{basedn} = $root->get_value('defaultNamingContext');
 390     }
 391
 392     # open temporary database files
 393
 394     my $olddbh = {};
 395
 396     foreach my $db (@dbs) {
 397         $self->{dbstat}->{$db}->{tmpfilename} = "$cachedir/$dir/${db}_tmp$$.db";
 398         $olddbh->{$db} = $self->{dbstat}->{$db}->{dbh};
 399     }
 400
 401     eval {
 402         foreach my $db (@dbs) {
 403             my $filename = $self->{dbstat}->{$db}->{tmpfilename};
 404             $self->{dbstat}->{$db}->{idcount} = 0;
 405             unlink $filename;
 406
 407             if ($db eq 'memberof') {
 408                 $self->{dbstat}->{$db}->{dbh} =
 409                     tie (my %h,  'DB_File', $filename,
 410                          O_CREAT|O_RDWR, 0666, $DB_BTREE);
 411             } else {
 412                 $self->{dbstat}->{$db}->{dbh} =
 413                     tie (my %h,  'DB_File', $filename,
 414                          O_CREAT|O_RDWR, 0666, $DB_HASH);
 415             }
 416
 417             die "unable to open database file '$filename': $!\n"
 418                 if !$self->{dbstat}->{$db}->{dbh};
 419         }
 420     };
 421
 422     my $err = $@;
 423
 424     if ($err) {
 425         # close and delete all files
 426         foreach my $db (@dbs) {
 427             undef $self->{dbstat}->{$db}->{dbh};
 428             unlink $self->{dbstat}->{$db}->{tmpfilename};
 429             $self->{dbstat}->{$db}->{dbh} = $olddbh->{$db};
 430         }
 431         $self->{errors} .= $err;
 432         syslog('err', $err);
 433
 434         return;
 435     }
 436
 437     $self->querygroups ($ldap) if $self->{groupbasedn};
 438
 439     if (!$self->{errors}) {
 440         $self->queryusers($ldap);
 441     }
 442
 443     $ldap->unbind;
 444
 445     if ($self->{errors}) {
 446         # close and delete all files
 447         foreach my $db (@dbs) {
 448             undef $self->{dbstat}->{$db}->{dbh};
 449             unlink $self->{dbstat}->{$db}->{tmpfilename};
 450             $self->{dbstat}->{$db}->{dbh} = $olddbh->{$db};
 451         }
 452     } else {
 453
 454         my $lock = lockdir($self->{id});
 455
 456         if (!$lock) {
 457             my $err = "unable to get database lock for ldap database '$self->{id}'";
 458             $self->{errors} .= "$err\n";
 459             syslog('err', $err);
 460
 461             # close and delete all files
 462             foreach my $db (@dbs) {
 463                 undef $self->{dbstat}->{$db}->{dbh};
 464                 unlink $self->{dbstat}->{$db}->{tmpfilename};
 465                 $self->{dbstat}->{$db}->{dbh} = $olddbh->{$db};
 466             }
 467         } else {
 468             foreach my $db (@dbs) {
 469                 my $filename = $self->{dbstat}->{$db}->{filename} =
 470                     "$cachedir/$dir/${db}.db";
 471                 $self->{dbstat}->{$db}->{dbh}->sync(); # flush everything
 472                 rename $self->{dbstat}->{$db}->{tmpfilename}, $filename;
 473             }
 474
 475             $lock->release;
 476
 477             $last_atime->{$self->{id}} = time();
 478
 479             $self->{gcount} = $self->{dbstat}->{groups}->{idcount};
 480             $self->{ucount} = __count_entries($self->{dbstat}->{accounts}->{dbh});
 481             $self->{mcount} = __count_entries($self->{dbstat}->{mails}->{dbh});
 482         }
 483     }
 484 }
 485
 486 sub __count_entries {
 487     my ($dbh) = @_;
 488
 489     return 0 if !$dbh;
 490
 491     my $key = 0 ;
 492     my $value = "" ;
 493     my $count = 0;
 494     my $status = $dbh->seq($key, $value, R_FIRST());
 495
 496     while ($status == 0) {
 497         $count++;
 498         $status = $dbh->seq($key, $value, R_NEXT());
 499     }
 500
 501     return $count;
 502 }
 503
 504 sub loadcache {
 505     my ($self, $try) = @_;
 506
 507     my $dir = "ldapdb_" .  $self->{id};
 508     mkdir "$cachedir/$dir";
 509
 510     my $filename = "$cachedir/$dir/mails.db";
 511
 512     return if $last_atime->{$self->{id}} &&
 513         PMG::Utils::file_older_than ($filename, $last_atime->{$self->{id}});
 514
 515     eval {
 516         foreach my $db (@dbs) {
 517             my $filename = $self->{dbstat}->{$db}->{filename} =
 518                 "$cachedir/$dir/${db}.db";
 519             $self->{dbstat}->{$db}->{idcount} = 0;
 520             if ($db eq 'memberof') {
 521                 $self->{dbstat}->{$db}->{dbh} =
 522                     tie (my %h,  'DB_File', $filename,
 523                          O_RDONLY, 0666, $DB_BTREE);
 524             } else {
 525                 $self->{dbstat}->{$db}->{dbh} =
 526                     tie (my %h,  'DB_File', $filename,
 527                          O_RDONLY, 0666, $DB_HASH);
 528             }
 529
 530             if (!$self->{dbstat}->{$db}->{dbh} && !$try) {
 531                 my $err = "ldap error - unable to open database file '$filename': $!";
 532                 $self->{errors} .= "$err\n";
 533                 syslog('err', $err) if !$self->{dbstat}->{$db}->{dbh};
 534             }
 535         }
 536     };
 537
 538     $last_atime->{$self->{id}} = time();
 539
 540     $self->{gcount} = __count_entries($self->{dbstat}->{groups}->{dbh});
 541     $self->{ucount} = __count_entries($self->{dbstat}->{accounts}->{dbh});
 542     $self->{mcount} = __count_entries($self->{dbstat}->{mails}->{dbh});
 543 }
 544
 545 sub loaddata {
 546     my ($self, $force) = @_;
 547
 548     $self->{errors} = '';
 549
 550     if (!$force) {
 551         # only sync if file is older than 1 hour
 552
 553         my $dir = "ldapdb_" .  $self->{id};
 554         mkdir "$cachedir/$dir";
 555         my $filename = "$cachedir/$dir/mails.db";
 556
 557         if (-e $filename &&
 558             !PMG::Utils::file_older_than($filename, time() - 3600)) {
 559             $self->loadcache();
 560             return;
 561         }
 562     }
 563
 564     $self->sync_database();
 565
 566     if ($self->{errors}) {
 567         $self->loadcache(1);
 568     }
 569 }
 570
 571 sub groups {
 572     my ($self) = @_;
 573
 574     my $dbh = $self->{dbstat}->{groups}->{dbh};
 575     return [] if !$dbh;
 576
 577     my $key = 0 ;
 578     my $value = "" ;
 579     my $status = $dbh->seq($key, $value, R_FIRST());
 580     my $keys;
 581
 582     while ($status == 0) {
 583         push @$keys, $key;
 584         $status = $dbh->seq($key, $value, R_NEXT());
 585     }
 586
 587     return $keys;
 588 }
 589
 590 sub mail_exists {
 591     my ($self, $mail) = @_;
 592
 593     my $dbh = $self->{dbstat}->{mails}->{dbh};
 594     return 0 if !$dbh;
 595
 596     $mail = lc($mail);
 597
 598     my $res;
 599     $dbh->get($mail, $res);
 600     return $res;
 601 }
 602
 603 sub account_exists {
 604     my ($self, $account) = @_;
 605
 606     my $dbh = $self->{dbstat}->{accounts}->{dbh};
 607     return 0 if !$dbh;
 608
 609     $account = lc($account);
 610
 611     my $res;
 612     $dbh->get($account, $res);
 613     return $res;
 614 }
 615
 616 sub account_has_address {
 617     my ($self, $account, $mail) = @_;
 618
 619     my $dbhmails = $self->{dbstat}->{mails}->{dbh};
 620     my $dbhaccounts = $self->{dbstat}->{accounts}->{dbh};
 621     return 0 if !$dbhmails || !$dbhaccounts;
 622
 623     $account = lc($account);
 624     $mail = lc($mail);
 625
 626     my $accid;
 627     $dbhaccounts->get($account, $accid);
 628     return 0 if !$accid;
 629
 630     my $mailid;
 631     $dbhmails->get($mail, $mailid);
 632     return 0 if !$mailid;
 633
 634     return ($accid == $mailid);
 635 }
 636
 637 sub user_in_group {
 638     my ($self, $mail, $group) = @_;
 639
 640     my $dbhmails = $self->{dbstat}->{mails}->{dbh};
 641     my $dbhgroups = $self->{dbstat}->{groups}->{dbh};
 642     my $dbhmemberof = $self->{dbstat}->{memberof}->{dbh};
 643
 644     return 0 if !$dbhmails || !$dbhgroups || !$dbhmemberof;
 645
 646     $mail = lc($mail);
 647
 648     my $cuid;
 649     $dbhmails->get($mail, $cuid);
 650     return 0 if !$cuid;
 651
 652     my $groupid;
 653     $dbhgroups->get($group, $groupid);
 654     return 0 if !$groupid;
 655
 656     my @gida  = $dbhmemberof->get_dup($cuid);
 657
 658     return grep { $_ eq $groupid } @gida;
 659 }
 660
 661 sub account_info {
 662     my ($self, $mail, $scan) = @_;
 663
 664     my $dbhmails = $self->{dbstat}->{mails}->{dbh};
 665     my $dbhusers = $self->{dbstat}->{users}->{dbh};
 666
 667     return undef if !$dbhmails || !$dbhusers;
 668
 669     $mail = lc($mail);
 670
 671     my $res = {};
 672
 673     my $cuid;
 674     $dbhmails->get($mail, $cuid);
 675     return undef if !$cuid;
 676
 677     my $rdata;
 678     $dbhusers->get($cuid, $rdata);
 679     return undef if !$rdata;
 680
 681     my ($pmail, $account, $dn) = unpack('n/a* n/a* n/a*', $rdata);
 682
 683     $res->{dn} = $dn;
 684     $res->{account} = $account;
 685     $res->{pmail} = $pmail;
 686
 687     if ($scan) {
 688         my $key = 0 ;
 689         my $value = "" ;
 690         my $status = $dbhmails->seq($key, $value, R_FIRST());
 691         my $mails;
 692
 693         while ($status == 0) {
 694             push @$mails, $key if $value == $cuid;
 695             $status = $dbhmails->seq($key, $value, R_NEXT());
 696         }
 697         $res->{mails} = $mails;
 698     }
 699
 700     return $res;
 701 }
 702
 703 1;