src/journal/catalog.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2012 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <fcntl.h>
  23 #include <stdio.h>
  24 #include <unistd.h>
  25 #include <errno.h>
  26 #include <string.h>
  27 #include <sys/mman.h>
  28 #include <locale.h>
  29
  30 #include "util.h"
  31 #include "log.h"
  32 #include "sparse-endian.h"
  33 #include "sd-id128.h"
  34 #include "hashmap.h"
  35 #include "strv.h"
  36 #include "strbuf.h"
  37 #include "strxcpyx.h"
  38 #include "conf-files.h"
  39 #include "mkdir.h"
  40 #include "catalog.h"
  41 #include "siphash24.h"
  42
  43 const char * const catalog_file_dirs[] = {
  44         "/usr/local/lib/systemd/catalog/",
  45         "/usr/lib/systemd/catalog/",
  46         NULL
  47 };
  48
  49 #define CATALOG_SIGNATURE (uint8_t[]) { 'R', 'H', 'H', 'H', 'K', 'S', 'L', 'P' }
  50
  51 typedef struct CatalogHeader {
  52         uint8_t signature[8];  /* "RHHHKSLP" */
  53         le32_t compatible_flags;
  54         le32_t incompatible_flags;
  55         le64_t header_size;
  56         le64_t n_items;
  57         le64_t catalog_item_size;
  58 } CatalogHeader;
  59
  60 typedef struct CatalogItem {
  61         sd_id128_t id;
  62         char language[32];
  63         le64_t offset;
  64 } CatalogItem;
  65
  66 static unsigned long catalog_hash_func(const void *p, const uint8_t hash_key[HASH_KEY_SIZE]) {
  67         const CatalogItem *i = p;
  68         uint64_t u;
  69         size_t l, sz;
  70         void *v;
  71
  72         l = strlen(i->language);
  73         sz = sizeof(i->id) + l;
  74         v = alloca(sz);
  75
  76         memcpy(mempcpy(v, &i->id, sizeof(i->id)), i->language, l);
  77
  78         siphash24((uint8_t*) &u, v, sz, hash_key);
  79
  80         return (unsigned long) u;
  81 }
  82
  83 static int catalog_compare_func(const void *a, const void *b) {
  84         const CatalogItem *i = a, *j = b;
  85         unsigned k;
  86
  87         for (k = 0; k < ELEMENTSOF(j->id.bytes); k++) {
  88                  if (i->id.bytes[k] < j->id.bytes[k])
  89                         return -1;
  90                  if (i->id.bytes[k] > j->id.bytes[k])
  91                         return 1;
  92         }
  93
  94         return strcmp(i->language, j->language);
  95 }
  96
  97 const struct hash_ops catalog_hash_ops = {
  98         .hash = catalog_hash_func,
  99         .compare = catalog_compare_func
 100 };
 101
 102 static int finish_item(
 103                 Hashmap *h,
 104                 struct strbuf *sb,
 105                 sd_id128_t id,
 106                 const char *language,
 107                 const char *payload) {
 108
 109         ssize_t offset;
 110         _cleanup_free_ CatalogItem *i = NULL;
 111         int r;
 112
 113         assert(h);
 114         assert(sb);
 115         assert(payload);
 116
 117         offset = strbuf_add_string(sb, payload, strlen(payload));
 118         if (offset < 0)
 119                 return log_oom();
 120
 121         i = new0(CatalogItem, 1);
 122         if (!i)
 123                 return log_oom();
 124
 125         i->id = id;
 126         if (language) {
 127                 assert(strlen(language) > 1 && strlen(language) < 32);
 128                 strcpy(i->language, language);
 129         }
 130         i->offset = htole64((uint64_t) offset);
 131
 132         r = hashmap_put(h, i, i);
 133         if (r == -EEXIST) {
 134                 log_warning("Duplicate entry for " SD_ID128_FORMAT_STR ".%s, ignoring.",
 135                             SD_ID128_FORMAT_VAL(id), language ? language : "C");
 136                 return 0;
 137         } else if (r < 0)
 138                 return r;
 139
 140         i = NULL;
 141         return 0;
 142 }
 143
 144 int catalog_file_lang(const char* filename, char **lang) {
 145         char *beg, *end, *_lang;
 146
 147         end = endswith(filename, ".catalog");
 148         if (!end)
 149                 return 0;
 150
 151         beg = end - 1;
 152         while (beg > filename && *beg != '.' && *beg != '/' && end - beg < 32)
 153                 beg --;
 154
 155         if (*beg != '.' || end <= beg + 1)
 156                 return 0;
 157
 158         _lang = strndup(beg + 1, end - beg - 1);
 159         if (!_lang)
 160                 return -ENOMEM;
 161
 162         *lang = _lang;
 163         return 1;
 164 }
 165
 166 static int catalog_entry_lang(const char* filename, int line,
 167                               const char* t, const char* deflang, char **lang) {
 168         size_t c;
 169
 170         c = strlen(t);
 171         if (c == 0) {
 172                 log_error("[%s:%u] Language too short.", filename, line);
 173                 return -EINVAL;
 174         }
 175         if (c > 31) {
 176                 log_error("[%s:%u] language too long.", filename, line);
 177                 return -EINVAL;
 178         }
 179
 180         if (deflang) {
 181                 if (streq(t, deflang)) {
 182                         log_warning("[%s:%u] language specified unnecessarily",
 183                                     filename, line);
 184                         return 0;
 185                 } else
 186                         log_warning("[%s:%u] language differs from default for file",
 187                                     filename, line);
 188         }
 189
 190         *lang = strdup(t);
 191         if (!*lang)
 192                         return -ENOMEM;
 193
 194         return 0;
 195 }
 196
 197 int catalog_import_file(Hashmap *h, struct strbuf *sb, const char *path) {
 198         _cleanup_fclose_ FILE *f = NULL;
 199         _cleanup_free_ char *payload = NULL;
 200         unsigned n = 0;
 201         sd_id128_t id;
 202         _cleanup_free_ char *deflang = NULL, *lang = NULL;
 203         bool got_id = false, empty_line = true;
 204         int r;
 205
 206         assert(h);
 207         assert(sb);
 208         assert(path);
 209
 210         f = fopen(path, "re");
 211         if (!f)
 212                 return log_error_errno(errno, "Failed to open file %s: %m", path);
 213
 214         r = catalog_file_lang(path, &deflang);
 215         if (r < 0)
 216                 log_error_errno(errno, "Failed to determine language for file %s: %m", path);
 217         if (r == 1)
 218                 log_debug("File %s has language %s.", path, deflang);
 219
 220         for (;;) {
 221                 char line[LINE_MAX];
 222                 size_t a, b, c;
 223                 char *t;
 224
 225                 if (!fgets(line, sizeof(line), f)) {
 226                         if (feof(f))
 227                                 break;
 228
 229                         log_error_errno(errno, "Failed to read file %s: %m", path);
 230                         return -errno;
 231                 }
 232
 233                 n++;
 234
 235                 truncate_nl(line);
 236
 237                 if (line[0] == 0) {
 238                         empty_line = true;
 239                         continue;
 240                 }
 241
 242                 if (strchr(COMMENTS "\n", line[0]))
 243                         continue;
 244
 245                 if (empty_line &&
 246                     strlen(line) >= 2+1+32 &&
 247                     line[0] == '-' &&
 248                     line[1] == '-' &&
 249                     line[2] == ' ' &&
 250                     (line[2+1+32] == ' ' || line[2+1+32] == '\0')) {
 251
 252                         bool with_language;
 253                         sd_id128_t jd;
 254
 255                         /* New entry */
 256
 257                         with_language = line[2+1+32] != '\0';
 258                         line[2+1+32] = '\0';
 259
 260                         if (sd_id128_from_string(line + 2 + 1, &jd) >= 0) {
 261
 262                                 if (got_id) {
 263                                         r = finish_item(h, sb, id, lang ?: deflang, payload);
 264                                         if (r < 0)
 265                                                 return r;
 266
 267                                         free(lang);
 268                                         lang = NULL;
 269                                 }
 270
 271                                 if (with_language) {
 272                                         t = strstrip(line + 2 + 1 + 32 + 1);
 273
 274                                         r = catalog_entry_lang(path, n, t, deflang, &lang);
 275                                         if (r < 0)
 276                                                 return r;
 277                                 }
 278
 279                                 got_id = true;
 280                                 empty_line = false;
 281                                 id = jd;
 282
 283                                 if (payload)
 284                                         payload[0] = '\0';
 285
 286                                 continue;
 287                         }
 288                 }
 289
 290                 /* Payload */
 291                 if (!got_id) {
 292                         log_error("[%s:%u] Got payload before ID.", path, n);
 293                         return -EINVAL;
 294                 }
 295
 296                 a = payload ? strlen(payload) : 0;
 297                 b = strlen(line);
 298
 299                 c = a + (empty_line ? 1 : 0) + b + 1 + 1;
 300                 t = realloc(payload, c);
 301                 if (!t)
 302                         return log_oom();
 303
 304                 if (empty_line) {
 305                         t[a] = '\n';
 306                         memcpy(t + a + 1, line, b);
 307                         t[a+b+1] = '\n';
 308                         t[a+b+2] = 0;
 309                 } else {
 310                         memcpy(t + a, line, b);
 311                         t[a+b] = '\n';
 312                         t[a+b+1] = 0;
 313                 }
 314
 315                 payload = t;
 316                 empty_line = false;
 317         }
 318
 319         if (got_id) {
 320                 r = finish_item(h, sb, id, lang ?: deflang, payload);
 321                 if (r < 0)
 322                         return r;
 323         }
 324
 325         return 0;
 326 }
 327
 328 static long write_catalog(const char *database, Hashmap *h, struct strbuf *sb,
 329                           CatalogItem *items, size_t n) {
 330         CatalogHeader header;
 331         _cleanup_fclose_ FILE *w = NULL;
 332         int r;
 333         _cleanup_free_ char *d, *p = NULL;
 334         size_t k;
 335
 336         d = dirname_malloc(database);
 337         if (!d)
 338                 return log_oom();
 339
 340         r = mkdir_p(d, 0775);
 341         if (r < 0)
 342                 return log_error_errno(r, "Recursive mkdir %s: %m", d);
 343
 344         r = fopen_temporary(database, &w, &p);
 345         if (r < 0)
 346                 return log_error_errno(r, "Failed to open database for writing: %s: %m",
 347                                        database);
 348
 349         zero(header);
 350         memcpy(header.signature, CATALOG_SIGNATURE, sizeof(header.signature));
 351         header.header_size = htole64(ALIGN_TO(sizeof(CatalogHeader), 8));
 352         header.catalog_item_size = htole64(sizeof(CatalogItem));
 353         header.n_items = htole64(hashmap_size(h));
 354
 355         r = -EIO;
 356
 357         k = fwrite(&header, 1, sizeof(header), w);
 358         if (k != sizeof(header)) {
 359                 log_error("%s: failed to write header.", p);
 360                 goto error;
 361         }
 362
 363         k = fwrite(items, 1, n * sizeof(CatalogItem), w);
 364         if (k != n * sizeof(CatalogItem)) {
 365                 log_error("%s: failed to write database.", p);
 366                 goto error;
 367         }
 368
 369         k = fwrite(sb->buf, 1, sb->len, w);
 370         if (k != sb->len) {
 371                 log_error("%s: failed to write strings.", p);
 372                 goto error;
 373         }
 374
 375         fflush(w);
 376
 377         if (ferror(w)) {
 378                 log_error("%s: failed to write database.", p);
 379                 goto error;
 380         }
 381
 382         fchmod(fileno(w), 0644);
 383
 384         if (rename(p, database) < 0) {
 385                 log_error_errno(errno, "rename (%s -> %s) failed: %m", p, database);
 386                 r = -errno;
 387                 goto error;
 388         }
 389
 390         return ftell(w);
 391
 392 error:
 393         unlink(p);
 394         return r;
 395 }
 396
 397 int catalog_update(const char* database, const char* root, const char* const* dirs) {
 398         _cleanup_strv_free_ char **files = NULL;
 399         char **f;
 400         struct strbuf *sb = NULL;
 401         _cleanup_hashmap_free_free_ Hashmap *h = NULL;
 402         _cleanup_free_ CatalogItem *items = NULL;
 403         CatalogItem *i;
 404         Iterator j;
 405         unsigned n;
 406         long r;
 407
 408         h = hashmap_new(&catalog_hash_ops);
 409         sb = strbuf_new();
 410
 411         if (!h || !sb) {
 412                 r = log_oom();
 413                 goto finish;
 414         }
 415
 416         r = conf_files_list_strv(&files, ".catalog", root, dirs);
 417         if (r < 0) {
 418                 log_error_errno(r, "Failed to get catalog files: %m");
 419                 goto finish;
 420         }
 421
 422         STRV_FOREACH(f, files) {
 423                 log_debug("Reading file '%s'", *f);
 424                 r = catalog_import_file(h, sb, *f);
 425                 if (r < 0) {
 426                         log_error("Failed to import file '%s': %s.",
 427                                   *f, strerror(-r));
 428                         goto finish;
 429                 }
 430         }
 431
 432         if (hashmap_size(h) <= 0) {
 433                 log_info("No items in catalog.");
 434                 goto finish;
 435         } else
 436                 log_debug("Found %u items in catalog.", hashmap_size(h));
 437
 438         strbuf_complete(sb);
 439
 440         items = new(CatalogItem, hashmap_size(h));
 441         if (!items) {
 442                 r = log_oom();
 443                 goto finish;
 444         }
 445
 446         n = 0;
 447         HASHMAP_FOREACH(i, h, j) {
 448                 log_debug("Found " SD_ID128_FORMAT_STR ", language %s",
 449                           SD_ID128_FORMAT_VAL(i->id),
 450                           isempty(i->language) ? "C" : i->language);
 451                 items[n++] = *i;
 452         }
 453
 454         assert(n == hashmap_size(h));
 455         qsort_safe(items, n, sizeof(CatalogItem), catalog_compare_func);
 456
 457         r = write_catalog(database, h, sb, items, n);
 458         if (r < 0)
 459                 log_error_errno(r, "Failed to write %s: %m", database);
 460         else
 461                 log_debug("%s: wrote %u items, with %zu bytes of strings, %ld total size.",
 462                           database, n, sb->len, r);
 463
 464 finish:
 465         if (sb)
 466                 strbuf_cleanup(sb);
 467
 468         return r < 0 ? r : 0;
 469 }
 470
 471 static int open_mmap(const char *database, int *_fd, struct stat *_st, void **_p) {
 472         const CatalogHeader *h;
 473         int fd;
 474         void *p;
 475         struct stat st;
 476
 477         assert(_fd);
 478         assert(_st);
 479         assert(_p);
 480
 481         fd = open(database, O_RDONLY|O_CLOEXEC);
 482         if (fd < 0)
 483                 return -errno;
 484
 485         if (fstat(fd, &st) < 0) {
 486                 safe_close(fd);
 487                 return -errno;
 488         }
 489
 490         if (st.st_size < (off_t) sizeof(CatalogHeader)) {
 491                 safe_close(fd);
 492                 return -EINVAL;
 493         }
 494
 495         p = mmap(NULL, PAGE_ALIGN(st.st_size), PROT_READ, MAP_SHARED, fd, 0);
 496         if (p == MAP_FAILED) {
 497                 safe_close(fd);
 498                 return -errno;
 499         }
 500
 501         h = p;
 502         if (memcmp(h->signature, CATALOG_SIGNATURE, sizeof(h->signature)) != 0 ||
 503             le64toh(h->header_size) < sizeof(CatalogHeader) ||
 504             le64toh(h->catalog_item_size) < sizeof(CatalogItem) ||
 505             h->incompatible_flags != 0 ||
 506             le64toh(h->n_items) <= 0 ||
 507             st.st_size < (off_t) (le64toh(h->header_size) + le64toh(h->catalog_item_size) * le64toh(h->n_items))) {
 508                 safe_close(fd);
 509                 munmap(p, st.st_size);
 510                 return -EBADMSG;
 511         }
 512
 513         *_fd = fd;
 514         *_st = st;
 515         *_p = p;
 516
 517         return 0;
 518 }
 519
 520 static const char *find_id(void *p, sd_id128_t id) {
 521         CatalogItem key, *f = NULL;
 522         const CatalogHeader *h = p;
 523         const char *loc;
 524
 525         zero(key);
 526         key.id = id;
 527
 528         loc = setlocale(LC_MESSAGES, NULL);
 529         if (loc && loc[0] && !streq(loc, "C") && !streq(loc, "POSIX")) {
 530                 strncpy(key.language, loc, sizeof(key.language));
 531                 key.language[strcspn(key.language, ".@")] = 0;
 532
 533                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), catalog_compare_func);
 534                 if (!f) {
 535                         char *e;
 536
 537                         e = strchr(key.language, '_');
 538                         if (e) {
 539                                 *e = 0;
 540                                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), catalog_compare_func);
 541                         }
 542                 }
 543         }
 544
 545         if (!f) {
 546                 zero(key.language);
 547                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), catalog_compare_func);
 548         }
 549
 550         if (!f)
 551                 return NULL;
 552
 553         return (const char*) p +
 554                 le64toh(h->header_size) +
 555                 le64toh(h->n_items) * le64toh(h->catalog_item_size) +
 556                 le64toh(f->offset);
 557 }
 558
 559 int catalog_get(const char* database, sd_id128_t id, char **_text) {
 560         _cleanup_close_ int fd = -1;
 561         void *p = NULL;
 562         struct stat st;
 563         char *text = NULL;
 564         int r;
 565         const char *s;
 566
 567         assert(_text);
 568
 569         r = open_mmap(database, &fd, &st, &p);
 570         if (r < 0)
 571                 return r;
 572
 573         s = find_id(p, id);
 574         if (!s) {
 575                 r = -ENOENT;
 576                 goto finish;
 577         }
 578
 579         text = strdup(s);
 580         if (!text) {
 581                 r = -ENOMEM;
 582                 goto finish;
 583         }
 584
 585         *_text = text;
 586         r = 0;
 587
 588 finish:
 589         if (p)
 590                 munmap(p, st.st_size);
 591
 592         return r;
 593 }
 594
 595 static char *find_header(const char *s, const char *header) {
 596
 597         for (;;) {
 598                 const char *v, *e;
 599
 600                 v = startswith(s, header);
 601                 if (v) {
 602                         v += strspn(v, WHITESPACE);
 603                         return strndup(v, strcspn(v, NEWLINE));
 604                 }
 605
 606                 /* End of text */
 607                 e = strchr(s, '\n');
 608                 if (!e)
 609                         return NULL;
 610
 611                 /* End of header */
 612                 if (e == s)
 613                         return NULL;
 614
 615                 s = e + 1;
 616         }
 617 }
 618
 619 static void dump_catalog_entry(FILE *f, sd_id128_t id, const char *s, bool oneline) {
 620         if (oneline) {
 621                 _cleanup_free_ char *subject = NULL, *defined_by = NULL;
 622
 623                 subject = find_header(s, "Subject:");
 624                 defined_by = find_header(s, "Defined-By:");
 625
 626                 fprintf(f, SD_ID128_FORMAT_STR " %s: %s\n",
 627                         SD_ID128_FORMAT_VAL(id),
 628                         strna(defined_by), strna(subject));
 629         } else
 630                 fprintf(f, "-- " SD_ID128_FORMAT_STR "\n%s\n",
 631                         SD_ID128_FORMAT_VAL(id), s);
 632 }
 633
 634
 635 int catalog_list(FILE *f, const char *database, bool oneline) {
 636         _cleanup_close_ int fd = -1;
 637         void *p = NULL;
 638         struct stat st;
 639         const CatalogHeader *h;
 640         const CatalogItem *items;
 641         int r;
 642         unsigned n;
 643         sd_id128_t last_id;
 644         bool last_id_set = false;
 645
 646         r = open_mmap(database, &fd, &st, &p);
 647         if (r < 0)
 648                 return r;
 649
 650         h = p;
 651         items = (const CatalogItem*) ((const uint8_t*) p + le64toh(h->header_size));
 652
 653         for (n = 0; n < le64toh(h->n_items); n++) {
 654                 const char *s;
 655
 656                 if (last_id_set && sd_id128_equal(last_id, items[n].id))
 657                         continue;
 658
 659                 assert_se(s = find_id(p, items[n].id));
 660
 661                 dump_catalog_entry(f, items[n].id, s, oneline);
 662
 663                 last_id_set = true;
 664                 last_id = items[n].id;
 665         }
 666
 667         munmap(p, st.st_size);
 668
 669         return 0;
 670 }
 671
 672 int catalog_list_items(FILE *f, const char *database, bool oneline, char **items) {
 673         char **item;
 674         int r = 0;
 675
 676         STRV_FOREACH(item, items) {
 677                 sd_id128_t id;
 678                 int k;
 679                 _cleanup_free_ char *msg = NULL;
 680
 681                 k = sd_id128_from_string(*item, &id);
 682                 if (k < 0) {
 683                         log_error_errno(k, "Failed to parse id128 '%s': %m",
 684                                         *item);
 685                         if (r == 0)
 686                                 r = k;
 687                         continue;
 688                 }
 689
 690                 k = catalog_get(database, id, &msg);
 691                 if (k < 0) {
 692                         log_full(k == -ENOENT ? LOG_NOTICE : LOG_ERR,
 693                                  "Failed to retrieve catalog entry for '%s': %s",
 694                                   *item, strerror(-k));
 695                         if (r == 0)
 696                                 r = k;
 697                         continue;
 698                 }
 699
 700                 dump_catalog_entry(f, id, msg, oneline);
 701         }
 702
 703         return r;
 704 }