data/src/status.c

   1 /*
   2   Copyright (C) 2010 - 2020 Proxmox Server Solutions GmbH
   3
   4   This program is free software: you can redistribute it and/or modify
   5   it under the terms of the GNU Affero General Public License as published by
   6   the Free Software Foundation, either version 3 of the License, or
   7   (at your option) any later version.
   8
   9   This program is distributed in the hope that it will be useful,
  10   but WITHOUT ANY WARRANTY; without even the implied warranty of
  11   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12   GNU Affero General Public License for more details.
  13
  14   You should have received a copy of the GNU Affero General Public License
  15   along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17   Author: Dietmar Maurer <dietmar@proxmox.com>
  18
  19 */
  20
  21 #define G_LOG_DOMAIN "status"
  22
  23 #ifdef HAVE_CONFIG_H
  24 #include <config.h>
  25 #endif /* HAVE_CONFIG_H */
  26
  27 #include <stdio.h>
  28 #include <stdint.h>
  29 #include <string.h>
  30 #include <errno.h>
  31 #include <glib.h>
  32 #include <sys/syslog.h>
  33 #include <rrd.h>
  34 #include <rrd_client.h>
  35 #include <time.h>
  36 #include <ctype.h>
  37
  38 #include "cfs-utils.h"
  39 #include "status.h"
  40 #include "memdb.h"
  41 #include "logger.h"
  42
  43 #define KVSTORE_CPG_GROUP_NAME "pve_kvstore_v1"
  44
  45 typedef enum {
  46         KVSTORE_MESSAGE_UPDATE = 1,
  47         KVSTORE_MESSAGE_UPDATE_COMPLETE = 2,
  48         KVSTORE_MESSAGE_LOG = 3,
  49 } kvstore_message_t;
  50
  51 static uint32_t vminfo_version_counter;
  52
  53 typedef struct {
  54         uint32_t vmid;
  55         char *nodename;
  56         int vmtype;
  57         uint32_t version;
  58 } vminfo_t;
  59
  60 typedef struct {
  61         char *key;
  62         gpointer data;
  63         size_t len;
  64         uint32_t version;
  65 } kventry_t;
  66
  67 typedef struct {
  68         char *key;
  69         gpointer data;
  70         size_t len;
  71         uint32_t time;
  72 } rrdentry_t;
  73
  74 typedef struct {
  75         char *path;
  76         uint32_t version;
  77 } memdb_change_t;
  78
  79 static memdb_change_t memdb_change_array[] = {
  80         { .path = "corosync.conf" },
  81         { .path = "corosync.conf.new" },
  82         { .path = "storage.cfg" },
  83         { .path = "user.cfg" },
  84         { .path = "domains.cfg" },
  85         { .path = "priv/shadow.cfg" },
  86         { .path = "priv/acme/plugins.cfg" },
  87         { .path = "priv/tfa.cfg" },
  88         { .path = "priv/token.cfg" },
  89         { .path = "priv/ipam.db" },
  90         { .path = "datacenter.cfg" },
  91         { .path = "vzdump.cron" },
  92         { .path = "jobs.cfg" },
  93         { .path = "ha/crm_commands" },
  94         { .path = "ha/manager_status" },
  95         { .path = "ha/resources.cfg" },
  96         { .path = "ha/groups.cfg" },
  97         { .path = "ha/fence.cfg" },
  98         { .path = "status.cfg" },
  99         { .path = "replication.cfg" },
 100         { .path = "ceph.conf" },
 101         { .path = "sdn/vnets.cfg" },
 102         { .path = "sdn/zones.cfg" },
 103         { .path = "sdn/controllers.cfg" },
 104         { .path = "sdn/subnets.cfg" },
 105         { .path = "sdn/ipams.cfg" },
 106         { .path = "sdn/dns.cfg" },
 107         { .path = "sdn/.running-config" },
 108         { .path = "virtual-guest/cpu-models.conf" },
 109 };
 110
 111 static GMutex mutex;
 112
 113 typedef struct {
 114         time_t start_time;
 115
 116         uint32_t quorate;
 117
 118         cfs_clinfo_t *clinfo;
 119         uint32_t clinfo_version;
 120
 121         GHashTable *vmlist;
 122         uint32_t vmlist_version;
 123
 124         dfsm_t *kvstore;
 125         GHashTable *kvhash;
 126         GHashTable *rrdhash;
 127         GHashTable *iphash;
 128
 129         GHashTable *memdb_changes;
 130
 131         clusterlog_t *clusterlog;
 132 } cfs_status_t;
 133
 134 static cfs_status_t cfs_status;
 135
 136 struct cfs_clnode {
 137         char *name;
 138         uint32_t nodeid;
 139         uint32_t votes;
 140         gboolean online;
 141         GHashTable *kvhash;
 142 };
 143
 144 struct cfs_clinfo {
 145         char *cluster_name;
 146         uint32_t cman_version;
 147
 148         GHashTable *nodes_byid;
 149         GHashTable *nodes_byname;
 150 };
 151
 152 static guint
 153 g_int32_hash (gconstpointer v)
 154 {
 155         return *(const uint32_t *) v;
 156 }
 157
 158 static gboolean
 159 g_int32_equal (gconstpointer v1,
 160                gconstpointer v2)
 161 {
 162         return *((const uint32_t*) v1) == *((const uint32_t*) v2);
 163 }
 164
 165 static void vminfo_free(vminfo_t *vminfo)
 166 {
 167         g_return_if_fail(vminfo != NULL);
 168
 169         if (vminfo->nodename)
 170                 g_free(vminfo->nodename);
 171
 172
 173         g_free(vminfo);
 174 }
 175
 176 static const char *vminfo_type_to_string(vminfo_t *vminfo)
 177 {
 178         if (vminfo->vmtype == VMTYPE_QEMU) {
 179                 return "qemu";
 180         } else if (vminfo->vmtype == VMTYPE_OPENVZ) {
 181                 // FIXME: remove openvz stuff for 7.x
 182                 return "openvz";
 183         } else if (vminfo->vmtype == VMTYPE_LXC) {
 184                 return "lxc";
 185         } else {
 186                 return "unknown";
 187         }
 188 }
 189
 190 static const char *vminfo_type_to_path_type(vminfo_t *vminfo)
 191 {
 192         if (vminfo->vmtype == VMTYPE_QEMU) {
 193                 return "qemu-server"; // special case..
 194         } else {
 195                 return vminfo_type_to_string(vminfo);
 196         }
 197 }
 198
 199 int vminfo_to_path(vminfo_t *vminfo, GString *path)
 200 {
 201         g_return_val_if_fail(vminfo != NULL, -1);
 202         g_return_val_if_fail(path != NULL, -1);
 203
 204         if (!vminfo->nodename)
 205                 return 0;
 206
 207         const char *type = vminfo_type_to_path_type(vminfo);
 208         g_string_printf(path, "/nodes/%s/%s/%u.conf", vminfo->nodename, type, vminfo->vmid);
 209
 210         return 1;
 211 }
 212
 213 void cfs_clnode_destroy(
 214         cfs_clnode_t *clnode)
 215 {
 216         g_return_if_fail(clnode != NULL);
 217
 218         if (clnode->kvhash)
 219                 g_hash_table_destroy(clnode->kvhash);
 220
 221         if (clnode->name)
 222                 g_free(clnode->name);
 223
 224         g_free(clnode);
 225 }
 226
 227 cfs_clnode_t *cfs_clnode_new(
 228         const char *name,
 229         uint32_t nodeid,
 230         uint32_t votes)
 231 {
 232         g_return_val_if_fail(name != NULL, NULL);
 233
 234         cfs_clnode_t *clnode = g_new0(cfs_clnode_t, 1);
 235         if (!clnode)
 236                 return NULL;
 237
 238         clnode->name = g_strdup(name);
 239         clnode->nodeid = nodeid;
 240         clnode->votes = votes;
 241
 242         return clnode;
 243 }
 244
 245 gboolean cfs_clinfo_destroy(
 246         cfs_clinfo_t *clinfo)
 247 {
 248         g_return_val_if_fail(clinfo != NULL, FALSE);
 249
 250         if (clinfo->cluster_name)
 251                 g_free(clinfo->cluster_name);
 252
 253         if (clinfo->nodes_byname)
 254                 g_hash_table_destroy(clinfo->nodes_byname);
 255
 256         if (clinfo->nodes_byid)
 257                 g_hash_table_destroy(clinfo->nodes_byid);
 258
 259         g_free(clinfo);
 260
 261         return TRUE;
 262 }
 263
 264 cfs_clinfo_t *cfs_clinfo_new(
 265         const char *cluster_name,
 266         uint32_t cman_version)
 267 {
 268         g_return_val_if_fail(cluster_name != NULL, NULL);
 269
 270         cfs_clinfo_t *clinfo = g_new0(cfs_clinfo_t, 1);
 271         if (!clinfo)
 272                 return NULL;
 273
 274         clinfo->cluster_name = g_strdup(cluster_name);
 275         clinfo->cman_version = cman_version;
 276
 277         if (!(clinfo->nodes_byid = g_hash_table_new_full(
 278                       g_int32_hash, g_int32_equal, NULL,
 279                       (GDestroyNotify)cfs_clnode_destroy)))
 280                 goto fail;
 281
 282         if (!(clinfo->nodes_byname = g_hash_table_new(g_str_hash, g_str_equal)))
 283                 goto fail;
 284
 285         return clinfo;
 286
 287 fail:
 288         cfs_clinfo_destroy(clinfo);
 289
 290         return NULL;
 291 }
 292
 293 gboolean cfs_clinfo_add_node(
 294         cfs_clinfo_t *clinfo,
 295         cfs_clnode_t *clnode)
 296 {
 297         g_return_val_if_fail(clinfo != NULL, FALSE);
 298         g_return_val_if_fail(clnode != NULL, FALSE);
 299
 300         g_hash_table_replace(clinfo->nodes_byid, &clnode->nodeid, clnode);
 301         g_hash_table_replace(clinfo->nodes_byname, clnode->name, clnode);
 302
 303         return TRUE;
 304 }
 305
 306 int
 307 cfs_create_memberlist_msg(
 308         GString *str)
 309 {
 310         g_return_val_if_fail(str != NULL, -EINVAL);
 311
 312         g_mutex_lock (&mutex);
 313
 314         g_string_append_printf(str,"{\n");
 315
 316         guint nodecount = 0;
 317
 318         cfs_clinfo_t *clinfo = cfs_status.clinfo;
 319
 320         if (clinfo && clinfo->nodes_byid)
 321                 nodecount = g_hash_table_size(clinfo->nodes_byid);
 322
 323         if (nodecount) {
 324                 g_string_append_printf(str, "\"nodename\": \"%s\",\n", cfs.nodename);
 325                 g_string_append_printf(str, "\"version\": %u,\n", cfs_status.clinfo_version);
 326
 327                 g_string_append_printf(str, "\"cluster\": { ");
 328                 g_string_append_printf(str, "\"name\": \"%s\", \"version\": %d, "
 329                                        "\"nodes\": %d, \"quorate\": %d ",
 330                                        clinfo->cluster_name, clinfo->cman_version,
 331                                        nodecount, cfs_status.quorate);
 332
 333                 g_string_append_printf(str,"},\n");
 334                 g_string_append_printf(str,"\"nodelist\": {\n");
 335
 336                 GHashTable *ht = clinfo->nodes_byid;
 337                 GHashTableIter iter;
 338                 gpointer key, value;
 339
 340                 g_hash_table_iter_init (&iter, ht);
 341
 342                 int i = 0;
 343                 while (g_hash_table_iter_next (&iter, &key, &value)) {
 344                         cfs_clnode_t *node = (cfs_clnode_t *)value;
 345                         if (i) g_string_append_printf(str, ",\n");
 346                         i++;
 347
 348                         g_string_append_printf(str, "  \"%s\": { \"id\": %d, \"online\": %d",
 349                                                node->name, node->nodeid, node->online);
 350
 351
 352                         char *ip = (char *)g_hash_table_lookup(cfs_status.iphash, node->name);
 353                         if (ip) {
 354                                 g_string_append_printf(str, ", \"ip\": \"%s\"", ip);
 355                         }
 356
 357                         g_string_append_printf(str, "}");
 358
 359                 }
 360                 g_string_append_printf(str,"\n  }\n");
 361         } else {
 362                 g_string_append_printf(str, "\"nodename\": \"%s\",\n", cfs.nodename);
 363                 g_string_append_printf(str, "\"version\": %u\n", cfs_status.clinfo_version);
 364         }
 365
 366         g_string_append_printf(str,"}\n");
 367
 368         g_mutex_unlock (&mutex);
 369
 370         return 0;
 371 }
 372
 373 static void
 374 kventry_free(kventry_t *entry)
 375 {
 376         g_return_if_fail(entry != NULL);
 377
 378         g_free(entry->key);
 379         g_free(entry->data);
 380         g_free(entry);
 381 }
 382
 383 static GHashTable *
 384 kventry_hash_new(void)
 385 {
 386         return g_hash_table_new_full(g_str_hash, g_str_equal, NULL,
 387                                      (GDestroyNotify)kventry_free);
 388 }
 389
 390 static void
 391 rrdentry_free(rrdentry_t *entry)
 392 {
 393         g_return_if_fail(entry != NULL);
 394
 395         g_free(entry->key);
 396         g_free(entry->data);
 397         g_free(entry);
 398 }
 399
 400 static GHashTable *
 401 rrdentry_hash_new(void)
 402 {
 403         return g_hash_table_new_full(g_str_hash, g_str_equal, NULL,
 404                                      (GDestroyNotify)rrdentry_free);
 405 }
 406
 407 void
 408 cfs_cluster_log_dump(GString *str, const char *user, guint max_entries)
 409 {
 410         clusterlog_dump(cfs_status.clusterlog, str, user, max_entries);
 411 }
 412
 413 void
 414 cfs_cluster_log(clog_entry_t *entry)
 415 {
 416         g_return_if_fail(entry != NULL);
 417
 418         clusterlog_insert(cfs_status.clusterlog, entry);
 419
 420         if (cfs_status.kvstore) {
 421                 struct iovec iov[1];
 422                 iov[0].iov_base = (char *)entry;
 423                 iov[0].iov_len = clog_entry_size(entry);
 424
 425                 if (dfsm_is_initialized(cfs_status.kvstore))
 426                         dfsm_send_message(cfs_status.kvstore, KVSTORE_MESSAGE_LOG, iov, 1);
 427         }
 428 }
 429
 430 void cfs_status_init(void)
 431 {
 432         g_mutex_lock (&mutex);
 433
 434         cfs_status.start_time = time(NULL);
 435
 436         cfs_status.vmlist = vmlist_hash_new();
 437
 438         cfs_status.kvhash = kventry_hash_new();
 439
 440         cfs_status.rrdhash = rrdentry_hash_new();
 441
 442         cfs_status.iphash = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, g_free);
 443
 444         cfs_status.memdb_changes = g_hash_table_new(g_str_hash, g_str_equal);
 445
 446         for (int i = 0; i < G_N_ELEMENTS(memdb_change_array); i++) {
 447                 g_hash_table_replace(cfs_status.memdb_changes,
 448                                      memdb_change_array[i].path,
 449                                      &memdb_change_array[i]);
 450         }
 451
 452         cfs_status.clusterlog = clusterlog_new();
 453
 454         // fixme:
 455         clusterlog_add(cfs_status.clusterlog, "root", "cluster", getpid(),
 456                        LOG_INFO, "starting cluster log");
 457
 458         g_mutex_unlock (&mutex);
 459 }
 460
 461 void cfs_status_cleanup(void)
 462 {
 463         g_mutex_lock (&mutex);
 464
 465         cfs_status.clinfo_version++;
 466
 467         if (cfs_status.clinfo) {
 468                 cfs_clinfo_destroy(cfs_status.clinfo);
 469                 cfs_status.clinfo = NULL;
 470         }
 471
 472         if (cfs_status.vmlist) {
 473                 g_hash_table_destroy(cfs_status.vmlist);
 474                 cfs_status.vmlist = NULL;
 475         }
 476
 477         if (cfs_status.kvhash) {
 478                 g_hash_table_destroy(cfs_status.kvhash);
 479                 cfs_status.kvhash = NULL;
 480         }
 481
 482         if (cfs_status.rrdhash) {
 483                 g_hash_table_destroy(cfs_status.rrdhash);
 484                 cfs_status.rrdhash = NULL;
 485         }
 486
 487         if (cfs_status.iphash) {
 488                 g_hash_table_destroy(cfs_status.iphash);
 489                 cfs_status.iphash = NULL;
 490         }
 491
 492         if (cfs_status.clusterlog)
 493                 clusterlog_destroy(cfs_status.clusterlog);
 494
 495         g_mutex_unlock (&mutex);
 496 }
 497
 498 void cfs_status_set_clinfo(
 499         cfs_clinfo_t *clinfo)
 500 {
 501         g_return_if_fail(clinfo != NULL);
 502
 503         g_mutex_lock (&mutex);
 504
 505         cfs_status.clinfo_version++;
 506
 507         cfs_clinfo_t *old = cfs_status.clinfo;
 508
 509         cfs_status.clinfo = clinfo;
 510
 511         cfs_message("update cluster info (cluster name  %s, version = %d)",
 512                     clinfo->cluster_name, clinfo->cman_version);
 513
 514
 515         if (old && old->nodes_byid && clinfo->nodes_byid) {
 516                 /* copy kvstore */
 517                 GHashTable *ht = clinfo->nodes_byid;
 518                 GHashTableIter iter;
 519                 gpointer key, value;
 520
 521                 g_hash_table_iter_init (&iter, ht);
 522
 523                 while (g_hash_table_iter_next (&iter, &key, &value)) {
 524                         cfs_clnode_t *node = (cfs_clnode_t *)value;
 525                         cfs_clnode_t *oldnode;
 526                         if ((oldnode = g_hash_table_lookup(old->nodes_byid, key))) {
 527                                 node->online = oldnode->online;
 528                                 node->kvhash = oldnode->kvhash;
 529                                 oldnode->kvhash = NULL;
 530                         }
 531                 }
 532
 533         }
 534
 535         if (old)
 536                 cfs_clinfo_destroy(old);
 537
 538
 539         g_mutex_unlock (&mutex);
 540 }
 541
 542 static void
 543 dump_kvstore_versions(
 544         GString *str,
 545         GHashTable *kvhash,
 546         const char *nodename)
 547 {
 548         g_return_if_fail(kvhash != NULL);
 549         g_return_if_fail(str != NULL);
 550         g_return_if_fail(nodename != NULL);
 551
 552         GHashTable *ht = kvhash;
 553         GHashTableIter iter;
 554         gpointer key, value;
 555
 556         g_string_append_printf(str, "\"%s\": {\n", nodename);
 557
 558         g_hash_table_iter_init (&iter, ht);
 559
 560         int i = 0;
 561         while (g_hash_table_iter_next (&iter, &key, &value)) {
 562                 kventry_t *entry = (kventry_t *)value;
 563                 if (i) g_string_append_printf(str, ",\n");
 564                 i++;
 565                 g_string_append_printf(str,"\"%s\": %u", entry->key, entry->version);
 566         }
 567
 568         g_string_append_printf(str, "}\n");
 569 }
 570
 571 int
 572 cfs_create_version_msg(GString *str)
 573 {
 574         g_return_val_if_fail(str != NULL, -EINVAL);
 575
 576         g_mutex_lock (&mutex);
 577
 578         g_string_append_printf(str,"{\n");
 579
 580         g_string_append_printf(str, "\"starttime\": %lu,\n", (unsigned long)cfs_status.start_time);
 581
 582         g_string_append_printf(str, "\"clinfo\": %u,\n", cfs_status.clinfo_version);
 583
 584         g_string_append_printf(str, "\"vmlist\": %u,\n", cfs_status.vmlist_version);
 585
 586         for (int i = 0; i < G_N_ELEMENTS(memdb_change_array); i++) {
 587                 g_string_append_printf(str, "\"%s\": %u,\n",
 588                                        memdb_change_array[i].path,
 589                                        memdb_change_array[i].version);
 590         }
 591
 592         g_string_append_printf(str, "\"kvstore\": {\n");
 593
 594         dump_kvstore_versions(str, cfs_status.kvhash, cfs.nodename);
 595
 596         cfs_clinfo_t *clinfo = cfs_status.clinfo;
 597
 598         if (clinfo && clinfo->nodes_byid) {
 599                 GHashTable *ht = clinfo->nodes_byid;
 600                 GHashTableIter iter;
 601                 gpointer key, value;
 602
 603                 g_hash_table_iter_init (&iter, ht);
 604
 605                 while (g_hash_table_iter_next (&iter, &key, &value)) {
 606                         cfs_clnode_t *node = (cfs_clnode_t *)value;
 607                         if (!node->kvhash)
 608                                 continue;
 609                         g_string_append_printf(str, ",\n");
 610                         dump_kvstore_versions(str, node->kvhash, node->name);
 611                 }
 612         }
 613
 614         g_string_append_printf(str,"}\n");
 615
 616         g_string_append_printf(str,"}\n");
 617
 618         g_mutex_unlock (&mutex);
 619
 620         return 0;
 621 }
 622
 623 GHashTable *
 624 vmlist_hash_new(void)
 625 {
 626         return g_hash_table_new_full(g_int_hash, g_int_equal, NULL,
 627                                      (GDestroyNotify)vminfo_free);
 628 }
 629
 630 gboolean
 631 vmlist_hash_insert_vm(
 632         GHashTable *vmlist,
 633         int vmtype,
 634         guint32 vmid,
 635         const char *nodename,
 636         gboolean replace)
 637 {
 638         g_return_val_if_fail(vmlist != NULL, FALSE);
 639         g_return_val_if_fail(nodename != NULL, FALSE);
 640         g_return_val_if_fail(vmid != 0, FALSE);
 641         // FIXME: remove openvz stuff for 7.x
 642         g_return_val_if_fail(vmtype == VMTYPE_QEMU || vmtype == VMTYPE_OPENVZ ||
 643                              vmtype == VMTYPE_LXC, FALSE);
 644
 645         if (!replace && g_hash_table_lookup(vmlist, &vmid)) {
 646                 cfs_critical("detected duplicate VMID %d", vmid);
 647                 return FALSE;
 648         }
 649
 650         vminfo_t *vminfo = g_new0(vminfo_t, 1);
 651
 652         vminfo->vmid = vmid;
 653         vminfo->vmtype = vmtype;
 654         vminfo->nodename = g_strdup(nodename);
 655
 656         vminfo->version = ++vminfo_version_counter;
 657
 658         g_hash_table_replace(vmlist, &vminfo->vmid, vminfo);
 659
 660         return TRUE;
 661 }
 662
 663 void
 664 vmlist_register_vm(
 665         int vmtype,
 666         guint32 vmid,
 667         const char *nodename)
 668 {
 669         g_return_if_fail(cfs_status.vmlist != NULL);
 670         g_return_if_fail(nodename != NULL);
 671         g_return_if_fail(vmid != 0);
 672         // FIXME: remove openvz stuff for 7.x
 673         g_return_if_fail(vmtype == VMTYPE_QEMU || vmtype == VMTYPE_OPENVZ ||
 674                          vmtype == VMTYPE_LXC);
 675
 676         cfs_debug("vmlist_register_vm: %s/%u %d", nodename, vmid, vmtype);
 677
 678         g_mutex_lock (&mutex);
 679
 680         cfs_status.vmlist_version++;
 681
 682         vmlist_hash_insert_vm(cfs_status.vmlist, vmtype, vmid, nodename, TRUE);
 683
 684         g_mutex_unlock (&mutex);
 685 }
 686
 687 gboolean
 688 vmlist_different_vm_exists(
 689         int vmtype,
 690         guint32 vmid,
 691         const char *nodename)
 692 {
 693         g_return_val_if_fail(cfs_status.vmlist != NULL, FALSE);
 694         g_return_val_if_fail(vmid != 0, FALSE);
 695
 696         gboolean res = FALSE;
 697
 698         g_mutex_lock (&mutex);
 699
 700         vminfo_t *vminfo;
 701         if ((vminfo = (vminfo_t *)g_hash_table_lookup(cfs_status.vmlist, &vmid))) {
 702                 if (!(vminfo->vmtype == vmtype && strcmp(vminfo->nodename, nodename) == 0))
 703                         res = TRUE;
 704         }
 705         g_mutex_unlock (&mutex);
 706
 707         return res;
 708 }
 709
 710 gboolean
 711 vmlist_vm_exists(
 712         guint32 vmid)
 713 {
 714         g_return_val_if_fail(cfs_status.vmlist != NULL, FALSE);
 715         g_return_val_if_fail(vmid != 0, FALSE);
 716
 717         g_mutex_lock (&mutex);
 718
 719         gpointer res = g_hash_table_lookup(cfs_status.vmlist, &vmid);
 720
 721         g_mutex_unlock (&mutex);
 722
 723         return res != NULL;
 724 }
 725
 726 void
 727 vmlist_delete_vm(
 728         guint32 vmid)
 729 {
 730         g_return_if_fail(cfs_status.vmlist != NULL);
 731         g_return_if_fail(vmid != 0);
 732
 733         g_mutex_lock (&mutex);
 734
 735         cfs_status.vmlist_version++;
 736
 737         g_hash_table_remove(cfs_status.vmlist, &vmid);
 738
 739         g_mutex_unlock (&mutex);
 740 }
 741
 742 void cfs_status_set_vmlist(
 743         GHashTable *vmlist)
 744 {
 745         g_return_if_fail(vmlist != NULL);
 746
 747         g_mutex_lock (&mutex);
 748
 749         cfs_status.vmlist_version++;
 750
 751         if (cfs_status.vmlist)
 752                 g_hash_table_destroy(cfs_status.vmlist);
 753
 754         cfs_status.vmlist = vmlist;
 755
 756         g_mutex_unlock (&mutex);
 757 }
 758
 759 int
 760 cfs_create_vmlist_msg(GString *str)
 761 {
 762         g_return_val_if_fail(cfs_status.vmlist != NULL, -EINVAL);
 763         g_return_val_if_fail(str != NULL, -EINVAL);
 764
 765         g_mutex_lock (&mutex);
 766
 767         g_string_append_printf(str,"{\n");
 768
 769         GHashTable *ht = cfs_status.vmlist;
 770
 771         guint count = g_hash_table_size(ht);
 772
 773         if (!count) {
 774                 g_string_append_printf(str,"\"version\": %u\n", cfs_status.vmlist_version);
 775         } else {
 776                 g_string_append_printf(str,"\"version\": %u,\n", cfs_status.vmlist_version);
 777
 778                 g_string_append_printf(str,"\"ids\": {\n");
 779
 780                 GHashTableIter iter;
 781                 gpointer key, value;
 782
 783                 g_hash_table_iter_init (&iter, ht);
 784
 785                 int first = 1;
 786                 while (g_hash_table_iter_next (&iter, &key, &value)) {
 787                         vminfo_t *vminfo = (vminfo_t *)value;
 788                         const char *type = vminfo_type_to_string(vminfo);
 789
 790                         if (!first)
 791                                 g_string_append_printf(str, ",\n");
 792                         first = 0;
 793
 794                         g_string_append_printf(str,"\"%u\": { \"node\": \"%s\", \"type\": \"%s\", \"version\": %u }",
 795                                                vminfo->vmid, vminfo->nodename, type, vminfo->version);
 796                 }
 797
 798                 g_string_append_printf(str,"}\n");
 799         }
 800         g_string_append_printf(str,"\n}\n");
 801
 802         g_mutex_unlock (&mutex);
 803
 804         return 0;
 805 }
 806
 807 // checks the conf for a line starting with '$prop:' and returns the value
 808 // afterwards, whitout initial whitespace(s), we only deal with the format
 809 // restricion imposed by our perl VM config parser, main reference is
 810 // PVE::QemuServer::parse_vm_config this allows to be very fast and still
 811 // relatively simple
 812 // main restrictions used for our advantage is the properties match reges:
 813 // ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) from parse_vm_config
 814 // currently we only look at the current configuration in place, i.e., *no*
 815 // snapshort and *no* pending changes
 816 static char *
 817 _get_property_value(char *conf, int conf_size, const char *prop, int prop_len)
 818 {
 819         const char *const conf_end = conf + conf_size;
 820         char *line = conf;
 821         size_t remaining_size;
 822
 823         char *next_newline = memchr(conf, '\n', conf_size);
 824         if (next_newline == NULL) {
 825                 return NULL; // valid property lines end with \n, but none in the config
 826         }
 827         *next_newline = '\0';
 828
 829         while (line != NULL) {
 830                 if (!line[0]) goto next;
 831
 832                 // snapshot or pending section start, but nothing found yet -> not found
 833                 if (line[0] == '[') return NULL;
 834                 // properties start with /^[a-z]/, so continue early if not
 835                 if (line[0] < 'a' || line[0] > 'z') goto next;
 836
 837                 int line_len = strlen(line);
 838                 if (line_len <= prop_len + 1) goto next;
 839
 840                 if (line[prop_len] == ':' && memcmp(line, prop, prop_len) == 0) { // found
 841                         char *v_start = &line[prop_len + 1];
 842
 843                         // drop initial value whitespaces here already
 844                         while (*v_start && isspace(*v_start)) v_start++;
 845
 846                         if (!*v_start) return NULL;
 847
 848                         char *v_end = &line[line_len - 1];
 849                         while (v_end > v_start && isspace(*v_end)) v_end--;
 850                         v_end[1] = '\0';
 851
 852                         return v_start;
 853                 }
 854 next:
 855                 line = next_newline + 1;
 856                 remaining_size = conf_end - line;
 857                 if (remaining_size <= prop_len) {
 858                         return NULL;
 859                 }
 860                 next_newline = memchr(line, '\n', remaining_size);
 861                 if (next_newline == NULL) {
 862                         return NULL; // valid property lines end with \n, but none in the config
 863                 }
 864                 *next_newline = '\0';
 865         }
 866
 867         return NULL; // not found
 868 }
 869
 870 static void
 871 _g_str_append_kv_jsonescaped(GString *str, const char *k, const char *v)
 872 {
 873         g_string_append_printf(str, "\"%s\": \"", k);
 874
 875         for (; *v; v++) {
 876                 if (*v == '\\' || *v == '"') {
 877                         g_string_append_c(str, '\\');
 878                 }
 879                 g_string_append_c(str, *v);
 880         }
 881
 882         g_string_append_c(str, '"');
 883 }
 884
 885 int
 886 cfs_create_guest_conf_property_msg(GString *str, memdb_t *memdb, const char *prop, uint32_t vmid)
 887 {
 888         g_return_val_if_fail(cfs_status.vmlist != NULL, -EINVAL);
 889         g_return_val_if_fail(str != NULL, -EINVAL);
 890
 891         int prop_len = strlen(prop);
 892         int res = 0;
 893         GString *path = NULL;
 894
 895         // Prelock &memdb->mutex in order to enable the usage of memdb_read_nolock
 896         // to prevent Deadlocks as in #2553
 897         g_mutex_lock (&memdb->mutex);
 898         g_mutex_lock (&mutex);
 899
 900         g_string_printf(str,"{\n");
 901
 902         GHashTable *ht = cfs_status.vmlist;
 903         gpointer tmp = NULL;
 904         if (!g_hash_table_size(ht)) {
 905                 goto ret;
 906         }
 907
 908         if ((path = g_string_sized_new(256)) == NULL) {
 909                 res = -ENOMEM;
 910                 goto ret;
 911         }
 912
 913         if (vmid >= 100) {
 914                 vminfo_t *vminfo = (vminfo_t *) g_hash_table_lookup(cfs_status.vmlist, &vmid);
 915                 if (vminfo == NULL) goto enoent;
 916
 917                 if (!vminfo_to_path(vminfo, path)) goto err;
 918
 919                 // use memdb_read_nolock because lock is handled here
 920                 int size = memdb_read_nolock(memdb, path->str, &tmp);
 921                 if (tmp == NULL) goto err;
 922                 if (size <= prop_len) goto ret;
 923
 924                 char *val = _get_property_value(tmp, size, prop, prop_len);
 925                 if (val == NULL) goto ret;
 926
 927                 g_string_append_printf(str, "\"%u\":{", vmid);
 928                 _g_str_append_kv_jsonescaped(str, prop, val);
 929                 g_string_append_c(str, '}');
 930
 931         } else {
 932                 GHashTableIter iter;
 933                 g_hash_table_iter_init (&iter, ht);
 934
 935                 gpointer key, value;
 936                 int first = 1;
 937                 while (g_hash_table_iter_next (&iter, &key, &value)) {
 938                         vminfo_t *vminfo = (vminfo_t *)value;
 939
 940                         if (!vminfo_to_path(vminfo, path)) goto err;
 941
 942                         g_free(tmp); // no-op if already null
 943                         tmp = NULL;
 944                         // use memdb_read_nolock because lock is handled here
 945                         int size = memdb_read_nolock(memdb, path->str, &tmp);
 946                         if (tmp == NULL || size <= prop_len) continue;
 947
 948                         char *val = _get_property_value(tmp, size, prop, prop_len);
 949                         if (val == NULL) continue;
 950
 951                         if (!first) g_string_append_printf(str, ",\n");
 952                         else first = 0;
 953
 954                         g_string_append_printf(str, "\"%u\":{", vminfo->vmid);
 955                         _g_str_append_kv_jsonescaped(str, prop, val);
 956                         g_string_append_c(str, '}');
 957                 }
 958         }
 959 ret:
 960         g_free(tmp);
 961         if (path != NULL) {
 962                 g_string_free(path, TRUE);
 963         }
 964         g_string_append_printf(str,"\n}\n");
 965         g_mutex_unlock (&mutex);
 966         g_mutex_unlock (&memdb->mutex);
 967         return res;
 968 err:
 969         res = -EIO;
 970         goto ret;
 971 enoent:
 972         res = -ENOENT;
 973         goto ret;
 974 }
 975
 976 void
 977 record_memdb_change(const char *path)
 978 {
 979         g_return_if_fail(cfs_status.memdb_changes != 0);
 980
 981         memdb_change_t *ce;
 982
 983         if ((ce = (memdb_change_t *)g_hash_table_lookup(cfs_status.memdb_changes, path))) {
 984                 ce->version++;
 985         }
 986 }
 987
 988 void
 989 record_memdb_reload(void)
 990 {
 991         for (int i = 0; i < G_N_ELEMENTS(memdb_change_array); i++) {
 992                 memdb_change_array[i].version++;
 993         }
 994 }
 995
 996 static gboolean
 997 kventry_hash_set(
 998         GHashTable *kvhash,
 999         const char *key,
1000         gconstpointer data,
1001         size_t len)
1002 {
1003         g_return_val_if_fail(kvhash != NULL, FALSE);
1004         g_return_val_if_fail(key != NULL, FALSE);
1005         g_return_val_if_fail(data != NULL, FALSE);
1006
1007         kventry_t *entry;
1008         if (!len) {
1009                 g_hash_table_remove(kvhash, key);
1010         } else if ((entry = (kventry_t *)g_hash_table_lookup(kvhash, key))) {
1011                 g_free(entry->data);
1012                 entry->data = g_memdup(data, len);
1013                 entry->len = len;
1014                 entry->version++;
1015         } else {
1016                 kventry_t *entry = g_new0(kventry_t, 1);
1017
1018                 entry->key = g_strdup(key);
1019                 entry->data = g_memdup(data, len);
1020                 entry->len = len;
1021
1022                 g_hash_table_replace(kvhash, entry->key, entry);
1023         }
1024
1025         return TRUE;
1026 }
1027
1028 static const char *rrd_def_node[] = {
1029         "DS:loadavg:GAUGE:120:0:U",
1030         "DS:maxcpu:GAUGE:120:0:U",
1031         "DS:cpu:GAUGE:120:0:U",
1032         "DS:iowait:GAUGE:120:0:U",
1033         "DS:memtotal:GAUGE:120:0:U",
1034         "DS:memused:GAUGE:120:0:U",
1035         "DS:swaptotal:GAUGE:120:0:U",
1036         "DS:swapused:GAUGE:120:0:U",
1037         "DS:roottotal:GAUGE:120:0:U",
1038         "DS:rootused:GAUGE:120:0:U",
1039         "DS:netin:DERIVE:120:0:U",
1040         "DS:netout:DERIVE:120:0:U",
1041
1042         "RRA:AVERAGE:0.5:1:70", // 1 min avg - one hour
1043         "RRA:AVERAGE:0.5:30:70", // 30 min avg - one day
1044         "RRA:AVERAGE:0.5:180:70", // 3 hour avg - one week
1045         "RRA:AVERAGE:0.5:720:70", // 12 hour avg - one month
1046         "RRA:AVERAGE:0.5:10080:70", // 7 day avg - ony year
1047
1048         "RRA:MAX:0.5:1:70", // 1 min max - one hour
1049         "RRA:MAX:0.5:30:70", // 30 min max - one day
1050         "RRA:MAX:0.5:180:70",  // 3 hour max - one week
1051         "RRA:MAX:0.5:720:70", // 12 hour max - one month
1052         "RRA:MAX:0.5:10080:70", // 7 day max - ony year
1053         NULL,
1054 };
1055
1056 static const char *rrd_def_vm[] = {
1057         "DS:maxcpu:GAUGE:120:0:U",
1058         "DS:cpu:GAUGE:120:0:U",
1059         "DS:maxmem:GAUGE:120:0:U",
1060         "DS:mem:GAUGE:120:0:U",
1061         "DS:maxdisk:GAUGE:120:0:U",
1062         "DS:disk:GAUGE:120:0:U",
1063         "DS:netin:DERIVE:120:0:U",
1064         "DS:netout:DERIVE:120:0:U",
1065         "DS:diskread:DERIVE:120:0:U",
1066         "DS:diskwrite:DERIVE:120:0:U",
1067
1068         "RRA:AVERAGE:0.5:1:70", // 1 min avg - one hour
1069         "RRA:AVERAGE:0.5:30:70", // 30 min avg - one day
1070         "RRA:AVERAGE:0.5:180:70", // 3 hour avg - one week
1071         "RRA:AVERAGE:0.5:720:70", // 12 hour avg - one month
1072         "RRA:AVERAGE:0.5:10080:70", // 7 day avg - ony year
1073
1074         "RRA:MAX:0.5:1:70", // 1 min max - one hour
1075         "RRA:MAX:0.5:30:70", // 30 min max - one day
1076         "RRA:MAX:0.5:180:70",  // 3 hour max - one week
1077         "RRA:MAX:0.5:720:70", // 12 hour max - one month
1078         "RRA:MAX:0.5:10080:70", // 7 day max - ony year
1079         NULL,
1080 };
1081
1082 static const char *rrd_def_storage[] = {
1083         "DS:total:GAUGE:120:0:U",
1084         "DS:used:GAUGE:120:0:U",
1085
1086         "RRA:AVERAGE:0.5:1:70", // 1 min avg - one hour
1087         "RRA:AVERAGE:0.5:30:70", // 30 min avg - one day
1088         "RRA:AVERAGE:0.5:180:70", // 3 hour avg - one week
1089         "RRA:AVERAGE:0.5:720:70", // 12 hour avg - one month
1090         "RRA:AVERAGE:0.5:10080:70", // 7 day avg - ony year
1091
1092         "RRA:MAX:0.5:1:70", // 1 min max - one hour
1093         "RRA:MAX:0.5:30:70", // 30 min max - one day
1094         "RRA:MAX:0.5:180:70",  // 3 hour max - one week
1095         "RRA:MAX:0.5:720:70", // 12 hour max - one month
1096         "RRA:MAX:0.5:10080:70", // 7 day max - ony year
1097         NULL,
1098 };
1099
1100 #define RRDDIR "/var/lib/rrdcached/db"
1101
1102 static void
1103 create_rrd_file(
1104         const char *filename,
1105         int argcount,
1106         const char *rrddef[])
1107 {
1108         /* start at day boundary */
1109         time_t ctime;
1110         time(&ctime);
1111         struct tm *ltm = localtime(&ctime);
1112         ltm->tm_sec = 0;
1113         ltm->tm_min = 0;
1114         ltm->tm_hour = 0;
1115
1116         rrd_clear_error();
1117         if (rrd_create_r(filename, 60, timelocal(ltm), argcount, rrddef)) {
1118                 cfs_message("RRD create error %s: %s", filename, rrd_get_error());
1119         }
1120 }
1121
1122 static inline const char *
1123 rrd_skip_data(
1124         const char *data,
1125         int count)
1126 {
1127         int found = 0;
1128         while (*data && found < count) {
1129                 if (*data++ == ':')
1130                         found++;
1131         }
1132         return data;
1133 }
1134
1135 static void
1136 update_rrd_data(
1137         const char *key,
1138         gconstpointer data,
1139         size_t len)
1140 {
1141         g_return_if_fail(key != NULL);
1142         g_return_if_fail(data != NULL);
1143         g_return_if_fail(len > 0);
1144         g_return_if_fail(len < 4096);
1145
1146         static const char *rrdcsock = "unix:/var/run/rrdcached.sock";
1147
1148         int use_daemon = 1;
1149         if (rrdc_connect(rrdcsock) != 0)
1150                 use_daemon = 0;
1151
1152         char *filename = NULL;
1153
1154         int skip = 0;
1155
1156         if (strncmp(key, "pve2-node/", 10) == 0) {
1157                 const char *node = key + 10;
1158
1159                 skip = 2;
1160
1161                 if (strchr(node, '/') != NULL)
1162                         goto keyerror;
1163
1164                 if (strlen(node) < 1)
1165                         goto keyerror;
1166
1167                 filename = g_strdup_printf(RRDDIR "/%s", key);
1168
1169                 if (!g_file_test(filename, G_FILE_TEST_EXISTS)) {
1170
1171                         mkdir(RRDDIR "/pve2-node", 0755);
1172                         int argcount = sizeof(rrd_def_node)/sizeof(void*) - 1;
1173                         create_rrd_file(filename, argcount, rrd_def_node);
1174                 }
1175
1176         } else if ((strncmp(key, "pve2-vm/", 8) == 0) ||
1177                    (strncmp(key, "pve2.3-vm/", 10) == 0)) {
1178                 const char *vmid;
1179
1180                 if (strncmp(key, "pve2-vm/", 8) == 0) {
1181                         vmid = key + 8;
1182                         skip = 2;
1183                 } else {
1184                         vmid = key + 10;
1185                         skip = 4;
1186                 }
1187
1188                 if (strchr(vmid, '/') != NULL)
1189                         goto keyerror;
1190
1191                 if (strlen(vmid) < 1)
1192                         goto keyerror;
1193
1194                 filename = g_strdup_printf(RRDDIR "/%s/%s", "pve2-vm", vmid);
1195
1196                 if (!g_file_test(filename, G_FILE_TEST_EXISTS)) {
1197
1198                         mkdir(RRDDIR "/pve2-vm", 0755);
1199                         int argcount = sizeof(rrd_def_vm)/sizeof(void*) - 1;
1200                         create_rrd_file(filename, argcount, rrd_def_vm);
1201                 }
1202
1203         } else if (strncmp(key, "pve2-storage/", 13) == 0) {
1204                 const char *node = key + 13;
1205
1206                 const char *storage = node;
1207                 while (*storage && *storage != '/')
1208                         storage++;
1209
1210                 if (*storage != '/' || ((storage - node) < 1))
1211                         goto keyerror;
1212
1213                 storage++;
1214
1215                 if (strchr(storage, '/') != NULL)
1216                         goto keyerror;
1217
1218                 if (strlen(storage) < 1)
1219                         goto keyerror;
1220
1221                 filename = g_strdup_printf(RRDDIR "/%s", key);
1222
1223                 if (!g_file_test(filename, G_FILE_TEST_EXISTS)) {
1224
1225                         mkdir(RRDDIR "/pve2-storage", 0755);
1226
1227                         char *dir = g_path_get_dirname(filename);
1228                         mkdir(dir, 0755);
1229                         g_free(dir);
1230
1231                         int argcount = sizeof(rrd_def_storage)/sizeof(void*) - 1;
1232                         create_rrd_file(filename, argcount, rrd_def_storage);
1233                 }
1234
1235         } else {
1236                 goto keyerror;
1237         }
1238
1239         const char *dp = skip ? rrd_skip_data(data, skip) : data;
1240
1241         const char *update_args[] = { dp, NULL };
1242
1243         if (use_daemon) {
1244                 int status;
1245                 if ((status = rrdc_update(filename, 1, update_args)) != 0) {
1246                         cfs_message("RRDC update error %s: %d", filename, status);
1247                         rrdc_disconnect();
1248                         rrd_clear_error();
1249                         if (rrd_update_r(filename, NULL, 1, update_args) != 0) {
1250                                 cfs_message("RRD update error %s: %s", filename, rrd_get_error());
1251                         }
1252                 }
1253
1254         } else {
1255                 rrd_clear_error();
1256                 if (rrd_update_r(filename, NULL, 1, update_args) != 0) {
1257                         cfs_message("RRD update error %s: %s", filename, rrd_get_error());
1258                 }
1259         }
1260
1261 ret:
1262         if (filename)
1263                 g_free(filename);
1264
1265         return;
1266
1267 keyerror:
1268         cfs_critical("RRD update error: unknown/wrong key %s", key);
1269         goto ret;
1270 }
1271
1272 static gboolean
1273 rrd_entry_is_old(
1274         gpointer key,
1275         gpointer value,
1276         gpointer user_data)
1277 {
1278         rrdentry_t *entry = (rrdentry_t *)value;
1279         uint32_t ctime = GPOINTER_TO_UINT(user_data);
1280
1281         int diff = ctime - entry->time;
1282
1283         /* remove everything older than 5 minutes */
1284         int expire = 60*5;
1285
1286         return (diff > expire) ? TRUE : FALSE;
1287 }
1288
1289 static char *rrd_dump_buf = NULL;
1290 static time_t rrd_dump_last = 0;
1291
1292 void
1293 cfs_rrd_dump(GString *str)
1294 {
1295         time_t ctime;
1296
1297         g_mutex_lock (&mutex);
1298
1299         time(&ctime);
1300         if (rrd_dump_buf && (ctime - rrd_dump_last) < 2) {
1301                 g_string_assign(str, rrd_dump_buf);
1302                 g_mutex_unlock (&mutex);
1303                 return;
1304         }
1305
1306         /* remove old data */
1307         g_hash_table_foreach_remove(cfs_status.rrdhash, rrd_entry_is_old,
1308                                     GUINT_TO_POINTER(ctime));
1309
1310         g_string_set_size(str, 0);
1311
1312         GHashTableIter iter;
1313         gpointer key, value;
1314
1315         g_hash_table_iter_init (&iter, cfs_status.rrdhash);
1316
1317         while (g_hash_table_iter_next (&iter, &key, &value)) {
1318                 rrdentry_t *entry = (rrdentry_t *)value;
1319                 g_string_append(str, key);
1320                 g_string_append(str, ":");
1321                 g_string_append(str, entry->data);
1322                 g_string_append(str, "\n");
1323         }
1324
1325         g_string_append_c(str, 0); // never return undef
1326
1327         rrd_dump_last = ctime;
1328         if (rrd_dump_buf)
1329                 g_free(rrd_dump_buf);
1330         rrd_dump_buf = g_strdup(str->str);
1331
1332         g_mutex_unlock (&mutex);
1333 }
1334
1335 static gboolean
1336 nodeip_hash_set(
1337         GHashTable *iphash,
1338         const char *nodename,
1339         const char *ip,
1340         size_t len)
1341 {
1342         g_return_val_if_fail(iphash != NULL, FALSE);
1343         g_return_val_if_fail(nodename != NULL, FALSE);
1344         g_return_val_if_fail(ip != NULL, FALSE);
1345         g_return_val_if_fail(len > 0, FALSE);
1346         g_return_val_if_fail(len < 256, FALSE);
1347         g_return_val_if_fail(ip[len-1] == 0, FALSE);
1348
1349         char *oldip = (char *)g_hash_table_lookup(iphash, nodename);
1350
1351         if (!oldip || (strcmp(oldip, ip) != 0)) {
1352                 cfs_status.clinfo_version++;
1353                 g_hash_table_replace(iphash, g_strdup(nodename), g_strdup(ip));
1354         }
1355
1356         return TRUE;
1357 }
1358
1359 static gboolean
1360 rrdentry_hash_set(
1361         GHashTable *rrdhash,
1362         const char *key,
1363         const char *data,
1364         size_t len)
1365 {
1366         g_return_val_if_fail(rrdhash != NULL, FALSE);
1367         g_return_val_if_fail(key != NULL, FALSE);
1368         g_return_val_if_fail(data != NULL, FALSE);
1369         g_return_val_if_fail(len > 0, FALSE);
1370         g_return_val_if_fail(len < 4096, FALSE);
1371         g_return_val_if_fail(data[len-1] == 0, FALSE);
1372
1373         rrdentry_t *entry;
1374         if ((entry = (rrdentry_t *)g_hash_table_lookup(rrdhash, key))) {
1375                 g_free(entry->data);
1376                 entry->data = g_memdup(data, len);
1377                 entry->len = len;
1378                 entry->time = time(NULL);
1379         } else {
1380                 rrdentry_t *entry = g_new0(rrdentry_t, 1);
1381
1382                 entry->key = g_strdup(key);
1383                 entry->data = g_memdup(data, len);
1384                 entry->len = len;
1385                 entry->time = time(NULL);
1386
1387                 g_hash_table_replace(rrdhash, entry->key, entry);
1388         }
1389
1390         update_rrd_data(key, data, len);
1391
1392         return TRUE;
1393 }
1394
1395 static int
1396 kvstore_send_update_message(
1397         dfsm_t *dfsm,
1398         const char *key,
1399         gpointer data,
1400         guint32 len)
1401 {
1402         if (!dfsm_is_initialized(dfsm))
1403                 return -EACCES;
1404
1405         struct iovec iov[2];
1406
1407         char name[256];
1408         g_strlcpy(name, key, sizeof(name));
1409
1410         iov[0].iov_base = &name;
1411         iov[0].iov_len = sizeof(name);
1412
1413         iov[1].iov_base = (char *)data;
1414         iov[1].iov_len = len;
1415
1416         if (dfsm_send_message(dfsm, KVSTORE_MESSAGE_UPDATE, iov, 2) == CS_OK)
1417                 return 0;
1418
1419         return -EACCES;
1420 }
1421
1422 static clog_entry_t *
1423 kvstore_parse_log_message(
1424         const void *msg,
1425         size_t msg_len)
1426 {
1427         g_return_val_if_fail(msg != NULL, NULL);
1428
1429         if (msg_len < sizeof(clog_entry_t)) {
1430                 cfs_critical("received short log message (%zu < %zu)", msg_len, sizeof(clog_entry_t));
1431                 return NULL;
1432         }
1433
1434         clog_entry_t *entry = (clog_entry_t *)msg;
1435
1436         uint32_t size = sizeof(clog_entry_t) + entry->node_len +
1437                 entry->ident_len + entry->tag_len + entry->msg_len;
1438
1439         if (msg_len != size) {
1440                 cfs_critical("received log message with wrong size (%zu != %u)", msg_len, size);
1441                 return NULL;
1442         }
1443
1444         char *msgptr = entry->data;
1445
1446         if (*((char *)msgptr + entry->node_len - 1)) {
1447                 cfs_critical("unterminated string in log message");
1448                 return NULL;
1449         }
1450         msgptr += entry->node_len;
1451
1452         if (*((char *)msgptr + entry->ident_len - 1)) {
1453                 cfs_critical("unterminated string in log message");
1454                 return NULL;
1455         }
1456         msgptr += entry->ident_len;
1457
1458         if (*((char *)msgptr + entry->tag_len - 1)) {
1459                 cfs_critical("unterminated string in log message");
1460                 return NULL;
1461         }
1462         msgptr += entry->tag_len;
1463
1464         if (*((char *)msgptr + entry->msg_len - 1)) {
1465                 cfs_critical("unterminated string in log message");
1466                 return NULL;
1467         }
1468
1469         return entry;
1470 }
1471
1472 static gboolean
1473 kvstore_parse_update_message(
1474         const void *msg,
1475         size_t msg_len,
1476         const char **key,
1477         gconstpointer *data,
1478         guint32 *len)
1479 {
1480         g_return_val_if_fail(msg != NULL, FALSE);
1481         g_return_val_if_fail(key != NULL, FALSE);
1482         g_return_val_if_fail(data != NULL, FALSE);
1483         g_return_val_if_fail(len != NULL, FALSE);
1484
1485         if (msg_len < 256) {
1486                 cfs_critical("received short kvstore message (%zu < 256)", msg_len);
1487                 return FALSE;
1488         }
1489
1490         /* test if key is null terminated */
1491         int i = 0;
1492         for (i = 0; i < 256; i++)
1493                 if (((char *)msg)[i] == 0)
1494                         break;
1495
1496         if (i == 256)
1497                 return FALSE;
1498
1499
1500         *len = msg_len - 256;
1501         *key = msg;
1502         *data = (char *) msg + 256;
1503
1504         return TRUE;
1505 }
1506
1507 int
1508 cfs_create_status_msg(
1509         GString *str,
1510         const char *nodename,
1511         const char *key)
1512 {
1513         g_return_val_if_fail(str != NULL, -EINVAL);
1514         g_return_val_if_fail(key != NULL, -EINVAL);
1515
1516         int res = -ENOENT;
1517
1518         GHashTable *kvhash = NULL;
1519
1520         g_mutex_lock (&mutex);
1521
1522         if (!nodename || !nodename[0] || !strcmp(nodename, cfs.nodename)) {
1523                 kvhash = cfs_status.kvhash;
1524         } else if (cfs_status.clinfo && cfs_status.clinfo->nodes_byname) {
1525                 cfs_clnode_t *clnode;
1526                 if ((clnode = g_hash_table_lookup(cfs_status.clinfo->nodes_byname, nodename)))
1527                         kvhash = clnode->kvhash;
1528         }
1529
1530         kventry_t *entry;
1531         if (kvhash && (entry = (kventry_t *)g_hash_table_lookup(kvhash, key))) {
1532                 g_string_append_len(str, entry->data, entry->len);
1533                 res = 0;
1534         }
1535
1536         g_mutex_unlock (&mutex);
1537
1538         return res;
1539 }
1540
1541 int
1542 cfs_status_set(
1543         const char *key,
1544         gpointer data,
1545         size_t len)
1546 {
1547         g_return_val_if_fail(key != NULL, FALSE);
1548         g_return_val_if_fail(data != NULL, FALSE);
1549         g_return_val_if_fail(cfs_status.kvhash != NULL, FALSE);
1550
1551         if (len > CFS_MAX_STATUS_SIZE)
1552                 return -EFBIG;
1553
1554         g_mutex_lock (&mutex);
1555
1556         gboolean res;
1557
1558         if (strncmp(key, "rrd/", 4) == 0) {
1559                 res = rrdentry_hash_set(cfs_status.rrdhash, key + 4, data, len);
1560         } else if (!strcmp(key, "nodeip")) {
1561                 res = nodeip_hash_set(cfs_status.iphash, cfs.nodename, data, len);
1562         } else {
1563                 res = kventry_hash_set(cfs_status.kvhash, key, data, len);
1564         }
1565         g_mutex_unlock (&mutex);
1566
1567         if (cfs_status.kvstore)
1568                 kvstore_send_update_message(cfs_status.kvstore, key, data, len);
1569
1570         return res ? 0 : -ENOMEM;
1571 }
1572
1573 gboolean
1574 cfs_kvstore_node_set(
1575         uint32_t nodeid,
1576         const char *key,
1577         gconstpointer data,
1578         size_t len)
1579 {
1580         g_return_val_if_fail(nodeid != 0, FALSE);
1581         g_return_val_if_fail(key != NULL, FALSE);
1582         g_return_val_if_fail(data != NULL, FALSE);
1583
1584         g_mutex_lock (&mutex);
1585
1586         if (!cfs_status.clinfo || !cfs_status.clinfo->nodes_byid)
1587                 goto ret; /* ignore */
1588
1589         cfs_clnode_t *clnode = g_hash_table_lookup(cfs_status.clinfo->nodes_byid, &nodeid);
1590         if (!clnode)
1591                 goto ret; /* ignore */
1592
1593         cfs_debug("got node %d status update %s", nodeid, key);
1594
1595         if (strncmp(key, "rrd/", 4) == 0) {
1596                 rrdentry_hash_set(cfs_status.rrdhash, key + 4, data, len);
1597         } else if (!strcmp(key, "nodeip")) {
1598                 nodeip_hash_set(cfs_status.iphash, clnode->name, data, len);
1599         } else {
1600                 if (!clnode->kvhash) {
1601                         if (!(clnode->kvhash = kventry_hash_new())) {
1602                                 goto ret; /*ignore */
1603                         }
1604                 }
1605
1606                 kventry_hash_set(clnode->kvhash, key, data, len);
1607
1608         }
1609 ret:
1610         g_mutex_unlock (&mutex);
1611
1612         return TRUE;
1613 }
1614
1615 static gboolean
1616 cfs_kvstore_sync(void)
1617 {
1618         g_return_val_if_fail(cfs_status.kvhash != NULL, FALSE);
1619         g_return_val_if_fail(cfs_status.kvstore != NULL, FALSE);
1620
1621         gboolean res = TRUE;
1622
1623         g_mutex_lock (&mutex);
1624
1625         GHashTable *ht = cfs_status.kvhash;
1626         GHashTableIter iter;
1627         gpointer key, value;
1628
1629         g_hash_table_iter_init (&iter, ht);
1630
1631         while (g_hash_table_iter_next (&iter, &key, &value)) {
1632                 kventry_t *entry = (kventry_t *)value;
1633                 kvstore_send_update_message(cfs_status.kvstore, entry->key, entry->data, entry->len);
1634         }
1635
1636         g_mutex_unlock (&mutex);
1637
1638         return res;
1639 }
1640
1641 static int
1642 dfsm_deliver(
1643         dfsm_t *dfsm,
1644         gpointer data,
1645         int *res_ptr,
1646         uint32_t nodeid,
1647         uint32_t pid,
1648         uint16_t msg_type,
1649         uint32_t msg_time,
1650         const void *msg,
1651         size_t msg_len)
1652 {
1653         g_return_val_if_fail(dfsm != NULL, -1);
1654         g_return_val_if_fail(msg != NULL, -1);
1655         g_return_val_if_fail(res_ptr != NULL, -1);
1656
1657         /* ignore message for ourself */
1658         if (dfsm_nodeid_is_local(dfsm, nodeid, pid))
1659                 goto ret;
1660
1661         if (msg_type == KVSTORE_MESSAGE_UPDATE) {
1662                 const char *key;
1663                 gconstpointer data;
1664                 guint32 len;
1665                 if (kvstore_parse_update_message(msg, msg_len, &key, &data, &len)) {
1666                         cfs_kvstore_node_set(nodeid, key, data, len);
1667                 } else {
1668                         cfs_critical("cant parse update message");
1669                 }
1670         } else if (msg_type == KVSTORE_MESSAGE_LOG) {
1671                 cfs_message("received log"); // fixme: remove
1672                 const clog_entry_t *entry;
1673                 if ((entry = kvstore_parse_log_message(msg, msg_len))) {
1674                         clusterlog_insert(cfs_status.clusterlog, entry);
1675                 } else {
1676                         cfs_critical("cant parse log message");
1677                 }
1678         } else {
1679                 cfs_critical("received unknown message type %d\n", msg_type);
1680                 goto fail;
1681         }
1682
1683 ret:
1684         *res_ptr = 0;
1685         return 1;
1686
1687 fail:
1688         *res_ptr = -EACCES;
1689         return 1;
1690 }
1691
1692 static void
1693 dfsm_confchg(
1694         dfsm_t *dfsm,
1695         gpointer data,
1696         const struct cpg_address *member_list,
1697         size_t member_list_entries)
1698 {
1699         g_return_if_fail(dfsm != NULL);
1700         g_return_if_fail(member_list != NULL);
1701
1702         cfs_debug("enter %s", __func__);
1703
1704         g_mutex_lock (&mutex);
1705
1706         cfs_clinfo_t *clinfo = cfs_status.clinfo;
1707
1708         if (clinfo && clinfo->nodes_byid) {
1709
1710                 GHashTable *ht = clinfo->nodes_byid;
1711                 GHashTableIter iter;
1712                 gpointer key, value;
1713
1714                 g_hash_table_iter_init (&iter, ht);
1715
1716                 while (g_hash_table_iter_next (&iter, &key, &value)) {
1717                         cfs_clnode_t *node = (cfs_clnode_t *)value;
1718                         node->online = FALSE;
1719                 }
1720
1721                 for (int i = 0; i < member_list_entries; i++) {
1722                         cfs_clnode_t *node;
1723                         if ((node = g_hash_table_lookup(clinfo->nodes_byid, &member_list[i].nodeid))) {
1724                                 node->online = TRUE;
1725                         }
1726                 }
1727
1728                 cfs_status.clinfo_version++;
1729         }
1730
1731         g_mutex_unlock (&mutex);
1732 }
1733
1734 static gpointer
1735 dfsm_get_state(
1736         dfsm_t *dfsm,
1737         gpointer data,
1738         unsigned int *res_len)
1739 {
1740         g_return_val_if_fail(dfsm != NULL, NULL);
1741
1742         gpointer msg = clusterlog_get_state(cfs_status.clusterlog, res_len);
1743
1744         return msg;
1745 }
1746
1747 static int
1748 dfsm_process_update(
1749         dfsm_t *dfsm,
1750         gpointer data,
1751         dfsm_sync_info_t *syncinfo,
1752         uint32_t nodeid,
1753         uint32_t pid,
1754         const void *msg,
1755         size_t msg_len)
1756 {
1757         cfs_critical("%s: received unexpected update message", __func__);
1758
1759         return -1;
1760 }
1761
1762 static int
1763 dfsm_process_state_update(
1764         dfsm_t *dfsm,
1765         gpointer data,
1766         dfsm_sync_info_t *syncinfo)
1767 {
1768         g_return_val_if_fail(dfsm != NULL, -1);
1769         g_return_val_if_fail(syncinfo != NULL, -1);
1770
1771         clog_base_t *clog[syncinfo->node_count];
1772
1773         int local_index = -1;
1774         for (int i = 0; i < syncinfo->node_count; i++) {
1775                 dfsm_node_info_t *ni = &syncinfo->nodes[i];
1776                 ni->synced = 1;
1777
1778                 if (syncinfo->local == ni)
1779                         local_index = i;
1780
1781                 clog_base_t *base = (clog_base_t *)ni->state;
1782                 if (ni->state_len > 8 && ni->state_len == clog_size(base)) {
1783                         clog[i] = ni->state;
1784                 } else {
1785                         cfs_critical("received log with wrong size %u", ni->state_len);
1786                         clog[i] = NULL;
1787                 }
1788         }
1789
1790         if (!clusterlog_merge(cfs_status.clusterlog, clog, syncinfo->node_count, local_index)) {
1791                 cfs_critical("unable to merge log files");
1792         }
1793
1794         cfs_kvstore_sync();
1795
1796         return 1;
1797 }
1798
1799 static int
1800 dfsm_commit(
1801         dfsm_t *dfsm,
1802         gpointer data,
1803         dfsm_sync_info_t *syncinfo)
1804 {
1805         g_return_val_if_fail(dfsm != NULL, -1);
1806         g_return_val_if_fail(syncinfo != NULL, -1);
1807
1808         return 1;
1809 }
1810
1811 static void
1812 dfsm_synced(dfsm_t *dfsm)
1813 {
1814         g_return_if_fail(dfsm != NULL);
1815
1816         char *ip = (char *)g_hash_table_lookup(cfs_status.iphash, cfs.nodename);
1817         if (!ip)
1818                 ip = cfs.ip;
1819
1820         cfs_status_set("nodeip", ip, strlen(ip) + 1);
1821 }
1822
1823 static int
1824 dfsm_cleanup(
1825         dfsm_t *dfsm,
1826         gpointer data,
1827         dfsm_sync_info_t *syncinfo)
1828 {
1829         return 1;
1830 }
1831
1832 static dfsm_callbacks_t kvstore_dfsm_callbacks = {
1833         .dfsm_deliver_fn = dfsm_deliver,
1834         .dfsm_confchg_fn = dfsm_confchg,
1835
1836         .dfsm_get_state_fn = dfsm_get_state,
1837         .dfsm_process_state_update_fn = dfsm_process_state_update,
1838         .dfsm_process_update_fn = dfsm_process_update,
1839         .dfsm_commit_fn = dfsm_commit,
1840         .dfsm_cleanup_fn = dfsm_cleanup,
1841         .dfsm_synced_fn = dfsm_synced,
1842 };
1843
1844 dfsm_t *
1845 cfs_status_dfsm_new(void)
1846 {
1847         g_mutex_lock (&mutex);
1848
1849         cfs_status.kvstore = dfsm_new(NULL, KVSTORE_CPG_GROUP_NAME, G_LOG_DOMAIN,
1850                                       0, &kvstore_dfsm_callbacks);
1851         g_mutex_unlock (&mutex);
1852
1853         return cfs_status.kvstore;
1854 }
1855
1856 gboolean
1857 cfs_is_quorate(void)
1858 {
1859         g_mutex_lock (&mutex);
1860         gboolean res =  cfs_status.quorate;
1861         g_mutex_unlock (&mutex);
1862
1863         return res;
1864 }
1865
1866 void
1867 cfs_set_quorate(
1868         uint32_t quorate,
1869         gboolean quiet)
1870 {
1871         g_mutex_lock (&mutex);
1872
1873         uint32_t prev_quorate = cfs_status.quorate;
1874         cfs_status.quorate = quorate;
1875
1876         if (!prev_quorate && cfs_status.quorate) {
1877                 if (!quiet)
1878                         cfs_message("node has quorum");
1879         }
1880
1881         if (prev_quorate && !cfs_status.quorate) {
1882                 if (!quiet)
1883                         cfs_message("node lost quorum");
1884         }
1885
1886         g_mutex_unlock (&mutex);
1887 }