data/src/status.c

   1 /*
   2   Copyright (C) 2010 - 2020 Proxmox Server Solutions GmbH
   3
   4   This program is free software: you can redistribute it and/or modify
   5   it under the terms of the GNU Affero General Public License as published by
   6   the Free Software Foundation, either version 3 of the License, or
   7   (at your option) any later version.
   8
   9   This program is distributed in the hope that it will be useful,
  10   but WITHOUT ANY WARRANTY; without even the implied warranty of
  11   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12   GNU Affero General Public License for more details.
  13
  14   You should have received a copy of the GNU Affero General Public License
  15   along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17   Author: Dietmar Maurer <dietmar@proxmox.com>
  18
  19 */
  20
  21 #define G_LOG_DOMAIN "status"
  22
  23 #ifdef HAVE_CONFIG_H
  24 #include <config.h>
  25 #endif /* HAVE_CONFIG_H */
  26
  27 #include <stdio.h>
  28 #include <stdint.h>
  29 #include <string.h>
  30 #include <errno.h>
  31 #include <glib.h>
  32 #include <sys/syslog.h>
  33 #include <rrd.h>
  34 #include <rrd_client.h>
  35 #include <time.h>
  36 #include <ctype.h>
  37
  38 #include "cfs-utils.h"
  39 #include "status.h"
  40 #include "memdb.h"
  41 #include "logger.h"
  42
  43 #define KVSTORE_CPG_GROUP_NAME "pve_kvstore_v1"
  44
  45 typedef enum {
  46         KVSTORE_MESSAGE_UPDATE = 1,
  47         KVSTORE_MESSAGE_UPDATE_COMPLETE = 2,
  48         KVSTORE_MESSAGE_LOG = 3,
  49 } kvstore_message_t;
  50
  51 static uint32_t vminfo_version_counter;
  52
  53 typedef struct {
  54         uint32_t vmid;
  55         char *nodename;
  56         int vmtype;
  57         uint32_t version;
  58 } vminfo_t;
  59
  60 typedef struct {
  61         char *key;
  62         gpointer data;
  63         size_t len;
  64         uint32_t version;
  65 } kventry_t;
  66
  67 typedef struct {
  68         char *key;
  69         gpointer data;
  70         size_t len;
  71         uint32_t time;
  72 } rrdentry_t;
  73
  74 typedef struct {
  75         char *path;
  76         uint32_t version;
  77 } memdb_change_t;
  78
  79 static memdb_change_t memdb_change_array[] = {
  80         { .path = "corosync.conf" },
  81         { .path = "corosync.conf.new" },
  82         { .path = "storage.cfg" },
  83         { .path = "user.cfg" },
  84         { .path = "domains.cfg" },
  85         { .path = "priv/shadow.cfg" },
  86         { .path = "priv/acme/plugins.cfg" },
  87         { .path = "priv/tfa.cfg" },
  88         { .path = "priv/token.cfg" },
  89         { .path = "priv/ipam.db" },
  90         { .path = "datacenter.cfg" },
  91         { .path = "vzdump.cron" },
  92         { .path = "vzdump.conf" },
  93         { .path = "jobs.cfg" },
  94         { .path = "ha/crm_commands" },
  95         { .path = "ha/manager_status" },
  96         { .path = "ha/resources.cfg" },
  97         { .path = "ha/groups.cfg" },
  98         { .path = "ha/fence.cfg" },
  99         { .path = "status.cfg" },
 100         { .path = "replication.cfg" },
 101         { .path = "ceph.conf" },
 102         { .path = "sdn/vnets.cfg" },
 103         { .path = "sdn/zones.cfg" },
 104         { .path = "sdn/controllers.cfg" },
 105         { .path = "sdn/subnets.cfg" },
 106         { .path = "sdn/ipams.cfg" },
 107         { .path = "sdn/dns.cfg" },
 108         { .path = "sdn/.running-config" },
 109         { .path = "virtual-guest/cpu-models.conf" },
 110         { .path = "firewall/cluster.fw" },
 111 };
 112
 113 static GMutex mutex;
 114
 115 typedef struct {
 116         time_t start_time;
 117
 118         uint32_t quorate;
 119
 120         cfs_clinfo_t *clinfo;
 121         uint32_t clinfo_version;
 122
 123         GHashTable *vmlist;
 124         uint32_t vmlist_version;
 125
 126         dfsm_t *kvstore;
 127         GHashTable *kvhash;
 128         GHashTable *rrdhash;
 129         GHashTable *iphash;
 130
 131         GHashTable *memdb_changes;
 132
 133         clusterlog_t *clusterlog;
 134 } cfs_status_t;
 135
 136 static cfs_status_t cfs_status;
 137
 138 struct cfs_clnode {
 139         char *name;
 140         uint32_t nodeid;
 141         uint32_t votes;
 142         gboolean online;
 143         GHashTable *kvhash;
 144 };
 145
 146 struct cfs_clinfo {
 147         char *cluster_name;
 148         uint32_t cman_version;
 149
 150         GHashTable *nodes_byid;
 151         GHashTable *nodes_byname;
 152 };
 153
 154 static guint
 155 g_int32_hash (gconstpointer v)
 156 {
 157         return *(const uint32_t *) v;
 158 }
 159
 160 static gboolean
 161 g_int32_equal (gconstpointer v1,
 162                gconstpointer v2)
 163 {
 164         return *((const uint32_t*) v1) == *((const uint32_t*) v2);
 165 }
 166
 167 static void vminfo_free(vminfo_t *vminfo)
 168 {
 169         g_return_if_fail(vminfo != NULL);
 170
 171         if (vminfo->nodename)
 172                 g_free(vminfo->nodename);
 173
 174
 175         g_free(vminfo);
 176 }
 177
 178 static const char *vminfo_type_to_string(vminfo_t *vminfo)
 179 {
 180         if (vminfo->vmtype == VMTYPE_QEMU) {
 181                 return "qemu";
 182         } else if (vminfo->vmtype == VMTYPE_OPENVZ) {
 183                 // FIXME: remove openvz stuff for 7.x
 184                 return "openvz";
 185         } else if (vminfo->vmtype == VMTYPE_LXC) {
 186                 return "lxc";
 187         } else {
 188                 return "unknown";
 189         }
 190 }
 191
 192 static const char *vminfo_type_to_path_type(vminfo_t *vminfo)
 193 {
 194         if (vminfo->vmtype == VMTYPE_QEMU) {
 195                 return "qemu-server"; // special case..
 196         } else {
 197                 return vminfo_type_to_string(vminfo);
 198         }
 199 }
 200
 201 int vminfo_to_path(vminfo_t *vminfo, GString *path)
 202 {
 203         g_return_val_if_fail(vminfo != NULL, -1);
 204         g_return_val_if_fail(path != NULL, -1);
 205
 206         if (!vminfo->nodename)
 207                 return 0;
 208
 209         const char *type = vminfo_type_to_path_type(vminfo);
 210         g_string_printf(path, "/nodes/%s/%s/%u.conf", vminfo->nodename, type, vminfo->vmid);
 211
 212         return 1;
 213 }
 214
 215 void cfs_clnode_destroy(
 216         cfs_clnode_t *clnode)
 217 {
 218         g_return_if_fail(clnode != NULL);
 219
 220         if (clnode->kvhash)
 221                 g_hash_table_destroy(clnode->kvhash);
 222
 223         if (clnode->name)
 224                 g_free(clnode->name);
 225
 226         g_free(clnode);
 227 }
 228
 229 cfs_clnode_t *cfs_clnode_new(
 230         const char *name,
 231         uint32_t nodeid,
 232         uint32_t votes)
 233 {
 234         g_return_val_if_fail(name != NULL, NULL);
 235
 236         cfs_clnode_t *clnode = g_new0(cfs_clnode_t, 1);
 237         if (!clnode)
 238                 return NULL;
 239
 240         clnode->name = g_strdup(name);
 241         clnode->nodeid = nodeid;
 242         clnode->votes = votes;
 243
 244         return clnode;
 245 }
 246
 247 gboolean cfs_clinfo_destroy(
 248         cfs_clinfo_t *clinfo)
 249 {
 250         g_return_val_if_fail(clinfo != NULL, FALSE);
 251
 252         if (clinfo->cluster_name)
 253                 g_free(clinfo->cluster_name);
 254
 255         if (clinfo->nodes_byname)
 256                 g_hash_table_destroy(clinfo->nodes_byname);
 257
 258         if (clinfo->nodes_byid)
 259                 g_hash_table_destroy(clinfo->nodes_byid);
 260
 261         g_free(clinfo);
 262
 263         return TRUE;
 264 }
 265
 266 cfs_clinfo_t *cfs_clinfo_new(
 267         const char *cluster_name,
 268         uint32_t cman_version)
 269 {
 270         g_return_val_if_fail(cluster_name != NULL, NULL);
 271
 272         cfs_clinfo_t *clinfo = g_new0(cfs_clinfo_t, 1);
 273         if (!clinfo)
 274                 return NULL;
 275
 276         clinfo->cluster_name = g_strdup(cluster_name);
 277         clinfo->cman_version = cman_version;
 278
 279         if (!(clinfo->nodes_byid = g_hash_table_new_full(
 280                       g_int32_hash, g_int32_equal, NULL,
 281                       (GDestroyNotify)cfs_clnode_destroy)))
 282                 goto fail;
 283
 284         if (!(clinfo->nodes_byname = g_hash_table_new(g_str_hash, g_str_equal)))
 285                 goto fail;
 286
 287         return clinfo;
 288
 289 fail:
 290         cfs_clinfo_destroy(clinfo);
 291
 292         return NULL;
 293 }
 294
 295 gboolean cfs_clinfo_add_node(
 296         cfs_clinfo_t *clinfo,
 297         cfs_clnode_t *clnode)
 298 {
 299         g_return_val_if_fail(clinfo != NULL, FALSE);
 300         g_return_val_if_fail(clnode != NULL, FALSE);
 301
 302         g_hash_table_replace(clinfo->nodes_byid, &clnode->nodeid, clnode);
 303         g_hash_table_replace(clinfo->nodes_byname, clnode->name, clnode);
 304
 305         return TRUE;
 306 }
 307
 308 int
 309 cfs_create_memberlist_msg(
 310         GString *str)
 311 {
 312         g_return_val_if_fail(str != NULL, -EINVAL);
 313
 314         g_mutex_lock (&mutex);
 315
 316         g_string_append_printf(str,"{\n");
 317
 318         guint nodecount = 0;
 319
 320         cfs_clinfo_t *clinfo = cfs_status.clinfo;
 321
 322         if (clinfo && clinfo->nodes_byid)
 323                 nodecount = g_hash_table_size(clinfo->nodes_byid);
 324
 325         if (nodecount) {
 326                 g_string_append_printf(str, "\"nodename\": \"%s\",\n", cfs.nodename);
 327                 g_string_append_printf(str, "\"version\": %u,\n", cfs_status.clinfo_version);
 328
 329                 g_string_append_printf(str, "\"cluster\": { ");
 330                 g_string_append_printf(str, "\"name\": \"%s\", \"version\": %d, "
 331                                        "\"nodes\": %d, \"quorate\": %d ",
 332                                        clinfo->cluster_name, clinfo->cman_version,
 333                                        nodecount, cfs_status.quorate);
 334
 335                 g_string_append_printf(str,"},\n");
 336                 g_string_append_printf(str,"\"nodelist\": {\n");
 337
 338                 GHashTable *ht = clinfo->nodes_byid;
 339                 GHashTableIter iter;
 340                 gpointer key, value;
 341
 342                 g_hash_table_iter_init (&iter, ht);
 343
 344                 int i = 0;
 345                 while (g_hash_table_iter_next (&iter, &key, &value)) {
 346                         cfs_clnode_t *node = (cfs_clnode_t *)value;
 347                         if (i) g_string_append_printf(str, ",\n");
 348                         i++;
 349
 350                         g_string_append_printf(str, "  \"%s\": { \"id\": %d, \"online\": %d",
 351                                                node->name, node->nodeid, node->online);
 352
 353
 354                         char *ip = (char *)g_hash_table_lookup(cfs_status.iphash, node->name);
 355                         if (ip) {
 356                                 g_string_append_printf(str, ", \"ip\": \"%s\"", ip);
 357                         }
 358
 359                         g_string_append_printf(str, "}");
 360
 361                 }
 362                 g_string_append_printf(str,"\n  }\n");
 363         } else {
 364                 g_string_append_printf(str, "\"nodename\": \"%s\",\n", cfs.nodename);
 365                 g_string_append_printf(str, "\"version\": %u\n", cfs_status.clinfo_version);
 366         }
 367
 368         g_string_append_printf(str,"}\n");
 369
 370         g_mutex_unlock (&mutex);
 371
 372         return 0;
 373 }
 374
 375 static void
 376 kventry_free(kventry_t *entry)
 377 {
 378         g_return_if_fail(entry != NULL);
 379
 380         g_free(entry->key);
 381         g_free(entry->data);
 382         g_free(entry);
 383 }
 384
 385 static GHashTable *
 386 kventry_hash_new(void)
 387 {
 388         return g_hash_table_new_full(g_str_hash, g_str_equal, NULL,
 389                                      (GDestroyNotify)kventry_free);
 390 }
 391
 392 static void
 393 rrdentry_free(rrdentry_t *entry)
 394 {
 395         g_return_if_fail(entry != NULL);
 396
 397         g_free(entry->key);
 398         g_free(entry->data);
 399         g_free(entry);
 400 }
 401
 402 static GHashTable *
 403 rrdentry_hash_new(void)
 404 {
 405         return g_hash_table_new_full(g_str_hash, g_str_equal, NULL,
 406                                      (GDestroyNotify)rrdentry_free);
 407 }
 408
 409 void
 410 cfs_cluster_log_dump(GString *str, const char *user, guint max_entries)
 411 {
 412         clusterlog_dump(cfs_status.clusterlog, str, user, max_entries);
 413 }
 414
 415 void
 416 cfs_cluster_log(clog_entry_t *entry)
 417 {
 418         g_return_if_fail(entry != NULL);
 419
 420         clusterlog_insert(cfs_status.clusterlog, entry);
 421
 422         if (cfs_status.kvstore) {
 423                 struct iovec iov[1];
 424                 iov[0].iov_base = (char *)entry;
 425                 iov[0].iov_len = clog_entry_size(entry);
 426
 427                 if (dfsm_is_initialized(cfs_status.kvstore))
 428                         dfsm_send_message(cfs_status.kvstore, KVSTORE_MESSAGE_LOG, iov, 1);
 429         }
 430 }
 431
 432 void cfs_status_init(void)
 433 {
 434         g_mutex_lock (&mutex);
 435
 436         cfs_status.start_time = time(NULL);
 437
 438         cfs_status.vmlist = vmlist_hash_new();
 439
 440         cfs_status.kvhash = kventry_hash_new();
 441
 442         cfs_status.rrdhash = rrdentry_hash_new();
 443
 444         cfs_status.iphash = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, g_free);
 445
 446         cfs_status.memdb_changes = g_hash_table_new(g_str_hash, g_str_equal);
 447
 448         for (int i = 0; i < G_N_ELEMENTS(memdb_change_array); i++) {
 449                 g_hash_table_replace(cfs_status.memdb_changes,
 450                                      memdb_change_array[i].path,
 451                                      &memdb_change_array[i]);
 452         }
 453
 454         cfs_status.clusterlog = clusterlog_new();
 455
 456         // fixme:
 457         clusterlog_add(cfs_status.clusterlog, "root", "cluster", getpid(),
 458                        LOG_INFO, "starting cluster log");
 459
 460         g_mutex_unlock (&mutex);
 461 }
 462
 463 void cfs_status_cleanup(void)
 464 {
 465         g_mutex_lock (&mutex);
 466
 467         cfs_status.clinfo_version++;
 468
 469         if (cfs_status.clinfo) {
 470                 cfs_clinfo_destroy(cfs_status.clinfo);
 471                 cfs_status.clinfo = NULL;
 472         }
 473
 474         if (cfs_status.vmlist) {
 475                 g_hash_table_destroy(cfs_status.vmlist);
 476                 cfs_status.vmlist = NULL;
 477         }
 478
 479         if (cfs_status.kvhash) {
 480                 g_hash_table_destroy(cfs_status.kvhash);
 481                 cfs_status.kvhash = NULL;
 482         }
 483
 484         if (cfs_status.rrdhash) {
 485                 g_hash_table_destroy(cfs_status.rrdhash);
 486                 cfs_status.rrdhash = NULL;
 487         }
 488
 489         if (cfs_status.iphash) {
 490                 g_hash_table_destroy(cfs_status.iphash);
 491                 cfs_status.iphash = NULL;
 492         }
 493
 494         if (cfs_status.clusterlog)
 495                 clusterlog_destroy(cfs_status.clusterlog);
 496
 497         g_mutex_unlock (&mutex);
 498 }
 499
 500 void cfs_status_set_clinfo(
 501         cfs_clinfo_t *clinfo)
 502 {
 503         g_return_if_fail(clinfo != NULL);
 504
 505         g_mutex_lock (&mutex);
 506
 507         cfs_status.clinfo_version++;
 508
 509         cfs_clinfo_t *old = cfs_status.clinfo;
 510
 511         cfs_status.clinfo = clinfo;
 512
 513         cfs_message("update cluster info (cluster name  %s, version = %d)",
 514                     clinfo->cluster_name, clinfo->cman_version);
 515
 516
 517         if (old && old->nodes_byid && clinfo->nodes_byid) {
 518                 /* copy kvstore */
 519                 GHashTable *ht = clinfo->nodes_byid;
 520                 GHashTableIter iter;
 521                 gpointer key, value;
 522
 523                 g_hash_table_iter_init (&iter, ht);
 524
 525                 while (g_hash_table_iter_next (&iter, &key, &value)) {
 526                         cfs_clnode_t *node = (cfs_clnode_t *)value;
 527                         cfs_clnode_t *oldnode;
 528                         if ((oldnode = g_hash_table_lookup(old->nodes_byid, key))) {
 529                                 node->online = oldnode->online;
 530                                 node->kvhash = oldnode->kvhash;
 531                                 oldnode->kvhash = NULL;
 532                         }
 533                 }
 534
 535         }
 536
 537         if (old)
 538                 cfs_clinfo_destroy(old);
 539
 540
 541         g_mutex_unlock (&mutex);
 542 }
 543
 544 static void
 545 dump_kvstore_versions(
 546         GString *str,
 547         GHashTable *kvhash,
 548         const char *nodename)
 549 {
 550         g_return_if_fail(kvhash != NULL);
 551         g_return_if_fail(str != NULL);
 552         g_return_if_fail(nodename != NULL);
 553
 554         GHashTable *ht = kvhash;
 555         GHashTableIter iter;
 556         gpointer key, value;
 557
 558         g_string_append_printf(str, "\"%s\": {\n", nodename);
 559
 560         g_hash_table_iter_init (&iter, ht);
 561
 562         int i = 0;
 563         while (g_hash_table_iter_next (&iter, &key, &value)) {
 564                 kventry_t *entry = (kventry_t *)value;
 565                 if (i) g_string_append_printf(str, ",\n");
 566                 i++;
 567                 g_string_append_printf(str,"\"%s\": %u", entry->key, entry->version);
 568         }
 569
 570         g_string_append_printf(str, "}\n");
 571 }
 572
 573 int
 574 cfs_create_version_msg(GString *str)
 575 {
 576         g_return_val_if_fail(str != NULL, -EINVAL);
 577
 578         g_mutex_lock (&mutex);
 579
 580         g_string_append_printf(str,"{\n");
 581
 582         g_string_append_printf(str, "\"starttime\": %lu,\n", (unsigned long)cfs_status.start_time);
 583
 584         g_string_append_printf(str, "\"clinfo\": %u,\n", cfs_status.clinfo_version);
 585
 586         g_string_append_printf(str, "\"vmlist\": %u,\n", cfs_status.vmlist_version);
 587
 588         for (int i = 0; i < G_N_ELEMENTS(memdb_change_array); i++) {
 589                 g_string_append_printf(str, "\"%s\": %u,\n",
 590                                        memdb_change_array[i].path,
 591                                        memdb_change_array[i].version);
 592         }
 593
 594         g_string_append_printf(str, "\"kvstore\": {\n");
 595
 596         dump_kvstore_versions(str, cfs_status.kvhash, cfs.nodename);
 597
 598         cfs_clinfo_t *clinfo = cfs_status.clinfo;
 599
 600         if (clinfo && clinfo->nodes_byid) {
 601                 GHashTable *ht = clinfo->nodes_byid;
 602                 GHashTableIter iter;
 603                 gpointer key, value;
 604
 605                 g_hash_table_iter_init (&iter, ht);
 606
 607                 while (g_hash_table_iter_next (&iter, &key, &value)) {
 608                         cfs_clnode_t *node = (cfs_clnode_t *)value;
 609                         if (!node->kvhash)
 610                                 continue;
 611                         g_string_append_printf(str, ",\n");
 612                         dump_kvstore_versions(str, node->kvhash, node->name);
 613                 }
 614         }
 615
 616         g_string_append_printf(str,"}\n");
 617
 618         g_string_append_printf(str,"}\n");
 619
 620         g_mutex_unlock (&mutex);
 621
 622         return 0;
 623 }
 624
 625 GHashTable *
 626 vmlist_hash_new(void)
 627 {
 628         return g_hash_table_new_full(g_int_hash, g_int_equal, NULL,
 629                                      (GDestroyNotify)vminfo_free);
 630 }
 631
 632 gboolean
 633 vmlist_hash_insert_vm(
 634         GHashTable *vmlist,
 635         int vmtype,
 636         guint32 vmid,
 637         const char *nodename,
 638         gboolean replace)
 639 {
 640         g_return_val_if_fail(vmlist != NULL, FALSE);
 641         g_return_val_if_fail(nodename != NULL, FALSE);
 642         g_return_val_if_fail(vmid != 0, FALSE);
 643         // FIXME: remove openvz stuff for 7.x
 644         g_return_val_if_fail(vmtype == VMTYPE_QEMU || vmtype == VMTYPE_OPENVZ ||
 645                              vmtype == VMTYPE_LXC, FALSE);
 646
 647         if (!replace && g_hash_table_lookup(vmlist, &vmid)) {
 648                 cfs_critical("detected duplicate VMID %d", vmid);
 649                 return FALSE;
 650         }
 651
 652         vminfo_t *vminfo = g_new0(vminfo_t, 1);
 653
 654         vminfo->vmid = vmid;
 655         vminfo->vmtype = vmtype;
 656         vminfo->nodename = g_strdup(nodename);
 657
 658         vminfo->version = ++vminfo_version_counter;
 659
 660         g_hash_table_replace(vmlist, &vminfo->vmid, vminfo);
 661
 662         return TRUE;
 663 }
 664
 665 void
 666 vmlist_register_vm(
 667         int vmtype,
 668         guint32 vmid,
 669         const char *nodename)
 670 {
 671         g_return_if_fail(cfs_status.vmlist != NULL);
 672         g_return_if_fail(nodename != NULL);
 673         g_return_if_fail(vmid != 0);
 674         // FIXME: remove openvz stuff for 7.x
 675         g_return_if_fail(vmtype == VMTYPE_QEMU || vmtype == VMTYPE_OPENVZ ||
 676                          vmtype == VMTYPE_LXC);
 677
 678         cfs_debug("vmlist_register_vm: %s/%u %d", nodename, vmid, vmtype);
 679
 680         g_mutex_lock (&mutex);
 681
 682         cfs_status.vmlist_version++;
 683
 684         vmlist_hash_insert_vm(cfs_status.vmlist, vmtype, vmid, nodename, TRUE);
 685
 686         g_mutex_unlock (&mutex);
 687 }
 688
 689 gboolean
 690 vmlist_different_vm_exists(
 691         int vmtype,
 692         guint32 vmid,
 693         const char *nodename)
 694 {
 695         g_return_val_if_fail(cfs_status.vmlist != NULL, FALSE);
 696         g_return_val_if_fail(vmid != 0, FALSE);
 697
 698         gboolean res = FALSE;
 699
 700         g_mutex_lock (&mutex);
 701
 702         vminfo_t *vminfo;
 703         if ((vminfo = (vminfo_t *)g_hash_table_lookup(cfs_status.vmlist, &vmid))) {
 704                 if (!(vminfo->vmtype == vmtype && strcmp(vminfo->nodename, nodename) == 0))
 705                         res = TRUE;
 706         }
 707         g_mutex_unlock (&mutex);
 708
 709         return res;
 710 }
 711
 712 gboolean
 713 vmlist_vm_exists(
 714         guint32 vmid)
 715 {
 716         g_return_val_if_fail(cfs_status.vmlist != NULL, FALSE);
 717         g_return_val_if_fail(vmid != 0, FALSE);
 718
 719         g_mutex_lock (&mutex);
 720
 721         gpointer res = g_hash_table_lookup(cfs_status.vmlist, &vmid);
 722
 723         g_mutex_unlock (&mutex);
 724
 725         return res != NULL;
 726 }
 727
 728 void
 729 vmlist_delete_vm(
 730         guint32 vmid)
 731 {
 732         g_return_if_fail(cfs_status.vmlist != NULL);
 733         g_return_if_fail(vmid != 0);
 734
 735         g_mutex_lock (&mutex);
 736
 737         cfs_status.vmlist_version++;
 738
 739         g_hash_table_remove(cfs_status.vmlist, &vmid);
 740
 741         g_mutex_unlock (&mutex);
 742 }
 743
 744 void cfs_status_set_vmlist(
 745         GHashTable *vmlist)
 746 {
 747         g_return_if_fail(vmlist != NULL);
 748
 749         g_mutex_lock (&mutex);
 750
 751         cfs_status.vmlist_version++;
 752
 753         if (cfs_status.vmlist)
 754                 g_hash_table_destroy(cfs_status.vmlist);
 755
 756         cfs_status.vmlist = vmlist;
 757
 758         g_mutex_unlock (&mutex);
 759 }
 760
 761 int
 762 cfs_create_vmlist_msg(GString *str)
 763 {
 764         g_return_val_if_fail(cfs_status.vmlist != NULL, -EINVAL);
 765         g_return_val_if_fail(str != NULL, -EINVAL);
 766
 767         g_mutex_lock (&mutex);
 768
 769         g_string_append_printf(str,"{\n");
 770
 771         GHashTable *ht = cfs_status.vmlist;
 772
 773         guint count = g_hash_table_size(ht);
 774
 775         if (!count) {
 776                 g_string_append_printf(str,"\"version\": %u\n", cfs_status.vmlist_version);
 777         } else {
 778                 g_string_append_printf(str,"\"version\": %u,\n", cfs_status.vmlist_version);
 779
 780                 g_string_append_printf(str,"\"ids\": {\n");
 781
 782                 GHashTableIter iter;
 783                 gpointer key, value;
 784
 785                 g_hash_table_iter_init (&iter, ht);
 786
 787                 int first = 1;
 788                 while (g_hash_table_iter_next (&iter, &key, &value)) {
 789                         vminfo_t *vminfo = (vminfo_t *)value;
 790                         const char *type = vminfo_type_to_string(vminfo);
 791
 792                         if (!first)
 793                                 g_string_append_printf(str, ",\n");
 794                         first = 0;
 795
 796                         g_string_append_printf(str,"\"%u\": { \"node\": \"%s\", \"type\": \"%s\", \"version\": %u }",
 797                                                vminfo->vmid, vminfo->nodename, type, vminfo->version);
 798                 }
 799
 800                 g_string_append_printf(str,"}\n");
 801         }
 802         g_string_append_printf(str,"\n}\n");
 803
 804         g_mutex_unlock (&mutex);
 805
 806         return 0;
 807 }
 808
 809 // checks if a config line starts with the given prop. if yes, writes a '\0'
 810 // at the end of the value, and returns the pointer where the value starts
 811 // note: line[line_end] needs to be guaranteed a null byte
 812 char *
 813 _get_property_value_from_line(char *line, size_t line_len, const char *prop, size_t prop_len)
 814 {
 815         if (line_len <= prop_len + 1) return NULL;
 816
 817         if (line[prop_len] == ':' && memcmp(line, prop, prop_len) == 0) { // found
 818                 char *v_start = &line[prop_len + 1];
 819                 char *v_end = &line[line_len - 1];
 820
 821                 // drop initial value whitespaces here already
 822                 while (v_start < v_end && *v_start && isspace(*v_start)) v_start++;
 823
 824                 if (!*v_start) return NULL;
 825
 826                 while (v_end > v_start && isspace(*v_end)) v_end--;
 827                 if (v_end < &line[line_len - 1]) {
 828                     v_end[1] = '\0';
 829                 }
 830
 831                 return v_start;
 832         }
 833
 834         return NULL;
 835 }
 836
 837 // checks the conf for lines starting with the given props and
 838 // writes the pointers into the correct positions into the 'found' array
 839 // afterwards, without initial whitespace(s), we only deal with the format
 840 // restriction imposed by our perl VM config parser, main reference is
 841 // PVE::QemuServer::parse_vm_config this allows to be very fast and still
 842 // relatively simple
 843 // main restrictions used for our advantage is the properties match regex:
 844 // ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) from parse_vm_config
 845 // currently we only look at the current configuration in place, i.e., *no*
 846 // snapshot and *no* pending changes
 847 //
 848 // min..max is the char range of the first character of the given props,
 849 // so that we can return early when checking the line
 850 // note: conf must end with a newline
 851 void
 852 _get_property_values(char **found, char *conf, int conf_size, const char **props, uint8_t num_props, char min, char max)
 853 {
 854         const char *const conf_end = conf + conf_size;
 855         char *line = conf;
 856         size_t remaining_size = conf_size;
 857         uint8_t count = 0;
 858
 859         if (conf_size == 0) {
 860             return;
 861         }
 862
 863         char *next_newline = memchr(conf, '\n', conf_size);
 864         if (next_newline == NULL) {
 865                 return; // valid property lines end with \n, but none in the config
 866         }
 867         *next_newline = '\0';
 868
 869         while (line != NULL) {
 870                 if (!line[0]) goto next;
 871
 872                 // snapshot or pending section start, but nothing found yet -> not found
 873                 if (line[0] == '[') return;
 874                 // continue early if line does not begin with the min/max char of the properties
 875                 if (line[0] < min || line[0] > max) goto next;
 876
 877                 size_t line_len = next_newline - line;
 878                 for (uint8_t i = 0; i < num_props; i++) {
 879                         char * value = _get_property_value_from_line(line, line_len, props[i], strlen(props[i]));
 880                         if (value != NULL) {
 881                                 count += (found[i] != NULL) & 0x1; // count newly found lines
 882                                 found[i] = value;
 883                         }
 884                 }
 885                 if (count == num_props) {
 886                         return; // found all
 887                 }
 888 next:
 889                 line = next_newline + 1;
 890                 remaining_size = conf_end - line;
 891                 next_newline = memchr(line, '\n', remaining_size);
 892                 if (next_newline == NULL) {
 893                         return; // valid property lines end with \n, but none in the config
 894                 }
 895                 *next_newline = '\0';
 896         }
 897
 898         return;
 899 }
 900
 901 static void
 902 _g_str_append_kv_jsonescaped(GString *str, const char *k, const char *v)
 903 {
 904         g_string_append_printf(str, "\"%s\": \"", k);
 905
 906         for (; *v; v++) {
 907                 if (*v == '\\' || *v == '"') {
 908                         g_string_append_c(str, '\\');
 909                 }
 910                 g_string_append_c(str, *v);
 911         }
 912
 913         g_string_append_c(str, '"');
 914 }
 915
 916 int
 917 _print_found_properties(
 918         GString *str,
 919         gpointer conf,
 920         int size,
 921         const char **props,
 922         uint8_t num_props,
 923         uint32_t vmid,
 924         char **values,
 925         char min,
 926         char max,
 927         int first)
 928 {
 929         _get_property_values(values, conf, size, props, num_props, min, max);
 930
 931         uint8_t found = 0;
 932         for (uint8_t i = 0; i < num_props; i++) {
 933                 if (values[i] == NULL) {
 934                         continue;
 935                 }
 936                 if (found) {
 937                         g_string_append_c(str, ',');
 938                 } else {
 939                         if (!first) {
 940                                 g_string_append_printf(str, ",\n");
 941                         } else {
 942                                 first = 0;
 943                         }
 944                         g_string_append_printf(str, "\"%u\":{", vmid);
 945                         found = 1;
 946                 }
 947                 _g_str_append_kv_jsonescaped(str, props[i], values[i]);
 948         }
 949
 950         if (found) {
 951                 g_string_append_c(str, '}');
 952         }
 953
 954         return first;
 955 }
 956
 957 int
 958 cfs_create_guest_conf_properties_msg(GString *str, memdb_t *memdb, const char **props, uint8_t num_props, uint32_t vmid)
 959 {
 960         g_return_val_if_fail(cfs_status.vmlist != NULL, -EINVAL);
 961         g_return_val_if_fail(str != NULL, -EINVAL);
 962
 963         // Prelock &memdb->mutex in order to enable the usage of memdb_read_nolock
 964         // to prevent Deadlocks as in #2553
 965         g_mutex_lock (&memdb->mutex);
 966         g_mutex_lock (&mutex);
 967
 968         g_string_printf(str, "{\n");
 969
 970         GHashTable *ht = cfs_status.vmlist;
 971
 972         int res = 0;
 973         GString *path = NULL;
 974         gpointer tmp = NULL;
 975         char **values = calloc(num_props, sizeof(char*));
 976         char min = 'z', max = 'a';
 977
 978         for (uint8_t i = 0; i < num_props; i++) {
 979                 if (props[i][0] > max) {
 980                         max = props[i][0];
 981                 }
 982
 983                 if (props[i][0] < min) {
 984                         min = props[i][0];
 985                 }
 986         }
 987
 988         if (!g_hash_table_size(ht)) {
 989                 goto ret;
 990         }
 991
 992         if ((path = g_string_sized_new(256)) == NULL) {
 993                 res = -ENOMEM;
 994                 goto ret;
 995         }
 996
 997         if (vmid >= 100) {
 998                 vminfo_t *vminfo = (vminfo_t *) g_hash_table_lookup(cfs_status.vmlist, &vmid);
 999                 if (vminfo == NULL) goto enoent;
1000
1001                 if (!vminfo_to_path(vminfo, path)) goto err;
1002
1003                 // use memdb_read_nolock because lock is handled here
1004                 int size = memdb_read_nolock(memdb, path->str, &tmp);
1005                 if (tmp == NULL) goto err;
1006
1007                 // conf needs to be newline terminated
1008                 if (((char *)tmp)[size - 1] != '\n') {
1009                         gpointer new = realloc(tmp, size + 1);
1010                         if (new == NULL) goto err;
1011                         tmp = new;
1012                         ((char *)tmp)[size++] = '\n';
1013                 }
1014                 _print_found_properties(str, tmp, size, props, num_props, vmid, values, min, max, 1);
1015         } else {
1016                 GHashTableIter iter;
1017                 g_hash_table_iter_init (&iter, ht);
1018
1019                 gpointer key, value;
1020                 int first = 1;
1021                 while (g_hash_table_iter_next (&iter, &key, &value)) {
1022                         vminfo_t *vminfo = (vminfo_t *)value;
1023
1024                         if (!vminfo_to_path(vminfo, path)) goto err;
1025
1026                         g_free(tmp); // no-op if already null
1027                         tmp = NULL;
1028                         // use memdb_read_nolock because lock is handled here
1029                         int size = memdb_read_nolock(memdb, path->str, &tmp);
1030                         if (tmp == NULL) continue;
1031
1032                         // conf needs to be newline terminated
1033                         if (((char *)tmp)[size - 1] != '\n') {
1034                             gpointer new = realloc(tmp, size + 1);
1035                             if (new == NULL) continue;
1036                             tmp = new;
1037                             ((char *)tmp)[size++] = '\n';
1038                         }
1039
1040                         memset(values, 0, sizeof(char*)*num_props); // reset array
1041                         first = _print_found_properties(str, tmp, size, props, num_props,
1042                                         vminfo->vmid, values, min, max, first);
1043                 }
1044         }
1045 ret:
1046         g_free(tmp);
1047         free(values);
1048         if (path != NULL) {
1049                 g_string_free(path, TRUE);
1050         }
1051         g_string_append_printf(str,"\n}\n");
1052         g_mutex_unlock (&mutex);
1053         g_mutex_unlock (&memdb->mutex);
1054         return res;
1055 err:
1056         res = -EIO;
1057         goto ret;
1058 enoent:
1059         res = -ENOENT;
1060         goto ret;
1061 }
1062
1063 int
1064 cfs_create_guest_conf_property_msg(GString *str, memdb_t *memdb, const char *prop, uint32_t vmid)
1065 {
1066         return cfs_create_guest_conf_properties_msg(str, memdb, &prop, 1, vmid);
1067 }
1068
1069 void
1070 record_memdb_change(const char *path)
1071 {
1072         g_return_if_fail(cfs_status.memdb_changes != 0);
1073
1074         memdb_change_t *ce;
1075
1076         if ((ce = (memdb_change_t *)g_hash_table_lookup(cfs_status.memdb_changes, path))) {
1077                 ce->version++;
1078         }
1079 }
1080
1081 void
1082 record_memdb_reload(void)
1083 {
1084         for (int i = 0; i < G_N_ELEMENTS(memdb_change_array); i++) {
1085                 memdb_change_array[i].version++;
1086         }
1087 }
1088
1089 static gboolean
1090 kventry_hash_set(
1091         GHashTable *kvhash,
1092         const char *key,
1093         gconstpointer data,
1094         size_t len)
1095 {
1096         g_return_val_if_fail(kvhash != NULL, FALSE);
1097         g_return_val_if_fail(key != NULL, FALSE);
1098         g_return_val_if_fail(data != NULL, FALSE);
1099
1100         kventry_t *entry;
1101         if (!len) {
1102                 g_hash_table_remove(kvhash, key);
1103         } else if ((entry = (kventry_t *)g_hash_table_lookup(kvhash, key))) {
1104                 g_free(entry->data);
1105                 entry->data = g_memdup(data, len);
1106                 entry->len = len;
1107                 entry->version++;
1108         } else {
1109                 kventry_t *entry = g_new0(kventry_t, 1);
1110
1111                 entry->key = g_strdup(key);
1112                 entry->data = g_memdup(data, len);
1113                 entry->len = len;
1114
1115                 g_hash_table_replace(kvhash, entry->key, entry);
1116         }
1117
1118         return TRUE;
1119 }
1120
1121 static const char *rrd_def_node[] = {
1122         "DS:loadavg:GAUGE:120:0:U",
1123         "DS:maxcpu:GAUGE:120:0:U",
1124         "DS:cpu:GAUGE:120:0:U",
1125         "DS:iowait:GAUGE:120:0:U",
1126         "DS:memtotal:GAUGE:120:0:U",
1127         "DS:memused:GAUGE:120:0:U",
1128         "DS:swaptotal:GAUGE:120:0:U",
1129         "DS:swapused:GAUGE:120:0:U",
1130         "DS:roottotal:GAUGE:120:0:U",
1131         "DS:rootused:GAUGE:120:0:U",
1132         "DS:netin:DERIVE:120:0:U",
1133         "DS:netout:DERIVE:120:0:U",
1134
1135         "RRA:AVERAGE:0.5:1:70", // 1 min avg - one hour
1136         "RRA:AVERAGE:0.5:30:70", // 30 min avg - one day
1137         "RRA:AVERAGE:0.5:180:70", // 3 hour avg - one week
1138         "RRA:AVERAGE:0.5:720:70", // 12 hour avg - one month
1139         "RRA:AVERAGE:0.5:10080:70", // 7 day avg - ony year
1140
1141         "RRA:MAX:0.5:1:70", // 1 min max - one hour
1142         "RRA:MAX:0.5:30:70", // 30 min max - one day
1143         "RRA:MAX:0.5:180:70",  // 3 hour max - one week
1144         "RRA:MAX:0.5:720:70", // 12 hour max - one month
1145         "RRA:MAX:0.5:10080:70", // 7 day max - ony year
1146         NULL,
1147 };
1148
1149 static const char *rrd_def_vm[] = {
1150         "DS:maxcpu:GAUGE:120:0:U",
1151         "DS:cpu:GAUGE:120:0:U",
1152         "DS:maxmem:GAUGE:120:0:U",
1153         "DS:mem:GAUGE:120:0:U",
1154         "DS:maxdisk:GAUGE:120:0:U",
1155         "DS:disk:GAUGE:120:0:U",
1156         "DS:netin:DERIVE:120:0:U",
1157         "DS:netout:DERIVE:120:0:U",
1158         "DS:diskread:DERIVE:120:0:U",
1159         "DS:diskwrite:DERIVE:120:0:U",
1160
1161         "RRA:AVERAGE:0.5:1:70", // 1 min avg - one hour
1162         "RRA:AVERAGE:0.5:30:70", // 30 min avg - one day
1163         "RRA:AVERAGE:0.5:180:70", // 3 hour avg - one week
1164         "RRA:AVERAGE:0.5:720:70", // 12 hour avg - one month
1165         "RRA:AVERAGE:0.5:10080:70", // 7 day avg - ony year
1166
1167         "RRA:MAX:0.5:1:70", // 1 min max - one hour
1168         "RRA:MAX:0.5:30:70", // 30 min max - one day
1169         "RRA:MAX:0.5:180:70",  // 3 hour max - one week
1170         "RRA:MAX:0.5:720:70", // 12 hour max - one month
1171         "RRA:MAX:0.5:10080:70", // 7 day max - ony year
1172         NULL,
1173 };
1174
1175 static const char *rrd_def_storage[] = {
1176         "DS:total:GAUGE:120:0:U",
1177         "DS:used:GAUGE:120:0:U",
1178
1179         "RRA:AVERAGE:0.5:1:70", // 1 min avg - one hour
1180         "RRA:AVERAGE:0.5:30:70", // 30 min avg - one day
1181         "RRA:AVERAGE:0.5:180:70", // 3 hour avg - one week
1182         "RRA:AVERAGE:0.5:720:70", // 12 hour avg - one month
1183         "RRA:AVERAGE:0.5:10080:70", // 7 day avg - ony year
1184
1185         "RRA:MAX:0.5:1:70", // 1 min max - one hour
1186         "RRA:MAX:0.5:30:70", // 30 min max - one day
1187         "RRA:MAX:0.5:180:70",  // 3 hour max - one week
1188         "RRA:MAX:0.5:720:70", // 12 hour max - one month
1189         "RRA:MAX:0.5:10080:70", // 7 day max - ony year
1190         NULL,
1191 };
1192
1193 #define RRDDIR "/var/lib/rrdcached/db"
1194
1195 static void
1196 create_rrd_file(
1197         const char *filename,
1198         int argcount,
1199         const char *rrddef[])
1200 {
1201         /* start at day boundary */
1202         time_t ctime;
1203         time(&ctime);
1204         struct tm *ltm = localtime(&ctime);
1205         ltm->tm_sec = 0;
1206         ltm->tm_min = 0;
1207         ltm->tm_hour = 0;
1208
1209         rrd_clear_error();
1210         if (rrd_create_r(filename, 60, timelocal(ltm), argcount, rrddef)) {
1211                 cfs_message("RRD create error %s: %s", filename, rrd_get_error());
1212         }
1213 }
1214
1215 static inline const char *
1216 rrd_skip_data(
1217         const char *data,
1218         int count)
1219 {
1220         int found = 0;
1221         while (*data && found < count) {
1222                 if (*data++ == ':')
1223                         found++;
1224         }
1225         return data;
1226 }
1227
1228 static void
1229 update_rrd_data(
1230         const char *key,
1231         gconstpointer data,
1232         size_t len)
1233 {
1234         g_return_if_fail(key != NULL);
1235         g_return_if_fail(data != NULL);
1236         g_return_if_fail(len > 0);
1237         g_return_if_fail(len < 4096);
1238
1239         static const char *rrdcsock = "unix:/var/run/rrdcached.sock";
1240
1241         int use_daemon = 1;
1242         if (rrdc_connect(rrdcsock) != 0)
1243                 use_daemon = 0;
1244
1245         char *filename = NULL;
1246
1247         int skip = 0;
1248
1249         if (strncmp(key, "pve2-node/", 10) == 0) {
1250                 const char *node = key + 10;
1251
1252                 skip = 2;
1253
1254                 if (strchr(node, '/') != NULL)
1255                         goto keyerror;
1256
1257                 if (strlen(node) < 1)
1258                         goto keyerror;
1259
1260                 filename = g_strdup_printf(RRDDIR "/%s", key);
1261
1262                 if (!g_file_test(filename, G_FILE_TEST_EXISTS)) {
1263
1264                         mkdir(RRDDIR "/pve2-node", 0755);
1265                         int argcount = sizeof(rrd_def_node)/sizeof(void*) - 1;
1266                         create_rrd_file(filename, argcount, rrd_def_node);
1267                 }
1268
1269         } else if ((strncmp(key, "pve2-vm/", 8) == 0) ||
1270                    (strncmp(key, "pve2.3-vm/", 10) == 0)) {
1271                 const char *vmid;
1272
1273                 if (strncmp(key, "pve2-vm/", 8) == 0) {
1274                         vmid = key + 8;
1275                         skip = 2;
1276                 } else {
1277                         vmid = key + 10;
1278                         skip = 4;
1279                 }
1280
1281                 if (strchr(vmid, '/') != NULL)
1282                         goto keyerror;
1283
1284                 if (strlen(vmid) < 1)
1285                         goto keyerror;
1286
1287                 filename = g_strdup_printf(RRDDIR "/%s/%s", "pve2-vm", vmid);
1288
1289                 if (!g_file_test(filename, G_FILE_TEST_EXISTS)) {
1290
1291                         mkdir(RRDDIR "/pve2-vm", 0755);
1292                         int argcount = sizeof(rrd_def_vm)/sizeof(void*) - 1;
1293                         create_rrd_file(filename, argcount, rrd_def_vm);
1294                 }
1295
1296         } else if (strncmp(key, "pve2-storage/", 13) == 0) {
1297                 const char *node = key + 13;
1298
1299                 const char *storage = node;
1300                 while (*storage && *storage != '/')
1301                         storage++;
1302
1303                 if (*storage != '/' || ((storage - node) < 1))
1304                         goto keyerror;
1305
1306                 storage++;
1307
1308                 if (strchr(storage, '/') != NULL)
1309                         goto keyerror;
1310
1311                 if (strlen(storage) < 1)
1312                         goto keyerror;
1313
1314                 filename = g_strdup_printf(RRDDIR "/%s", key);
1315
1316                 if (!g_file_test(filename, G_FILE_TEST_EXISTS)) {
1317
1318                         mkdir(RRDDIR "/pve2-storage", 0755);
1319
1320                         char *dir = g_path_get_dirname(filename);
1321                         mkdir(dir, 0755);
1322                         g_free(dir);
1323
1324                         int argcount = sizeof(rrd_def_storage)/sizeof(void*) - 1;
1325                         create_rrd_file(filename, argcount, rrd_def_storage);
1326                 }
1327
1328         } else {
1329                 goto keyerror;
1330         }
1331
1332         const char *dp = skip ? rrd_skip_data(data, skip) : data;
1333
1334         const char *update_args[] = { dp, NULL };
1335
1336         if (use_daemon) {
1337                 int status;
1338                 if ((status = rrdc_update(filename, 1, update_args)) != 0) {
1339                         cfs_message("RRDC update error %s: %d", filename, status);
1340                         rrdc_disconnect();
1341                         rrd_clear_error();
1342                         if (rrd_update_r(filename, NULL, 1, update_args) != 0) {
1343                                 cfs_message("RRD update error %s: %s", filename, rrd_get_error());
1344                         }
1345                 }
1346
1347         } else {
1348                 rrd_clear_error();
1349                 if (rrd_update_r(filename, NULL, 1, update_args) != 0) {
1350                         cfs_message("RRD update error %s: %s", filename, rrd_get_error());
1351                 }
1352         }
1353
1354 ret:
1355         if (filename)
1356                 g_free(filename);
1357
1358         return;
1359
1360 keyerror:
1361         cfs_critical("RRD update error: unknown/wrong key %s", key);
1362         goto ret;
1363 }
1364
1365 static gboolean
1366 rrd_entry_is_old(
1367         gpointer key,
1368         gpointer value,
1369         gpointer user_data)
1370 {
1371         rrdentry_t *entry = (rrdentry_t *)value;
1372         uint32_t ctime = GPOINTER_TO_UINT(user_data);
1373
1374         int diff = ctime - entry->time;
1375
1376         /* remove everything older than 5 minutes */
1377         int expire = 60*5;
1378
1379         return (diff > expire) ? TRUE : FALSE;
1380 }
1381
1382 static char *rrd_dump_buf = NULL;
1383 static time_t rrd_dump_last = 0;
1384
1385 void
1386 cfs_rrd_dump(GString *str)
1387 {
1388         time_t ctime;
1389
1390         g_mutex_lock (&mutex);
1391
1392         time(&ctime);
1393         if (rrd_dump_buf && (ctime - rrd_dump_last) < 2) {
1394                 g_string_assign(str, rrd_dump_buf);
1395                 g_mutex_unlock (&mutex);
1396                 return;
1397         }
1398
1399         /* remove old data */
1400         g_hash_table_foreach_remove(cfs_status.rrdhash, rrd_entry_is_old,
1401                                     GUINT_TO_POINTER(ctime));
1402
1403         g_string_set_size(str, 0);
1404
1405         GHashTableIter iter;
1406         gpointer key, value;
1407
1408         g_hash_table_iter_init (&iter, cfs_status.rrdhash);
1409
1410         while (g_hash_table_iter_next (&iter, &key, &value)) {
1411                 rrdentry_t *entry = (rrdentry_t *)value;
1412                 g_string_append(str, key);
1413                 g_string_append(str, ":");
1414                 g_string_append(str, entry->data);
1415                 g_string_append(str, "\n");
1416         }
1417
1418         g_string_append_c(str, 0); // never return undef
1419
1420         rrd_dump_last = ctime;
1421         if (rrd_dump_buf)
1422                 g_free(rrd_dump_buf);
1423         rrd_dump_buf = g_strdup(str->str);
1424
1425         g_mutex_unlock (&mutex);
1426 }
1427
1428 static gboolean
1429 nodeip_hash_set(
1430         GHashTable *iphash,
1431         const char *nodename,
1432         const char *ip,
1433         size_t len)
1434 {
1435         g_return_val_if_fail(iphash != NULL, FALSE);
1436         g_return_val_if_fail(nodename != NULL, FALSE);
1437         g_return_val_if_fail(ip != NULL, FALSE);
1438         g_return_val_if_fail(len > 0, FALSE);
1439         g_return_val_if_fail(len < 256, FALSE);
1440         g_return_val_if_fail(ip[len-1] == 0, FALSE);
1441
1442         char *oldip = (char *)g_hash_table_lookup(iphash, nodename);
1443
1444         if (!oldip || (strcmp(oldip, ip) != 0)) {
1445                 cfs_status.clinfo_version++;
1446                 g_hash_table_replace(iphash, g_strdup(nodename), g_strdup(ip));
1447         }
1448
1449         return TRUE;
1450 }
1451
1452 static gboolean
1453 rrdentry_hash_set(
1454         GHashTable *rrdhash,
1455         const char *key,
1456         const char *data,
1457         size_t len)
1458 {
1459         g_return_val_if_fail(rrdhash != NULL, FALSE);
1460         g_return_val_if_fail(key != NULL, FALSE);
1461         g_return_val_if_fail(data != NULL, FALSE);
1462         g_return_val_if_fail(len > 0, FALSE);
1463         g_return_val_if_fail(len < 4096, FALSE);
1464         g_return_val_if_fail(data[len-1] == 0, FALSE);
1465
1466         rrdentry_t *entry;
1467         if ((entry = (rrdentry_t *)g_hash_table_lookup(rrdhash, key))) {
1468                 g_free(entry->data);
1469                 entry->data = g_memdup(data, len);
1470                 entry->len = len;
1471                 entry->time = time(NULL);
1472         } else {
1473                 rrdentry_t *entry = g_new0(rrdentry_t, 1);
1474
1475                 entry->key = g_strdup(key);
1476                 entry->data = g_memdup(data, len);
1477                 entry->len = len;
1478                 entry->time = time(NULL);
1479
1480                 g_hash_table_replace(rrdhash, entry->key, entry);
1481         }
1482
1483         update_rrd_data(key, data, len);
1484
1485         return TRUE;
1486 }
1487
1488 static int
1489 kvstore_send_update_message(
1490         dfsm_t *dfsm,
1491         const char *key,
1492         gpointer data,
1493         guint32 len)
1494 {
1495         if (!dfsm_is_initialized(dfsm))
1496                 return -EACCES;
1497
1498         struct iovec iov[2];
1499
1500         char name[256];
1501         g_strlcpy(name, key, sizeof(name));
1502
1503         iov[0].iov_base = &name;
1504         iov[0].iov_len = sizeof(name);
1505
1506         iov[1].iov_base = (char *)data;
1507         iov[1].iov_len = len;
1508
1509         if (dfsm_send_message(dfsm, KVSTORE_MESSAGE_UPDATE, iov, 2) == CS_OK)
1510                 return 0;
1511
1512         return -EACCES;
1513 }
1514
1515 static clog_entry_t *
1516 kvstore_parse_log_message(
1517         const void *msg,
1518         size_t msg_len)
1519 {
1520         g_return_val_if_fail(msg != NULL, NULL);
1521
1522         if (msg_len < sizeof(clog_entry_t)) {
1523                 cfs_critical("received short log message (%zu < %zu)", msg_len, sizeof(clog_entry_t));
1524                 return NULL;
1525         }
1526
1527         clog_entry_t *entry = (clog_entry_t *)msg;
1528
1529         uint32_t size = sizeof(clog_entry_t) + entry->node_len +
1530                 entry->ident_len + entry->tag_len + entry->msg_len;
1531
1532         if (msg_len != size) {
1533                 cfs_critical("received log message with wrong size (%zu != %u)", msg_len, size);
1534                 return NULL;
1535         }
1536
1537         char *msgptr = entry->data;
1538
1539         if (*((char *)msgptr + entry->node_len - 1)) {
1540                 cfs_critical("unterminated string in log message");
1541                 return NULL;
1542         }
1543         msgptr += entry->node_len;
1544
1545         if (*((char *)msgptr + entry->ident_len - 1)) {
1546                 cfs_critical("unterminated string in log message");
1547                 return NULL;
1548         }
1549         msgptr += entry->ident_len;
1550
1551         if (*((char *)msgptr + entry->tag_len - 1)) {
1552                 cfs_critical("unterminated string in log message");
1553                 return NULL;
1554         }
1555         msgptr += entry->tag_len;
1556
1557         if (*((char *)msgptr + entry->msg_len - 1)) {
1558                 cfs_critical("unterminated string in log message");
1559                 return NULL;
1560         }
1561
1562         return entry;
1563 }
1564
1565 static gboolean
1566 kvstore_parse_update_message(
1567         const void *msg,
1568         size_t msg_len,
1569         const char **key,
1570         gconstpointer *data,
1571         guint32 *len)
1572 {
1573         g_return_val_if_fail(msg != NULL, FALSE);
1574         g_return_val_if_fail(key != NULL, FALSE);
1575         g_return_val_if_fail(data != NULL, FALSE);
1576         g_return_val_if_fail(len != NULL, FALSE);
1577
1578         if (msg_len < 256) {
1579                 cfs_critical("received short kvstore message (%zu < 256)", msg_len);
1580                 return FALSE;
1581         }
1582
1583         /* test if key is null terminated */
1584         int i = 0;
1585         for (i = 0; i < 256; i++)
1586                 if (((char *)msg)[i] == 0)
1587                         break;
1588
1589         if (i == 256)
1590                 return FALSE;
1591
1592
1593         *len = msg_len - 256;
1594         *key = msg;
1595         *data = (char *) msg + 256;
1596
1597         return TRUE;
1598 }
1599
1600 int
1601 cfs_create_status_msg(
1602         GString *str,
1603         const char *nodename,
1604         const char *key)
1605 {
1606         g_return_val_if_fail(str != NULL, -EINVAL);
1607         g_return_val_if_fail(key != NULL, -EINVAL);
1608
1609         int res = -ENOENT;
1610
1611         GHashTable *kvhash = NULL;
1612
1613         g_mutex_lock (&mutex);
1614
1615         if (!nodename || !nodename[0] || !strcmp(nodename, cfs.nodename)) {
1616                 kvhash = cfs_status.kvhash;
1617         } else if (cfs_status.clinfo && cfs_status.clinfo->nodes_byname) {
1618                 cfs_clnode_t *clnode;
1619                 if ((clnode = g_hash_table_lookup(cfs_status.clinfo->nodes_byname, nodename)))
1620                         kvhash = clnode->kvhash;
1621         }
1622
1623         kventry_t *entry;
1624         if (kvhash && (entry = (kventry_t *)g_hash_table_lookup(kvhash, key))) {
1625                 g_string_append_len(str, entry->data, entry->len);
1626                 res = 0;
1627         }
1628
1629         g_mutex_unlock (&mutex);
1630
1631         return res;
1632 }
1633
1634 int
1635 cfs_status_set(
1636         const char *key,
1637         gpointer data,
1638         size_t len)
1639 {
1640         g_return_val_if_fail(key != NULL, FALSE);
1641         g_return_val_if_fail(data != NULL, FALSE);
1642         g_return_val_if_fail(cfs_status.kvhash != NULL, FALSE);
1643
1644         if (len > CFS_MAX_STATUS_SIZE)
1645                 return -EFBIG;
1646
1647         g_mutex_lock (&mutex);
1648
1649         gboolean res;
1650
1651         if (strncmp(key, "rrd/", 4) == 0) {
1652                 res = rrdentry_hash_set(cfs_status.rrdhash, key + 4, data, len);
1653         } else if (!strcmp(key, "nodeip")) {
1654                 res = nodeip_hash_set(cfs_status.iphash, cfs.nodename, data, len);
1655         } else {
1656                 res = kventry_hash_set(cfs_status.kvhash, key, data, len);
1657         }
1658         g_mutex_unlock (&mutex);
1659
1660         if (cfs_status.kvstore)
1661                 kvstore_send_update_message(cfs_status.kvstore, key, data, len);
1662
1663         return res ? 0 : -ENOMEM;
1664 }
1665
1666 gboolean
1667 cfs_kvstore_node_set(
1668         uint32_t nodeid,
1669         const char *key,
1670         gconstpointer data,
1671         size_t len)
1672 {
1673         g_return_val_if_fail(nodeid != 0, FALSE);
1674         g_return_val_if_fail(key != NULL, FALSE);
1675         g_return_val_if_fail(data != NULL, FALSE);
1676
1677         g_mutex_lock (&mutex);
1678
1679         if (!cfs_status.clinfo || !cfs_status.clinfo->nodes_byid)
1680                 goto ret; /* ignore */
1681
1682         cfs_clnode_t *clnode = g_hash_table_lookup(cfs_status.clinfo->nodes_byid, &nodeid);
1683         if (!clnode)
1684                 goto ret; /* ignore */
1685
1686         cfs_debug("got node %d status update %s", nodeid, key);
1687
1688         if (strncmp(key, "rrd/", 4) == 0) {
1689                 rrdentry_hash_set(cfs_status.rrdhash, key + 4, data, len);
1690         } else if (!strcmp(key, "nodeip")) {
1691                 nodeip_hash_set(cfs_status.iphash, clnode->name, data, len);
1692         } else {
1693                 if (!clnode->kvhash) {
1694                         if (!(clnode->kvhash = kventry_hash_new())) {
1695                                 goto ret; /*ignore */
1696                         }
1697                 }
1698
1699                 kventry_hash_set(clnode->kvhash, key, data, len);
1700
1701         }
1702 ret:
1703         g_mutex_unlock (&mutex);
1704
1705         return TRUE;
1706 }
1707
1708 static gboolean
1709 cfs_kvstore_sync(void)
1710 {
1711         g_return_val_if_fail(cfs_status.kvhash != NULL, FALSE);
1712         g_return_val_if_fail(cfs_status.kvstore != NULL, FALSE);
1713
1714         gboolean res = TRUE;
1715
1716         g_mutex_lock (&mutex);
1717
1718         GHashTable *ht = cfs_status.kvhash;
1719         GHashTableIter iter;
1720         gpointer key, value;
1721
1722         g_hash_table_iter_init (&iter, ht);
1723
1724         while (g_hash_table_iter_next (&iter, &key, &value)) {
1725                 kventry_t *entry = (kventry_t *)value;
1726                 kvstore_send_update_message(cfs_status.kvstore, entry->key, entry->data, entry->len);
1727         }
1728
1729         g_mutex_unlock (&mutex);
1730
1731         return res;
1732 }
1733
1734 static int
1735 dfsm_deliver(
1736         dfsm_t *dfsm,
1737         gpointer data,
1738         int *res_ptr,
1739         uint32_t nodeid,
1740         uint32_t pid,
1741         uint16_t msg_type,
1742         uint32_t msg_time,
1743         const void *msg,
1744         size_t msg_len)
1745 {
1746         g_return_val_if_fail(dfsm != NULL, -1);
1747         g_return_val_if_fail(msg != NULL, -1);
1748         g_return_val_if_fail(res_ptr != NULL, -1);
1749
1750         /* ignore message for ourself */
1751         if (dfsm_nodeid_is_local(dfsm, nodeid, pid))
1752                 goto ret;
1753
1754         if (msg_type == KVSTORE_MESSAGE_UPDATE) {
1755                 const char *key;
1756                 gconstpointer data;
1757                 guint32 len;
1758                 if (kvstore_parse_update_message(msg, msg_len, &key, &data, &len)) {
1759                         cfs_kvstore_node_set(nodeid, key, data, len);
1760                 } else {
1761                         cfs_critical("cant parse update message");
1762                 }
1763         } else if (msg_type == KVSTORE_MESSAGE_LOG) {
1764                 cfs_message("received log"); // fixme: remove
1765                 const clog_entry_t *entry;
1766                 if ((entry = kvstore_parse_log_message(msg, msg_len))) {
1767                         clusterlog_insert(cfs_status.clusterlog, entry);
1768                 } else {
1769                         cfs_critical("cant parse log message");
1770                 }
1771         } else {
1772                 cfs_critical("received unknown message type %d\n", msg_type);
1773                 goto fail;
1774         }
1775
1776 ret:
1777         *res_ptr = 0;
1778         return 1;
1779
1780 fail:
1781         *res_ptr = -EACCES;
1782         return 1;
1783 }
1784
1785 static void
1786 dfsm_confchg(
1787         dfsm_t *dfsm,
1788         gpointer data,
1789         const struct cpg_address *member_list,
1790         size_t member_list_entries)
1791 {
1792         g_return_if_fail(dfsm != NULL);
1793         g_return_if_fail(member_list != NULL);
1794
1795         cfs_debug("enter %s", __func__);
1796
1797         g_mutex_lock (&mutex);
1798
1799         cfs_clinfo_t *clinfo = cfs_status.clinfo;
1800
1801         if (clinfo && clinfo->nodes_byid) {
1802
1803                 GHashTable *ht = clinfo->nodes_byid;
1804                 GHashTableIter iter;
1805                 gpointer key, value;
1806
1807                 g_hash_table_iter_init (&iter, ht);
1808
1809                 while (g_hash_table_iter_next (&iter, &key, &value)) {
1810                         cfs_clnode_t *node = (cfs_clnode_t *)value;
1811                         node->online = FALSE;
1812                 }
1813
1814                 for (int i = 0; i < member_list_entries; i++) {
1815                         cfs_clnode_t *node;
1816                         if ((node = g_hash_table_lookup(clinfo->nodes_byid, &member_list[i].nodeid))) {
1817                                 node->online = TRUE;
1818                         }
1819                 }
1820
1821                 cfs_status.clinfo_version++;
1822         }
1823
1824         g_mutex_unlock (&mutex);
1825 }
1826
1827 static gpointer
1828 dfsm_get_state(
1829         dfsm_t *dfsm,
1830         gpointer data,
1831         unsigned int *res_len)
1832 {
1833         g_return_val_if_fail(dfsm != NULL, NULL);
1834
1835         gpointer msg = clusterlog_get_state(cfs_status.clusterlog, res_len);
1836
1837         return msg;
1838 }
1839
1840 static int
1841 dfsm_process_update(
1842         dfsm_t *dfsm,
1843         gpointer data,
1844         dfsm_sync_info_t *syncinfo,
1845         uint32_t nodeid,
1846         uint32_t pid,
1847         const void *msg,
1848         size_t msg_len)
1849 {
1850         cfs_critical("%s: received unexpected update message", __func__);
1851
1852         return -1;
1853 }
1854
1855 static int
1856 dfsm_process_state_update(
1857         dfsm_t *dfsm,
1858         gpointer data,
1859         dfsm_sync_info_t *syncinfo)
1860 {
1861         g_return_val_if_fail(dfsm != NULL, -1);
1862         g_return_val_if_fail(syncinfo != NULL, -1);
1863
1864         clog_base_t *clog[syncinfo->node_count];
1865
1866         int local_index = -1;
1867         for (int i = 0; i < syncinfo->node_count; i++) {
1868                 dfsm_node_info_t *ni = &syncinfo->nodes[i];
1869                 ni->synced = 1;
1870
1871                 if (syncinfo->local == ni)
1872                         local_index = i;
1873
1874                 clog_base_t *base = (clog_base_t *)ni->state;
1875                 if (ni->state_len > 8 && ni->state_len == clog_size(base)) {
1876                         clog[i] = ni->state;
1877                 } else {
1878                         cfs_critical("received log with wrong size %u", ni->state_len);
1879                         clog[i] = NULL;
1880                 }
1881         }
1882
1883         if (!clusterlog_merge(cfs_status.clusterlog, clog, syncinfo->node_count, local_index)) {
1884                 cfs_critical("unable to merge log files");
1885         }
1886
1887         cfs_kvstore_sync();
1888
1889         return 1;
1890 }
1891
1892 static int
1893 dfsm_commit(
1894         dfsm_t *dfsm,
1895         gpointer data,
1896         dfsm_sync_info_t *syncinfo)
1897 {
1898         g_return_val_if_fail(dfsm != NULL, -1);
1899         g_return_val_if_fail(syncinfo != NULL, -1);
1900
1901         return 1;
1902 }
1903
1904 static void
1905 dfsm_synced(dfsm_t *dfsm)
1906 {
1907         g_return_if_fail(dfsm != NULL);
1908
1909         char *ip = (char *)g_hash_table_lookup(cfs_status.iphash, cfs.nodename);
1910         if (!ip)
1911                 ip = cfs.ip;
1912
1913         cfs_status_set("nodeip", ip, strlen(ip) + 1);
1914 }
1915
1916 static int
1917 dfsm_cleanup(
1918         dfsm_t *dfsm,
1919         gpointer data,
1920         dfsm_sync_info_t *syncinfo)
1921 {
1922         return 1;
1923 }
1924
1925 static dfsm_callbacks_t kvstore_dfsm_callbacks = {
1926         .dfsm_deliver_fn = dfsm_deliver,
1927         .dfsm_confchg_fn = dfsm_confchg,
1928
1929         .dfsm_get_state_fn = dfsm_get_state,
1930         .dfsm_process_state_update_fn = dfsm_process_state_update,
1931         .dfsm_process_update_fn = dfsm_process_update,
1932         .dfsm_commit_fn = dfsm_commit,
1933         .dfsm_cleanup_fn = dfsm_cleanup,
1934         .dfsm_synced_fn = dfsm_synced,
1935 };
1936
1937 dfsm_t *
1938 cfs_status_dfsm_new(void)
1939 {
1940         g_mutex_lock (&mutex);
1941
1942         cfs_status.kvstore = dfsm_new(NULL, KVSTORE_CPG_GROUP_NAME, G_LOG_DOMAIN,
1943                                       0, &kvstore_dfsm_callbacks);
1944         g_mutex_unlock (&mutex);
1945
1946         return cfs_status.kvstore;
1947 }
1948
1949 gboolean
1950 cfs_is_quorate(void)
1951 {
1952         g_mutex_lock (&mutex);
1953         gboolean res =  cfs_status.quorate;
1954         g_mutex_unlock (&mutex);
1955
1956         return res;
1957 }
1958
1959 void
1960 cfs_set_quorate(
1961         uint32_t quorate,
1962         gboolean quiet)
1963 {
1964         g_mutex_lock (&mutex);
1965
1966         uint32_t prev_quorate = cfs_status.quorate;
1967         cfs_status.quorate = quorate;
1968
1969         if (!prev_quorate && cfs_status.quorate) {
1970                 if (!quiet)
1971                         cfs_message("node has quorum");
1972         }
1973
1974         if (prev_quorate && !cfs_status.quorate) {
1975                 if (!quiet)
1976                         cfs_message("node lost quorum");
1977         }
1978
1979         g_mutex_unlock (&mutex);
1980 }