[qemu-server.git] / qmeventd / qmeventd.c

// SPDX-License-Identifier: AGPL-3.0-or-later
/*
    Copyright (C) 2018 - 2021 Proxmox Server Solutions GmbH

    Author: Dominik Csapak <d.csapak@proxmox.com>
    Author: Stefan Reiter <s.reiter@proxmox.com>

    Description:

    qmeventd listens on a given socket, and waits for qemu processes to
    connect. After accepting a connection qmeventd waits for shutdown events
    followed by the closing of the socket. Once that happens `qm cleanup` will
    be executed with following three arguments:
    VMID <graceful> <guest>
    Where `graceful` can be `1` or `0` depending if shutdown event was observed
    before the socket got closed. The second parameter `guest` is also boolean
    `1` or `0` depending if the shutdown was requested from the guest OS
    (i.e., the "inside").
*/

#ifndef _GNU_SOURCE
#define _GNU_SOURCE
#endif

#include <errno.h>
#include <fcntl.h>
#include <gmodule.h>
#include <json.h>
#include <signal.h>
#include <stdbool.h>
#include <stdio.h>
#include <string.h>
#include <sys/epoll.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <sys/un.h>
#include <sys/wait.h>
#include <unistd.h>

#include "qmeventd.h"

static int verbose = 0;
static int epoll_fd = 0;
static const char *progname;
GHashTable *vm_clients; // key=vmid (freed on remove), value=*Client (free manually)
GSList *forced_cleanups;
volatile sig_atomic_t alarm_triggered = 0;

/*
 * Helper functions
 */

static void
usage()
{
    fprintf(stderr, "Usage: %s [-f] [-v] PATH\n", progname);
    fprintf(stderr, "  -f       run in foreground (default: false)\n");
    fprintf(stderr, "  -v       verbose (default: false)\n");
    fprintf(stderr, "  PATH     use PATH for socket\n");
}

static pid_t
get_pid_from_fd(int fd)
{
    struct ucred credentials = { .pid = 0, .uid = 0, .gid = 0 };
    socklen_t len = sizeof(struct ucred);
    log_neg(getsockopt(fd, SOL_SOCKET, SO_PEERCRED, &credentials, &len), "getsockopt");
    return credentials.pid;
}

/*
 * reads the vmid from /proc/<pid>/cmdline
 * after the '-id' argument
 */
static unsigned long
get_vmid_from_pid(pid_t pid)
{
    char filename[32] = { 0 };
    int len = snprintf(filename, sizeof(filename), "/proc/%d/cmdline", pid);
    if (len < 0) {
	fprintf(stderr, "error during snprintf for %d: %s\n", pid,
		strerror(errno));
	return 0;
    }
    if ((size_t)len >= sizeof(filename)) {
	fprintf(stderr, "error: pid %d too long\n", pid);
	return 0;
    }
    FILE *fp = fopen(filename, "re");
    if (fp == NULL) {
	fprintf(stderr, "error opening %s: %s\n", filename, strerror(errno));
	return 0;
    }

    unsigned long vmid = 0;
    ssize_t rc = 0;
    char *buf = NULL;
    size_t buflen = 0;
    while ((rc = getdelim(&buf, &buflen, '\0', fp)) >= 0) {
	if (!strcmp(buf, "-id")) {
	    break;
	}
    }

    if (rc < 0) {
	goto err;
    }

    if (getdelim(&buf, &buflen, '\0', fp) >= 0) {
	if (buf[0] == '-' || buf[0] == '\0') {
	    fprintf(stderr, "invalid vmid %s\n", buf);
	    goto ret;
	}

	errno = 0;
	char *endptr = NULL;
	vmid = strtoul(buf, &endptr, 10);
	if (errno != 0) {
	    vmid = 0;
	    goto err;
	} else if (*endptr != '\0') {
	    fprintf(stderr, "invalid vmid %s\n", buf);
	    vmid = 0;
	}

	goto ret;
    }

err:
    fprintf(stderr, "error parsing vmid for %d: %s\n", pid, strerror(errno));

ret:
    free(buf);
    fclose(fp);
    return vmid;
}

static bool
must_write(int fd, const char *buf, size_t len)
{
    ssize_t wlen;
    do {
	wlen = write(fd, buf, len);
    } while (wlen < 0 && errno == EINTR);

    return (wlen == (ssize_t)len);
}

/*
 * qmp handling functions
 */

static void
send_qmp_cmd(struct Client *client, const char *buf, size_t len)
{
    if (!must_write(client->fd, buf, len - 1)) {
	fprintf(stderr, "%s: cannot send QMP message\n", client->qemu.vmid);
	cleanup_client(client);
    }
}

void
handle_qmp_handshake(struct Client *client)
{
    VERBOSE_PRINT("pid%d: got QMP handshake, assuming QEMU client\n", client->pid);

    // extract vmid from cmdline, now that we know it's a QEMU process
    unsigned long vmid = get_vmid_from_pid(client->pid);
    int res = snprintf(client->qemu.vmid, sizeof(client->qemu.vmid), "%lu", vmid);
    if (vmid == 0 || res < 0 || res >= (int)sizeof(client->qemu.vmid)) {
	fprintf(stderr, "could not get vmid from pid %d\n", client->pid);
	cleanup_client(client);
	return;
    }

    VERBOSE_PRINT("pid%d: assigned VMID: %s\n", client->pid, client->qemu.vmid);
    client->type = CLIENT_QEMU;
    if(!g_hash_table_insert(vm_clients, strdup(client->qemu.vmid), client)) {
	// not fatal, just means backup handling won't work
	fprintf(stderr, "%s: could not insert client into VMID->client table\n",
		client->qemu.vmid);
    }

    static const char qmp_answer[] = "{\"execute\":\"qmp_capabilities\"}\n";
    send_qmp_cmd(client, qmp_answer, sizeof(qmp_answer));
}

void
handle_qmp_event(struct Client *client, struct json_object *obj)
{
    struct json_object *event;
    if (!json_object_object_get_ex(obj, "event", &event)) {
	return;
    }
    VERBOSE_PRINT("%s: got QMP event: %s\n", client->qemu.vmid, json_object_get_string(event));

    if (client->state == STATE_TERMINATING) {
	// QEMU sometimes sends a second SHUTDOWN after SIGTERM, ignore
	VERBOSE_PRINT("%s: event was after termination, ignoring\n", client->qemu.vmid);
	return;
    }

    // event, check if shutdown and get guest parameter
    if (!strcmp(json_object_get_string(event), "SHUTDOWN")) {
	client->qemu.graceful = 1;
	struct json_object *data;
	struct json_object *guest;
	if (json_object_object_get_ex(obj, "data", &data) &&
	    json_object_object_get_ex(data, "guest", &guest))
	{
	    client->qemu.guest = (unsigned short)json_object_get_boolean(guest);
	}

	// check if a backup is running and kill QEMU process if not
	terminate_check(client);
    }
}

void
terminate_check(struct Client *client)
{
    if (client->state != STATE_IDLE) {
	// if we're already in a request, queue this one until after
	VERBOSE_PRINT("%s: terminate_check queued\n", client->qemu.vmid);
	client->qemu.term_check_queued = true;
	return;
    }

    client->qemu.term_check_queued = false;

    VERBOSE_PRINT("%s: query-status\n", client->qemu.vmid);
    client->state = STATE_EXPECT_STATUS_RESP;
    static const char qmp_req[] = "{\"execute\":\"query-status\"}\n";
    send_qmp_cmd(client, qmp_req, sizeof(qmp_req));
}

void
handle_qmp_return(struct Client *client, struct json_object *data, bool error)
{
    if (error) {
        const char *msg = "n/a";
        struct json_object *desc;
        if (json_object_object_get_ex(data, "desc", &desc)) {
            msg = json_object_get_string(desc);
        }
        fprintf(stderr, "%s: received error from QMP: %s\n",
                client->qemu.vmid, msg);
        client->state = STATE_IDLE;
        goto out;
    }

    struct json_object *status;
    json_bool has_status = data &&
	json_object_object_get_ex(data, "status", &status);

    bool active = false;
    if (has_status) {
	const char *status_str = json_object_get_string(status);
	active = status_str &&
	    (!strcmp(status_str, "running") || !strcmp(status_str, "paused"));
    }

    switch (client->state) {
	case STATE_EXPECT_STATUS_RESP:
	    client->state = STATE_IDLE;
	    if (active) {
		VERBOSE_PRINT("%s: got status: VM is active\n", client->qemu.vmid);
	    } else if (!client->qemu.backup) {
		terminate_client(client);
	    } else {
		// if we're in a backup, don't do anything, vzdump will notify
		// us when the backup finishes
		VERBOSE_PRINT("%s: not active, but running backup - keep alive\n",
			      client->qemu.vmid);
	    }
	    break;

	// this means we received the empty return from our handshake answer
	case STATE_HANDSHAKE:
	    client->state = STATE_IDLE;
	    VERBOSE_PRINT("%s: QMP handshake complete\n", client->qemu.vmid);
	    break;

	case STATE_IDLE:
	case STATE_TERMINATING:
	    VERBOSE_PRINT("%s: spurious return value received\n",
			  client->qemu.vmid);
	    break;
    }

out:
    if (client->qemu.term_check_queued) {
	terminate_check(client);
    }
}

/*
 * VZDump specific client functions
 */

void
handle_vzdump_handshake(struct Client *client, struct json_object *data)
{
    client->state = STATE_IDLE;

    struct json_object *vmid_obj;
    json_bool has_vmid = data && json_object_object_get_ex(data, "vmid", &vmid_obj);

    if (!has_vmid) {
	VERBOSE_PRINT("pid%d: invalid vzdump handshake: no vmid\n", client->pid);
	return;
    }

    const char *vmid_str = json_object_get_string(vmid_obj);

    if (!vmid_str) {
	VERBOSE_PRINT("pid%d: invalid vzdump handshake: vmid is not a string\n", client->pid);
	return;
    }

    int res = snprintf(client->vzdump.vmid, sizeof(client->vzdump.vmid), "%s", vmid_str);
    if (res < 0 || res >= (int)sizeof(client->vzdump.vmid)) {
	VERBOSE_PRINT("pid%d: invalid vzdump handshake: vmid too long or invalid\n", client->pid);
	return;
    }

    struct Client *vmc = (struct Client*) g_hash_table_lookup(vm_clients, client->vzdump.vmid);
    if (vmc) {
	vmc->qemu.backup = true;

	// only mark as VZDUMP once we have set everything up, otherwise 'cleanup'
	// might try to access an invalid value
	client->type = CLIENT_VZDUMP;
	VERBOSE_PRINT("%s: vzdump backup started\n", client->vzdump.vmid);
    } else {
	VERBOSE_PRINT("%s: vzdump requested backup start for unregistered VM\n", client->vzdump.vmid);
    }
}

/*
 * client management functions
 */

void
add_new_client(int client_fd)
{
    struct Client *client = calloc(sizeof(struct Client), 1);
    if (client == NULL) {
	fprintf(stderr, "could not add new client - allocation failed!\n");
	fflush(stderr);
	return;
    }
    client->state = STATE_HANDSHAKE;
    client->type = CLIENT_NONE;
    client->fd = client_fd;
    client->pid = get_pid_from_fd(client_fd);
    if (client->pid == 0) {
	fprintf(stderr, "could not get pid from client\n");
	goto err;
    }

    struct epoll_event ev;
    ev.events = EPOLLIN;
    ev.data.ptr = client;
    int res = epoll_ctl(epoll_fd, EPOLL_CTL_ADD, client_fd, &ev);
    if (res < 0) {
	perror("epoll_ctl client add");
	goto err;
    }

    VERBOSE_PRINT("added new client, pid: %d\n", client->pid);

    return;
err:
    (void)close(client_fd);
    free(client);
}

static void
cleanup_qemu_client(struct Client *client)
{
    unsigned short graceful = client->qemu.graceful;
    unsigned short guest = client->qemu.guest;
    char vmid[sizeof(client->qemu.vmid)];
    strncpy(vmid, client->qemu.vmid, sizeof(vmid));
    g_hash_table_remove(vm_clients, &vmid); // frees key, ignore errors
    VERBOSE_PRINT("%s: executing cleanup (graceful: %d, guest: %d)\n",
		vmid, graceful, guest);

    int pid = fork();
    if (pid < 0) {
	fprintf(stderr, "fork failed: %s\n", strerror(errno));
	return;
    }
    if (pid == 0) {
	char *script = "/usr/sbin/qm";

	char *args[] = {
	    script,
	    "cleanup",
	    vmid,
	    graceful ? "1" : "0",
	    guest    ? "1" : "0",
	    NULL
	};

	execvp(script, args);
	perror("execvp");
	_exit(1);
    }
}

void
cleanup_client(struct Client *client)
{
    log_neg(epoll_ctl(epoll_fd, EPOLL_CTL_DEL, client->fd, NULL), "epoll del");
    (void)close(client->fd);

    struct Client *vmc;
    switch (client->type) {
	case CLIENT_QEMU:
	    cleanup_qemu_client(client);
	    break;

	case CLIENT_VZDUMP:
	    vmc = (struct Client*) g_hash_table_lookup(vm_clients, client->vzdump.vmid);
	    if (vmc) {
		VERBOSE_PRINT("%s: backup ended\n", client->vzdump.vmid);
		vmc->qemu.backup = false;
		terminate_check(vmc);
	    }
	    break;

	case CLIENT_NONE:
	    // do nothing, only close socket
	    break;
    }

    free(client);
}

void
terminate_client(struct Client *client)
{
    VERBOSE_PRINT("%s: terminating client (pid %d)\n", client->qemu.vmid, client->pid);

    client->state = STATE_TERMINATING;

    // open a pidfd before kill for later cleanup
    int pidfd = pidfd_open(client->pid, 0);
    if (pidfd < 0) {
	switch (errno) {
	    case ESRCH:
		// process already dead for some reason, cleanup done
		VERBOSE_PRINT("%s: failed to open pidfd, process already dead (pid %d)\n",
			      client->qemu.vmid, client->pid);
		return;

	    // otherwise fall back to just using the PID directly, but don't
	    // print if we only failed because we're running on an older kernel
	    case ENOSYS:
		break;
	    default:
		perror("failed to open QEMU pidfd for cleanup");
		break;
	}
    }

    int err = kill(client->pid, SIGTERM);
    log_neg(err, "kill");

    struct CleanupData *data_ptr = malloc(sizeof(struct CleanupData));
    struct CleanupData data = {
	.pid = client->pid,
	.pidfd = pidfd
    };
    *data_ptr = data;
    forced_cleanups = g_slist_prepend(forced_cleanups, (void *)data_ptr);

    // resets any other alarms, but will fire eventually and cleanup all
    alarm(5);
}

void
handle_client(struct Client *client)
{
    VERBOSE_PRINT("pid%d: entering handle\n", client->pid);
    ssize_t len;
    do {
	len = read(client->fd, (client->buf+client->buflen),
		   sizeof(client->buf) - client->buflen);
    } while (len < 0 && errno == EINTR);

    if (len < 0) {
	if (!(errno == EAGAIN || errno == EWOULDBLOCK)) {
	    log_neg((int)len, "read");
	    cleanup_client(client);
	}
	return;
    } else if (len == 0) {
	VERBOSE_PRINT("pid%d: got EOF\n", client->pid);
	cleanup_client(client);
	return;
    }

    VERBOSE_PRINT("pid%d: read %ld bytes\n", client->pid, len);
    client->buflen += len;

    struct json_tokener *tok = json_tokener_new();
    struct json_object *jobj = NULL;
    enum json_tokener_error jerr = json_tokener_success;
    while (jerr == json_tokener_success && client->buflen != 0) {
	jobj = json_tokener_parse_ex(tok, client->buf, (int)client->buflen);
	jerr = json_tokener_get_error(tok);
	unsigned int offset = (unsigned int)tok->char_offset;
	switch (jerr) {
	    case json_tokener_success:
		// move rest from buffer to front
		memmove(client->buf, client->buf + offset, client->buflen - offset);
		client->buflen -= offset;
		if (json_object_is_type(jobj, json_type_object)) {
		    struct json_object *obj;
		    if (json_object_object_get_ex(jobj, "QMP", &obj)) {
			handle_qmp_handshake(client);
		    } else if (json_object_object_get_ex(jobj, "event", &obj)) {
			handle_qmp_event(client, jobj);
		    } else if (json_object_object_get_ex(jobj, "return", &obj)) {
			handle_qmp_return(client, obj, false);
		    } else if (json_object_object_get_ex(jobj, "error", &obj)) {
			handle_qmp_return(client, obj, true);
		    } else if (json_object_object_get_ex(jobj, "vzdump", &obj)) {
			handle_vzdump_handshake(client, obj);
		    } // else ignore message
		}
		break;
	    case json_tokener_continue:
		if (client->buflen >= sizeof(client->buf)) {
		    VERBOSE_PRINT("pid%d: msg too large, discarding buffer\n", client->pid);
		    memset(client->buf, 0, sizeof(client->buf));
		    client->buflen = 0;
		} // else we have enough space try again after next read
		break;
	    default:
		VERBOSE_PRINT("pid%d: parse error: %d, discarding buffer\n", client->pid, jerr);
		memset(client->buf, 0, client->buflen);
		client->buflen = 0;
		break;
	}
	json_object_put(jobj);
    }
    json_tokener_free(tok);
}


/*
 * SIGALRM and cleanup handling
 *
 * terminate_client will set an alarm for 5 seconds and add its client's PID to
 * the forced_cleanups list - when the timer expires, we iterate the list and
 * attempt to issue SIGKILL to all processes which haven't yet stopped.
 */

static void
alarm_handler(__attribute__((unused)) int signum)
{
    alarm_triggered = 1;
}

static void
sigkill(void *ptr, __attribute__((unused)) void *unused)
{
    struct CleanupData data = *((struct CleanupData *)ptr);
    int err;

    if (data.pidfd > 0) {
	err = pidfd_send_signal(data.pidfd, SIGKILL, NULL, 0);
	(void)close(data.pidfd);
    } else {
	err = kill(data.pid, SIGKILL);
    }

    if (err < 0) {
	if (errno != ESRCH) {
	    fprintf(stderr, "SIGKILL cleanup of pid '%d' failed - %s\n",
		    data.pid, strerror(errno));
	}
    } else {
	fprintf(stderr, "cleanup failed, terminating pid '%d' with SIGKILL\n",
		data.pid);
    }
}

static void
handle_forced_cleanup()
{
    if (alarm_triggered) {
	VERBOSE_PRINT("clearing forced cleanup backlog\n");
	alarm_triggered = 0;
	g_slist_foreach(forced_cleanups, sigkill, NULL);
	g_slist_free_full(forced_cleanups, free);
	forced_cleanups = NULL;
    }
}


int
main(int argc, char *argv[])
{
    int opt;
    int daemonize = 1;
    char *socket_path = NULL;
    progname = argv[0];

    while ((opt = getopt(argc, argv, "hfv")) != -1) {
	switch (opt) {
	    case 'f':
		daemonize = 0;
		break;
	    case 'v':
		verbose = 1;
		break;
	    case 'h':
		usage();
		exit(EXIT_SUCCESS);
		break;
	    default:
		usage();
		exit(EXIT_FAILURE);
	}
    }

    if (optind >= argc) {
	usage();
	exit(EXIT_FAILURE);
    }

    signal(SIGCHLD, SIG_IGN);
    signal(SIGALRM, alarm_handler);

    socket_path = argv[optind];

    int sock = socket(AF_UNIX, SOCK_STREAM, 0);
    bail_neg(sock, "socket");

    struct sockaddr_un addr;
    memset(&addr, 0, sizeof(addr));
    addr.sun_family = AF_UNIX;
    strncpy(addr.sun_path, socket_path, sizeof(addr.sun_path) - 1);

    unlink(socket_path);
    bail_neg(bind(sock, (struct sockaddr*)&addr, sizeof(addr)), "bind");

    struct epoll_event ev, events[1];
    epoll_fd = epoll_create1(EPOLL_CLOEXEC);
    bail_neg(epoll_fd, "epoll_create1");

    ev.events = EPOLLIN;
    ev.data.fd = sock;
    bail_neg(epoll_ctl(epoll_fd, EPOLL_CTL_ADD, sock, &ev), "epoll_ctl");

    bail_neg(listen(sock, 10), "listen");

    if (daemonize) {
	bail_neg(daemon(0, 1), "daemon");
    }

    vm_clients = g_hash_table_new_full(g_str_hash, g_str_equal, free, NULL);

    int nevents;

    for(;;) {
	nevents = epoll_wait(epoll_fd, events, 1, -1);
	if (nevents < 0 && errno == EINTR) {
	    handle_forced_cleanup();
	    continue;
	}
	bail_neg(nevents, "epoll_wait");

	for (int n = 0; n < nevents; n++) {
	    if (events[n].data.fd == sock) {

		int conn_sock = accept4(sock, NULL, NULL, SOCK_NONBLOCK | SOCK_CLOEXEC);
		log_neg(conn_sock, "accept");
		if (conn_sock > -1) {
		    add_new_client(conn_sock);
		}
	    } else {
		handle_client((struct Client *)events[n].data.ptr);
	    }
	}

	handle_forced_cleanup();
    }
}
Commit	Line	Data
649dbf42	1	// SPDX-License-Identifier: AGPL-3.0-or-later
4c17b2e3	2	/*
649dbf42	3	Copyright (C) 2018 - 2021 Proxmox Server Solutions GmbH
4c17b2e3 DC	4
4c17b2e3 DC	5	Author: Dominik Csapak <d.csapak@proxmox.com>
aedf8208	6	Author: Stefan Reiter <s.reiter@proxmox.com>
4c17b2e3	7
aedf8208	8	Description:
4c17b2e3	9
aedf8208 TL	10	qmeventd listens on a given socket, and waits for qemu processes to
	11	connect. After accepting a connection qmeventd waits for shutdown events
	12	followed by the closing of the socket. Once that happens `qm cleanup` will
	13	be executed with following three arguments:
	14	VMID <graceful> <guest>
	15	Where `graceful` can be `1` or `0` depending if shutdown event was observed
	16	before the socket got closed. The second parameter `guest` is also boolean
	17	`1` or `0` depending if the shutdown was requested from the guest OS
	18	(i.e., the "inside").
4c17b2e3 DC	19	*/
	20
	21	#ifndef _GNU_SOURCE
	22	#define _GNU_SOURCE
	23	#endif
	24
	25	#include <errno.h>
	26	#include <fcntl.h>
aedf8208	27	#include <gmodule.h>
4c17b2e3 DC	28	#include <json.h>
	29	#include <signal.h>
	30	#include <stdbool.h>
	31	#include <stdio.h>
	32	#include <string.h>
	33	#include <sys/epoll.h>
	34	#include <sys/socket.h>
	35	#include <sys/types.h>
	36	#include <sys/un.h>
	37	#include <sys/wait.h>
	38	#include <unistd.h>
	39
	40	#include "qmeventd.h"
	41
	42	static int verbose = 0;
	43	static int epoll_fd = 0;
	44	static const char *progname;
3ff85001	45	GHashTable vm_clients; // key=vmid (freed on remove), value=Client (free manually)
4c500f16 SR	46	GSList *forced_cleanups;
4c500f16 SR	47	volatile sig_atomic_t alarm_triggered = 0;
3ff85001	48
4c17b2e3 DC	49	/*
	50	* Helper functions
	51	*/
	52
	53	static void
	54	usage()
	55	{
	56	fprintf(stderr, "Usage: %s [-f] [-v] PATH\n", progname);
	57	fprintf(stderr, " -f run in foreground (default: false)\n");
	58	fprintf(stderr, " -v verbose (default: false)\n");
	59	fprintf(stderr, " PATH use PATH for socket\n");
	60	}
	61
	62	static pid_t
	63	get_pid_from_fd(int fd)
	64	{
	65	struct ucred credentials = { .pid = 0, .uid = 0, .gid = 0 };
	66	socklen_t len = sizeof(struct ucred);
	67	log_neg(getsockopt(fd, SOL_SOCKET, SO_PEERCRED, &credentials, &len), "getsockopt");
	68	return credentials.pid;
	69	}
	70
	71	/*
	72	* reads the vmid from /proc/<pid>/cmdline
	73	* after the '-id' argument
	74	*/
	75	static unsigned long
	76	get_vmid_from_pid(pid_t pid)
	77	{
	78	char filename[32] = { 0 };
	79	int len = snprintf(filename, sizeof(filename), "/proc/%d/cmdline", pid);
	80	if (len < 0) {
	81	fprintf(stderr, "error during snprintf for %d: %s\n", pid,
	82	strerror(errno));
	83	return 0;
	84	}
	85	if ((size_t)len >= sizeof(filename)) {
	86	fprintf(stderr, "error: pid %d too long\n", pid);
	87	return 0;
	88	}
	89	FILE *fp = fopen(filename, "re");
	90	if (fp == NULL) {
	91	fprintf(stderr, "error opening %s: %s\n", filename, strerror(errno));
	92	return 0;
	93	}
	94
	95	unsigned long vmid = 0;
	96	ssize_t rc = 0;
	97	char *buf = NULL;
	98	size_t buflen = 0;
	99	while ((rc = getdelim(&buf, &buflen, '\0', fp)) >= 0) {
	100	if (!strcmp(buf, "-id")) {
	101	break;
	102	}
	103	}
	104
	105	if (rc < 0) {
	106	goto err;
	107	}
	108
	109	if (getdelim(&buf, &buflen, '\0', fp) >= 0) {
	110	if (buf[0] == '-' \|\| buf[0] == '\0') {
	111	fprintf(stderr, "invalid vmid %s\n", buf);
	112	goto ret;
113	}
114
115	errno = 0;
116	char *endptr = NULL;
117	vmid = strtoul(buf, &endptr, 10);
118	if (errno != 0) {
119	vmid = 0;
120	goto err;
121	} else if (*endptr != '\0') {
122	fprintf(stderr, "invalid vmid %s\n", buf);
123	vmid = 0;
124	}
125
126	goto ret;
127	}
128
129	err:
130	fprintf(stderr, "error parsing vmid for %d: %s\n", pid, strerror(errno));
131
132	ret:
133	free(buf);
134	fclose(fp);
135	return vmid;
136	}
137
138	static bool
139	must_write(int fd, const char *buf, size_t len)
140	{
141	ssize_t wlen;
142	do {
143	wlen = write(fd, buf, len);
144	} while (wlen < 0 && errno == EINTR);
145
146	return (wlen == (ssize_t)len);
147	}
148
149	/*
150	* qmp handling functions
151	*/
152
3ff85001 SR	153	static void
	154	send_qmp_cmd(struct Client client, const char buf, size_t len)
	155	{
	156	if (!must_write(client->fd, buf, len - 1)) {
	157	fprintf(stderr, "%s: cannot send QMP message\n", client->qemu.vmid);
	158	cleanup_client(client);
	159	}
	160	}
	161
4c17b2e3 DC	162	void
	163	handle_qmp_handshake(struct Client *client)
	164	{
3ff85001 SR	165	VERBOSE_PRINT("pid%d: got QMP handshake, assuming QEMU client\n", client->pid);
	166
	167	// extract vmid from cmdline, now that we know it's a QEMU process
	168	unsigned long vmid = get_vmid_from_pid(client->pid);
	169	int res = snprintf(client->qemu.vmid, sizeof(client->qemu.vmid), "%lu", vmid);
	170	if (vmid == 0 \|\| res < 0 \|\| res >= (int)sizeof(client->qemu.vmid)) {
	171	fprintf(stderr, "could not get vmid from pid %d\n", client->pid);
4c17b2e3	172	cleanup_client(client);
3ff85001 SR	173	return;
	174	}
	175
	176	VERBOSE_PRINT("pid%d: assigned VMID: %s\n", client->pid, client->qemu.vmid);
	177	client->type = CLIENT_QEMU;
	178	if(!g_hash_table_insert(vm_clients, strdup(client->qemu.vmid), client)) {
	179	// not fatal, just means backup handling won't work
	180	fprintf(stderr, "%s: could not insert client into VMID->client table\n",
	181	client->qemu.vmid);
4c17b2e3	182	}
3ff85001 SR	183
	184	static const char qmp_answer[] = "{\"execute\":\"qmp_capabilities\"}\n";
	185	send_qmp_cmd(client, qmp_answer, sizeof(qmp_answer));
4c17b2e3 DC	186	}
	187
	188	void
	189	handle_qmp_event(struct Client client, struct json_object obj)
	190	{
	191	struct json_object *event;
	192	if (!json_object_object_get_ex(obj, "event", &event)) {
	193	return;
	194	}
a2488e4c	195	VERBOSE_PRINT("%s: got QMP event: %s\n", client->qemu.vmid, json_object_get_string(event));
3ff85001 SR	196
	197	if (client->state == STATE_TERMINATING) {
	198	// QEMU sometimes sends a second SHUTDOWN after SIGTERM, ignore
a2488e4c	199	VERBOSE_PRINT("%s: event was after termination, ignoring\n", client->qemu.vmid);
3ff85001 SR	200	return;
	201	}
	202
4c17b2e3 DC	203	// event, check if shutdown and get guest parameter
4c17b2e3 DC	204	if (!strcmp(json_object_get_string(event), "SHUTDOWN")) {
3ff85001	205	client->qemu.graceful = 1;
4c17b2e3 DC	206	struct json_object *data;
	207	struct json_object *guest;
	208	if (json_object_object_get_ex(obj, "data", &data) &&
	209	json_object_object_get_ex(data, "guest", &guest))
	210	{
3ff85001	211	client->qemu.guest = (unsigned short)json_object_get_boolean(guest);
4c17b2e3	212	}
3ff85001 SR	213
	214	// check if a backup is running and kill QEMU process if not
	215	terminate_check(client);
	216	}
	217	}
	218
	219	void
	220	terminate_check(struct Client *client)
	221	{
	222	if (client->state != STATE_IDLE) {
	223	// if we're already in a request, queue this one until after
	224	VERBOSE_PRINT("%s: terminate_check queued\n", client->qemu.vmid);
	225	client->qemu.term_check_queued = true;
	226	return;
	227	}
	228
	229	client->qemu.term_check_queued = false;
	230
	231	VERBOSE_PRINT("%s: query-status\n", client->qemu.vmid);
	232	client->state = STATE_EXPECT_STATUS_RESP;
	233	static const char qmp_req[] = "{\"execute\":\"query-status\"}\n";
	234	send_qmp_cmd(client, qmp_req, sizeof(qmp_req));
	235	}
	236
	237	void
	238	handle_qmp_return(struct Client client, struct json_object data, bool error)
	239	{
	240	if (error) {
	241	const char *msg = "n/a";
	242	struct json_object *desc;
	243	if (json_object_object_get_ex(data, "desc", &desc)) {
	244	msg = json_object_get_string(desc);
	245	}
	246	fprintf(stderr, "%s: received error from QMP: %s\n",
	247	client->qemu.vmid, msg);
	248	client->state = STATE_IDLE;
	249	goto out;
	250	}
	251
	252	struct json_object *status;
	253	json_bool has_status = data &&
	254	json_object_object_get_ex(data, "status", &status);
	255
	256	bool active = false;
	257	if (has_status) {
	258	const char *status_str = json_object_get_string(status);
	259	active = status_str &&
	260	(!strcmp(status_str, "running") \|\| !strcmp(status_str, "paused"));
	261	}
	262
	263	switch (client->state) {
	264	case STATE_EXPECT_STATUS_RESP:
	265	client->state = STATE_IDLE;
	266	if (active) {
	267	VERBOSE_PRINT("%s: got status: VM is active\n", client->qemu.vmid);
	268	} else if (!client->qemu.backup) {
	269	terminate_client(client);
	270	} else {
	271	// if we're in a backup, don't do anything, vzdump will notify
	272	// us when the backup finishes
	273	VERBOSE_PRINT("%s: not active, but running backup - keep alive\n",
	274	client->qemu.vmid);
	275	}
	276	break;
277
278	// this means we received the empty return from our handshake answer
279	case STATE_HANDSHAKE:
280	client->state = STATE_IDLE;
281	VERBOSE_PRINT("%s: QMP handshake complete\n", client->qemu.vmid);
282	break;
283
284	case STATE_IDLE:
285	case STATE_TERMINATING:
286	VERBOSE_PRINT("%s: spurious return value received\n",
287	client->qemu.vmid);
288	break;
289	}
290
291	out:
292	if (client->qemu.term_check_queued) {
293	terminate_check(client);
294	}
295	}
296
297	/*
298	* VZDump specific client functions
299	*/
300
301	void
302	handle_vzdump_handshake(struct Client client, struct json_object data)
303	{
304	client->state = STATE_IDLE;
305
306	struct json_object *vmid_obj;
307	json_bool has_vmid = data && json_object_object_get_ex(data, "vmid", &vmid_obj);
308
309	if (!has_vmid) {
42edf948	310	VERBOSE_PRINT("pid%d: invalid vzdump handshake: no vmid\n", client->pid);
3ff85001 SR	311	return;
	312	}
	313
	314	const char *vmid_str = json_object_get_string(vmid_obj);
	315
	316	if (!vmid_str) {
42edf948	317	VERBOSE_PRINT("pid%d: invalid vzdump handshake: vmid is not a string\n", client->pid);
3ff85001 SR	318	return;
	319	}
	320
	321	int res = snprintf(client->vzdump.vmid, sizeof(client->vzdump.vmid), "%s", vmid_str);
	322	if (res < 0 \|\| res >= (int)sizeof(client->vzdump.vmid)) {
42edf948	323	VERBOSE_PRINT("pid%d: invalid vzdump handshake: vmid too long or invalid\n", client->pid);
3ff85001 SR	324	return;
	325	}
	326
42edf948	327	struct Client vmc = (struct Client) g_hash_table_lookup(vm_clients, client->vzdump.vmid);
3ff85001 SR	328	if (vmc) {
	329	vmc->qemu.backup = true;
	330
	331	// only mark as VZDUMP once we have set everything up, otherwise 'cleanup'
	332	// might try to access an invalid value
	333	client->type = CLIENT_VZDUMP;
42edf948	334	VERBOSE_PRINT("%s: vzdump backup started\n", client->vzdump.vmid);
3ff85001	335	} else {
42edf948	336	VERBOSE_PRINT("%s: vzdump requested backup start for unregistered VM\n", client->vzdump.vmid);
4c17b2e3 DC	337	}
	338	}
	339
	340	/*
	341	* client management functions
	342	*/
	343
	344	void
	345	add_new_client(int client_fd)
	346	{
	347	struct Client *client = calloc(sizeof(struct Client), 1);
6d4f89b6 TL	348	if (client == NULL) {
	349	fprintf(stderr, "could not add new client - allocation failed!\n");
	350	fflush(stderr);
	351	return;
	352	}
3ff85001 SR	353	client->state = STATE_HANDSHAKE;
3ff85001 SR	354	client->type = CLIENT_NONE;
4c17b2e3 DC	355	client->fd = client_fd;
	356	client->pid = get_pid_from_fd(client_fd);
	357	if (client->pid == 0) {
	358	fprintf(stderr, "could not get pid from client\n");
	359	goto err;
	360	}
4c17b2e3 DC	361
	362	struct epoll_event ev;
	363	ev.events = EPOLLIN;
	364	ev.data.ptr = client;
3ff85001	365	int res = epoll_ctl(epoll_fd, EPOLL_CTL_ADD, client_fd, &ev);
4c17b2e3 DC	366	if (res < 0) {
	367	perror("epoll_ctl client add");
	368	goto err;
	369	}
	370
3ff85001	371	VERBOSE_PRINT("added new client, pid: %d\n", client->pid);
4c17b2e3 DC	372
	373	return;
	374	err:
	375	(void)close(client_fd);
	376	free(client);
	377	}
	378
3ff85001 SR	379	static void
3ff85001 SR	380	cleanup_qemu_client(struct Client *client)
4c17b2e3	381	{
3ff85001 SR	382	unsigned short graceful = client->qemu.graceful;
	383	unsigned short guest = client->qemu.guest;
	384	char vmid[sizeof(client->qemu.vmid)];
	385	strncpy(vmid, client->qemu.vmid, sizeof(vmid));
	386	g_hash_table_remove(vm_clients, &vmid); // frees key, ignore errors
	387	VERBOSE_PRINT("%s: executing cleanup (graceful: %d, guest: %d)\n",
	388	vmid, graceful, guest);
4c17b2e3 DC	389
	390	int pid = fork();
	391	if (pid < 0) {
	392	fprintf(stderr, "fork failed: %s\n", strerror(errno));
	393	return;
	394	}
	395	if (pid == 0) {
	396	char *script = "/usr/sbin/qm";
	397
	398	char *args[] = {
	399	script,
	400	"cleanup",
	401	vmid,
	402	graceful ? "1" : "0",
	403	guest ? "1" : "0",
	404	NULL
	405	};
	406
	407	execvp(script, args);
	408	perror("execvp");
	409	_exit(1);
	410	}
	411	}
	412
3ff85001 SR	413	void
	414	cleanup_client(struct Client *client)
	415	{
	416	log_neg(epoll_ctl(epoll_fd, EPOLL_CTL_DEL, client->fd, NULL), "epoll del");
	417	(void)close(client->fd);
	418
	419	struct Client *vmc;
	420	switch (client->type) {
	421	case CLIENT_QEMU:
	422	cleanup_qemu_client(client);
	423	break;
	424
	425	case CLIENT_VZDUMP:
	426	vmc = (struct Client*) g_hash_table_lookup(vm_clients, client->vzdump.vmid);
	427	if (vmc) {
	428	VERBOSE_PRINT("%s: backup ended\n", client->vzdump.vmid);
	429	vmc->qemu.backup = false;
	430	terminate_check(vmc);
	431	}
	432	break;
	433
	434	case CLIENT_NONE:
	435	// do nothing, only close socket
	436	break;
	437	}
	438
	439	free(client);
	440	}
	441
	442	void
	443	terminate_client(struct Client *client)
	444	{
42edf948	445	VERBOSE_PRINT("%s: terminating client (pid %d)\n", client->qemu.vmid, client->pid);
3ff85001 SR	446
	447	client->state = STATE_TERMINATING;
	448
4c500f16 SR	449	// open a pidfd before kill for later cleanup
	450	int pidfd = pidfd_open(client->pid, 0);
	451	if (pidfd < 0) {
	452	switch (errno) {
	453	case ESRCH:
	454	// process already dead for some reason, cleanup done
	455	VERBOSE_PRINT("%s: failed to open pidfd, process already dead (pid %d)\n",
	456	client->qemu.vmid, client->pid);
	457	return;
	458
	459	// otherwise fall back to just using the PID directly, but don't
	460	// print if we only failed because we're running on an older kernel
	461	case ENOSYS:
	462	break;
	463	default:
	464	perror("failed to open QEMU pidfd for cleanup");
	465	break;
	466	}
	467	}
	468
3ff85001 SR	469	int err = kill(client->pid, SIGTERM);
3ff85001 SR	470	log_neg(err, "kill");
4c500f16 SR	471
	472	struct CleanupData *data_ptr = malloc(sizeof(struct CleanupData));
	473	struct CleanupData data = {
	474	.pid = client->pid,
	475	.pidfd = pidfd
	476	};
	477	*data_ptr = data;
	478	forced_cleanups = g_slist_prepend(forced_cleanups, (void *)data_ptr);
	479
	480	// resets any other alarms, but will fire eventually and cleanup all
	481	alarm(5);
3ff85001 SR	482	}
3ff85001 SR	483
4c17b2e3 DC	484	void
	485	handle_client(struct Client *client)
	486	{
3ff85001	487	VERBOSE_PRINT("pid%d: entering handle\n", client->pid);
4c17b2e3 DC	488	ssize_t len;
	489	do {
	490	len = read(client->fd, (client->buf+client->buflen),
	491	sizeof(client->buf) - client->buflen);
	492	} while (len < 0 && errno == EINTR);
	493
	494	if (len < 0) {
	495	if (!(errno == EAGAIN \|\| errno == EWOULDBLOCK)) {
	496	log_neg((int)len, "read");
	497	cleanup_client(client);
	498	}
	499	return;
	500	} else if (len == 0) {
3ff85001	501	VERBOSE_PRINT("pid%d: got EOF\n", client->pid);
4c17b2e3 DC	502	cleanup_client(client);
	503	return;
	504	}
	505
3ff85001	506	VERBOSE_PRINT("pid%d: read %ld bytes\n", client->pid, len);
4c17b2e3 DC	507	client->buflen += len;
	508
	509	struct json_tokener *tok = json_tokener_new();
	510	struct json_object *jobj = NULL;
	511	enum json_tokener_error jerr = json_tokener_success;
	512	while (jerr == json_tokener_success && client->buflen != 0) {
	513	jobj = json_tokener_parse_ex(tok, client->buf, (int)client->buflen);
	514	jerr = json_tokener_get_error(tok);
	515	unsigned int offset = (unsigned int)tok->char_offset;
	516	switch (jerr) {
	517	case json_tokener_success:
	518	// move rest from buffer to front
	519	memmove(client->buf, client->buf + offset, client->buflen - offset);
	520	client->buflen -= offset;
	521	if (json_object_is_type(jobj, json_type_object)) {
	522	struct json_object *obj;
	523	if (json_object_object_get_ex(jobj, "QMP", &obj)) {
	524	handle_qmp_handshake(client);
	525	} else if (json_object_object_get_ex(jobj, "event", &obj)) {
	526	handle_qmp_event(client, jobj);
3ff85001 SR	527	} else if (json_object_object_get_ex(jobj, "return", &obj)) {
	528	handle_qmp_return(client, obj, false);
	529	} else if (json_object_object_get_ex(jobj, "error", &obj)) {
	530	handle_qmp_return(client, obj, true);
	531	} else if (json_object_object_get_ex(jobj, "vzdump", &obj)) {
	532	handle_vzdump_handshake(client, obj);
4c17b2e3 DC	533	} // else ignore message
	534	}
	535	break;
	536	case json_tokener_continue:
	537	if (client->buflen >= sizeof(client->buf)) {
a2488e4c	538	VERBOSE_PRINT("pid%d: msg too large, discarding buffer\n", client->pid);
4c17b2e3 DC	539	memset(client->buf, 0, sizeof(client->buf));
	540	client->buflen = 0;
	541	} // else we have enough space try again after next read
	542	break;
	543	default:
a2488e4c	544	VERBOSE_PRINT("pid%d: parse error: %d, discarding buffer\n", client->pid, jerr);
4c17b2e3 DC	545	memset(client->buf, 0, client->buflen);
	546	client->buflen = 0;
	547	break;
	548	}
	549	json_object_put(jobj);
	550	}
	551	json_tokener_free(tok);
	552	}
	553
	554
4c500f16 SR	555	/*
	556	* SIGALRM and cleanup handling
	557	*
	558	* terminate_client will set an alarm for 5 seconds and add its client's PID to
	559	* the forced_cleanups list - when the timer expires, we iterate the list and
	560	* attempt to issue SIGKILL to all processes which haven't yet stopped.
	561	*/
	562
	563	static void
	564	alarm_handler(__attribute__((unused)) int signum)
	565	{
	566	alarm_triggered = 1;
	567	}
	568
	569	static void
	570	sigkill(void ptr, __attribute__((unused)) void unused)
	571	{
	572	struct CleanupData data = ((struct CleanupData )ptr);
	573	int err;
	574
	575	if (data.pidfd > 0) {
	576	err = pidfd_send_signal(data.pidfd, SIGKILL, NULL, 0);
0a279963	577	(void)close(data.pidfd);
4c500f16 SR	578	} else {
	579	err = kill(data.pid, SIGKILL);
	580	}
	581
	582	if (err < 0) {
	583	if (errno != ESRCH) {
	584	fprintf(stderr, "SIGKILL cleanup of pid '%d' failed - %s\n",
	585	data.pid, strerror(errno));
	586	}
	587	} else {
	588	fprintf(stderr, "cleanup failed, terminating pid '%d' with SIGKILL\n",
	589	data.pid);
	590	}
	591	}
	592
	593	static void
	594	handle_forced_cleanup()
	595	{
	596	if (alarm_triggered) {
0a279963	597	VERBOSE_PRINT("clearing forced cleanup backlog\n");
4c500f16 SR	598	alarm_triggered = 0;
	599	g_slist_foreach(forced_cleanups, sigkill, NULL);
	600	g_slist_free_full(forced_cleanups, free);
	601	forced_cleanups = NULL;
	602	}
	603	}
	604
	605
4c17b2e3 DC	606	int
	607	main(int argc, char *argv[])
	608	{
	609	int opt;
	610	int daemonize = 1;
	611	char *socket_path = NULL;
	612	progname = argv[0];
	613
	614	while ((opt = getopt(argc, argv, "hfv")) != -1) {
	615	switch (opt) {
	616	case 'f':
	617	daemonize = 0;
	618	break;
	619	case 'v':
	620	verbose = 1;
	621	break;
	622	case 'h':
	623	usage();
	624	exit(EXIT_SUCCESS);
	625	break;
	626	default:
	627	usage();
	628	exit(EXIT_FAILURE);
	629	}
	630	}
	631
	632	if (optind >= argc) {
	633	usage();
	634	exit(EXIT_FAILURE);
	635	}
	636
	637	signal(SIGCHLD, SIG_IGN);
4c500f16	638	signal(SIGALRM, alarm_handler);
4c17b2e3 DC	639
	640	socket_path = argv[optind];
	641
	642	int sock = socket(AF_UNIX, SOCK_STREAM, 0);
	643	bail_neg(sock, "socket");
	644
	645	struct sockaddr_un addr;
	646	memset(&addr, 0, sizeof(addr));
	647	addr.sun_family = AF_UNIX;
	648	strncpy(addr.sun_path, socket_path, sizeof(addr.sun_path) - 1);
	649
	650	unlink(socket_path);
	651	bail_neg(bind(sock, (struct sockaddr*)&addr, sizeof(addr)), "bind");
	652
	653	struct epoll_event ev, events[1];
	654	epoll_fd = epoll_create1(EPOLL_CLOEXEC);
	655	bail_neg(epoll_fd, "epoll_create1");
	656
	657	ev.events = EPOLLIN;
	658	ev.data.fd = sock;
	659	bail_neg(epoll_ctl(epoll_fd, EPOLL_CTL_ADD, sock, &ev), "epoll_ctl");
	660
	661	bail_neg(listen(sock, 10), "listen");
	662
	663	if (daemonize) {
	664	bail_neg(daemon(0, 1), "daemon");
	665	}
	666
3ff85001 SR	667	vm_clients = g_hash_table_new_full(g_str_hash, g_str_equal, free, NULL);
3ff85001 SR	668
4c17b2e3 DC	669	int nevents;
	670
	671	for(;;) {
	672	nevents = epoll_wait(epoll_fd, events, 1, -1);
	673	if (nevents < 0 && errno == EINTR) {
4c500f16	674	handle_forced_cleanup();
4c17b2e3 DC	675	continue;
	676	}
	677	bail_neg(nevents, "epoll_wait");
	678
	679	for (int n = 0; n < nevents; n++) {
	680	if (events[n].data.fd == sock) {
	681
42edf948	682	int conn_sock = accept4(sock, NULL, NULL, SOCK_NONBLOCK \| SOCK_CLOEXEC);
4c17b2e3 DC	683	log_neg(conn_sock, "accept");
	684	if (conn_sock > -1) {
	685	add_new_client(conn_sock);
	686	}
	687	} else {
	688	handle_client((struct Client *)events[n].data.ptr);
	689	}
	690	}
4c500f16 SR	691
4c500f16 SR	692	handle_forced_cleanup();
4c17b2e3 DC	693	}
4c17b2e3 DC	694	}