[mirror_ovs.git] / datapath / meter.c

/*
 * Copyright (c) 2017 Nicira, Inc.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of version 2 of the GNU General Public
 * License as published by the Free Software Foundation.
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/if.h>
#include <linux/skbuff.h>
#include <linux/ip.h>
#include <linux/kernel.h>
#include <linux/openvswitch.h>
#include <linux/overflow.h>
#include <linux/netlink.h>
#include <linux/rculist.h>

#include <net/netlink.h>
#include <net/genetlink.h>
#include <linux/mm.h>

#include "datapath.h"
#include "meter.h"

#define METER_HASH_BUCKETS 1024

static const struct nla_policy meter_policy[OVS_METER_ATTR_MAX + 1] = {
	[OVS_METER_ATTR_ID] = { .type = NLA_U32, },
	[OVS_METER_ATTR_KBPS] = { .type = NLA_FLAG },
	[OVS_METER_ATTR_STATS] = { .len = sizeof(struct ovs_flow_stats) },
	[OVS_METER_ATTR_BANDS] = { .type = NLA_NESTED },
	[OVS_METER_ATTR_USED] = { .type = NLA_U64 },
	[OVS_METER_ATTR_CLEAR] = { .type = NLA_FLAG },
	[OVS_METER_ATTR_MAX_METERS] = { .type = NLA_U32 },
	[OVS_METER_ATTR_MAX_BANDS] = { .type = NLA_U32 },
};

static const struct nla_policy band_policy[OVS_BAND_ATTR_MAX + 1] = {
	[OVS_BAND_ATTR_TYPE] = { .type = NLA_U32, },
	[OVS_BAND_ATTR_RATE] = { .type = NLA_U32, },
	[OVS_BAND_ATTR_BURST] = { .type = NLA_U32, },
	[OVS_BAND_ATTR_STATS] = { .len = sizeof(struct ovs_flow_stats) },
};

static void ovs_meter_free(struct dp_meter *meter)
{
	if (!meter)
		return;

	kfree_rcu(meter, rcu);
}

static struct hlist_head *meter_hash_bucket(const struct datapath *dp,
					    u32 meter_id)
{
	return &dp->meters[meter_id & (METER_HASH_BUCKETS - 1)];
}

/* Call with ovs_mutex or RCU read lock. */
static struct dp_meter *lookup_meter(const struct datapath *dp,
				     u32 meter_id)
{
	struct dp_meter *meter;
	struct hlist_head *head;

	head = meter_hash_bucket(dp, meter_id);
	hlist_for_each_entry_rcu(meter, head, dp_hash_node) {
		if (meter->id == meter_id)
			return meter;
	}
	return NULL;
}

static void attach_meter(struct datapath *dp, struct dp_meter *meter)
{
	struct hlist_head *head = meter_hash_bucket(dp, meter->id);

	hlist_add_head_rcu(&meter->dp_hash_node, head);
}

static void detach_meter(struct dp_meter *meter)
{
	ASSERT_OVSL();
	if (meter)
		hlist_del_rcu(&meter->dp_hash_node);
}

static struct sk_buff *
ovs_meter_cmd_reply_start(struct genl_info *info, u8 cmd,
			  struct ovs_header **ovs_reply_header)
{
	struct sk_buff *skb;
	struct ovs_header *ovs_header = info->userhdr;

	skb = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_ATOMIC);
	if (!skb)
		return ERR_PTR(-ENOMEM);

	*ovs_reply_header = genlmsg_put(skb, info->snd_portid,
					info->snd_seq,
					&dp_meter_genl_family, 0, cmd);
	if (!*ovs_reply_header) {
		nlmsg_free(skb);
		return ERR_PTR(-EMSGSIZE);
	}
	(*ovs_reply_header)->dp_ifindex = ovs_header->dp_ifindex;

	return skb;
}

static int ovs_meter_cmd_reply_stats(struct sk_buff *reply, u32 meter_id,
				     struct dp_meter *meter)
{
	struct nlattr *nla;
	struct dp_meter_band *band;
	u16 i;

	if (nla_put_u32(reply, OVS_METER_ATTR_ID, meter_id))
		goto error;

	if (!meter)
		return 0;

	if (nla_put(reply, OVS_METER_ATTR_STATS,
		    sizeof(struct ovs_flow_stats), &meter->stats) ||
	    nla_put_u64_64bit(reply, OVS_METER_ATTR_USED, meter->used,
			      OVS_METER_ATTR_PAD))
		goto error;

	nla = nla_nest_start_noflag(reply, OVS_METER_ATTR_BANDS);
	if (!nla)
		goto error;

	band = meter->bands;

	for (i = 0; i < meter->n_bands; ++i, ++band) {
		struct nlattr *band_nla;

		band_nla = nla_nest_start_noflag(reply, OVS_BAND_ATTR_UNSPEC);
		if (!band_nla || nla_put(reply, OVS_BAND_ATTR_STATS,
					 sizeof(struct ovs_flow_stats),
					 &band->stats))
			goto error;
		nla_nest_end(reply, band_nla);
	}
	nla_nest_end(reply, nla);

	return 0;
error:
	return -EMSGSIZE;
}

static int ovs_meter_cmd_features(struct sk_buff *skb, struct genl_info *info)
{
	struct sk_buff *reply;
	struct ovs_header *ovs_reply_header;
	struct nlattr *nla, *band_nla;
	int err;

	reply = ovs_meter_cmd_reply_start(info, OVS_METER_CMD_FEATURES,
					  &ovs_reply_header);
	if (IS_ERR(reply))
		return PTR_ERR(reply);

	if (nla_put_u32(reply, OVS_METER_ATTR_MAX_METERS, U32_MAX) ||
	    nla_put_u32(reply, OVS_METER_ATTR_MAX_BANDS, DP_MAX_BANDS))
		goto nla_put_failure;

	nla = nla_nest_start_noflag(reply, OVS_METER_ATTR_BANDS);
	if (!nla)
		goto nla_put_failure;

	band_nla = nla_nest_start_noflag(reply, OVS_BAND_ATTR_UNSPEC);
	if (!band_nla)
		goto nla_put_failure;
	/* Currently only DROP band type is supported. */
	if (nla_put_u32(reply, OVS_BAND_ATTR_TYPE, OVS_METER_BAND_TYPE_DROP))
		goto nla_put_failure;
	nla_nest_end(reply, band_nla);
	nla_nest_end(reply, nla);

	genlmsg_end(reply, ovs_reply_header);
	return genlmsg_reply(reply, info);

nla_put_failure:
	nlmsg_free(reply);
	err = -EMSGSIZE;
	return err;
}

#ifndef HAVE_KTIME_GET_NS
#ifndef ktime_to_ns
#define ktime_to_ns(kt)			((kt).tv64)
#endif
static inline u64 ktime_get_ns(void)
{
	return ktime_to_ns(ktime_get());
}
#endif

static struct dp_meter *dp_meter_create(struct nlattr **a)
{
	struct nlattr *nla;
	int rem;
	u16 n_bands = 0;
	struct dp_meter *meter;
	struct dp_meter_band *band;
	int err;

	/* Validate attributes, count the bands. */
	if (!a[OVS_METER_ATTR_BANDS])
		return ERR_PTR(-EINVAL);

	nla_for_each_nested(nla, a[OVS_METER_ATTR_BANDS], rem)
		if (++n_bands > DP_MAX_BANDS)
			return ERR_PTR(-EINVAL);

	/* Allocate and set up the meter before locking anything. */
	meter = kzalloc(struct_size(meter, bands, n_bands), GFP_KERNEL);
	if (!meter)
		return ERR_PTR(-ENOMEM);

	meter->id = nla_get_u32(a[OVS_METER_ATTR_ID]);
	meter->used = div_u64(ktime_get_ns(), 1000 * 1000);
	meter->kbps = a[OVS_METER_ATTR_KBPS] ? 1 : 0;
	meter->keep_stats = !a[OVS_METER_ATTR_CLEAR];
	spin_lock_init(&meter->lock);
	if (meter->keep_stats && a[OVS_METER_ATTR_STATS]) {
		meter->stats = *(struct ovs_flow_stats *)
			nla_data(a[OVS_METER_ATTR_STATS]);
	}
	meter->n_bands = n_bands;

	/* Set up meter bands. */
	band = meter->bands;
	nla_for_each_nested(nla, a[OVS_METER_ATTR_BANDS], rem) {
		struct nlattr *attr[OVS_BAND_ATTR_MAX + 1];
		u32 band_max_delta_t;

		err = nla_parse_deprecated_strict((struct nlattr **)&attr,
						  OVS_BAND_ATTR_MAX,
						  nla_data(nla),
						  nla_len(nla),
						  band_policy, NULL);
		if (err)
			goto exit_free_meter;

		if (!attr[OVS_BAND_ATTR_TYPE] ||
		    !attr[OVS_BAND_ATTR_RATE] ||
		    !attr[OVS_BAND_ATTR_BURST]) {
			err = -EINVAL;
			goto exit_free_meter;
		}

		band->type = nla_get_u32(attr[OVS_BAND_ATTR_TYPE]);
		band->rate = nla_get_u32(attr[OVS_BAND_ATTR_RATE]);
		if (band->rate == 0) {
			err = -EINVAL;
			goto exit_free_meter;
		}

		band->burst_size = nla_get_u32(attr[OVS_BAND_ATTR_BURST]);
		/* Figure out max delta_t that is enough to fill any bucket.
		 * Keep max_delta_t size to the bucket units:
		 * pkts => 1/1000 packets, kilobits => bits.
		 *
		 * Start with a full bucket.
		 */
		band->bucket = (band->burst_size + band->rate) * 1000;
		band_max_delta_t = band->bucket / band->rate;
		if (band_max_delta_t > meter->max_delta_t)
			meter->max_delta_t = band_max_delta_t;
		band++;
	}

	return meter;

exit_free_meter:
	kfree(meter);
	return ERR_PTR(err);
}

static int ovs_meter_cmd_set(struct sk_buff *skb, struct genl_info *info)
{
	struct nlattr **a = info->attrs;
	struct dp_meter *meter, *old_meter;
	struct sk_buff *reply;
	struct ovs_header *ovs_reply_header;
	struct ovs_header *ovs_header = info->userhdr;
	struct datapath *dp;
	int err;
	u32 meter_id;
	bool failed;

	if (!a[OVS_METER_ATTR_ID]) {
		return -ENODEV;
	}

	meter = dp_meter_create(a);
	if (IS_ERR_OR_NULL(meter))
		return PTR_ERR(meter);

	reply = ovs_meter_cmd_reply_start(info, OVS_METER_CMD_SET,
					  &ovs_reply_header);
	if (IS_ERR(reply)) {
		err = PTR_ERR(reply);
		goto exit_free_meter;
	}

	ovs_lock();
	dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
	if (!dp) {
		err = -ENODEV;
		goto exit_unlock;
	}

	meter_id = nla_get_u32(a[OVS_METER_ATTR_ID]);

	/* Cannot fail after this. */
	old_meter = lookup_meter(dp, meter_id);
	detach_meter(old_meter);
	attach_meter(dp, meter);
	ovs_unlock();

	/* Build response with the meter_id and stats from
	 * the old meter, if any.
	 */
	failed = nla_put_u32(reply, OVS_METER_ATTR_ID, meter_id);
	WARN_ON(failed);
	if (old_meter) {
		spin_lock_bh(&old_meter->lock);
		if (old_meter->keep_stats) {
			err = ovs_meter_cmd_reply_stats(reply, meter_id,
							old_meter);
			WARN_ON(err);
		}
		spin_unlock_bh(&old_meter->lock);
		ovs_meter_free(old_meter);
	}

	genlmsg_end(reply, ovs_reply_header);
	return genlmsg_reply(reply, info);

exit_unlock:
	ovs_unlock();
	nlmsg_free(reply);
exit_free_meter:
	kfree(meter);
	return err;
}

static int ovs_meter_cmd_get(struct sk_buff *skb, struct genl_info *info)
{
	struct nlattr **a = info->attrs;
	u32 meter_id;
	struct ovs_header *ovs_header = info->userhdr;
	struct ovs_header *ovs_reply_header;
	struct datapath *dp;
	int err;
	struct sk_buff *reply;
	struct dp_meter *meter;

	if (!a[OVS_METER_ATTR_ID])
		return -EINVAL;

	meter_id = nla_get_u32(a[OVS_METER_ATTR_ID]);

	reply = ovs_meter_cmd_reply_start(info, OVS_METER_CMD_GET,
					  &ovs_reply_header);
	if (IS_ERR(reply))
		return PTR_ERR(reply);

	ovs_lock();

	dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
	if (!dp) {
		err = -ENODEV;
		goto exit_unlock;
	}

	/* Locate meter, copy stats. */
	meter = lookup_meter(dp, meter_id);
	if (!meter) {
		err = -ENOENT;
		goto exit_unlock;
	}

	spin_lock_bh(&meter->lock);
	err = ovs_meter_cmd_reply_stats(reply, meter_id, meter);
	spin_unlock_bh(&meter->lock);
	if (err)
		goto exit_unlock;

	ovs_unlock();

	genlmsg_end(reply, ovs_reply_header);
	return genlmsg_reply(reply, info);

exit_unlock:
	ovs_unlock();
	nlmsg_free(reply);
	return err;
}

static int ovs_meter_cmd_del(struct sk_buff *skb, struct genl_info *info)
{
	struct nlattr **a = info->attrs;
	u32 meter_id;
	struct ovs_header *ovs_header = info->userhdr;
	struct ovs_header *ovs_reply_header;
	struct datapath *dp;
	int err;
	struct sk_buff *reply;
	struct dp_meter *old_meter;

	if (!a[OVS_METER_ATTR_ID])
		return -EINVAL;
	meter_id = nla_get_u32(a[OVS_METER_ATTR_ID]);

	reply = ovs_meter_cmd_reply_start(info, OVS_METER_CMD_DEL,
					  &ovs_reply_header);
	if (IS_ERR(reply))
		return PTR_ERR(reply);

	ovs_lock();

	dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
	if (!dp) {
		err = -ENODEV;
		goto exit_unlock;
	}

	old_meter = lookup_meter(dp, meter_id);
	if (old_meter) {
		spin_lock_bh(&old_meter->lock);
		err = ovs_meter_cmd_reply_stats(reply, meter_id, old_meter);
		WARN_ON(err);
		spin_unlock_bh(&old_meter->lock);
		detach_meter(old_meter);
	}
	ovs_unlock();
	ovs_meter_free(old_meter);
	genlmsg_end(reply, ovs_reply_header);
	return genlmsg_reply(reply, info);

exit_unlock:
	ovs_unlock();
	nlmsg_free(reply);
	return err;
}

/* Meter action execution.
 *
 * Return true 'meter_id' drop band is triggered. The 'skb' should be
 * dropped by the caller'.
 */
bool ovs_meter_execute(struct datapath *dp, struct sk_buff *skb,
		       struct sw_flow_key *key, u32 meter_id)
{
	struct dp_meter *meter;
	struct dp_meter_band *band;
	long long int now_ms = div_u64(ktime_get_ns(), 1000 * 1000);
	long long int long_delta_ms;
	u32 delta_ms;
	u32 cost;
	int i, band_exceeded_max = -1;
	u32 band_exceeded_rate = 0;

	meter = lookup_meter(dp, meter_id);
	/* Do not drop the packet when there is no meter. */
	if (!meter)
		return false;

	/* Lock the meter while using it. */
	spin_lock(&meter->lock);

	long_delta_ms = (now_ms - meter->used); /* ms */

	/* Make sure delta_ms will not be too large, so that bucket will not
	 * wrap around below.
	 */
	delta_ms = (long_delta_ms > (long long int)meter->max_delta_t)
		   ? meter->max_delta_t : (u32)long_delta_ms;

	/* Update meter statistics.
	 */
	meter->used = now_ms;
	meter->stats.n_packets += 1;
	meter->stats.n_bytes += skb->len;

	/* Bucket rate is either in kilobits per second, or in packets per
	 * second.  We maintain the bucket in the units of either bits or
	 * 1/1000th of a packet, correspondingly.
	 * Then, when rate is multiplied with milliseconds, we get the
	 * bucket units:
	 * msec * kbps = bits, and
	 * msec * packets/sec = 1/1000 packets.
	 *
	 * 'cost' is the number of bucket units in this packet.
	 */
	cost = (meter->kbps) ? skb->len * 8 : 1000;

	/* Update all bands and find the one hit with the highest rate. */
	for (i = 0; i < meter->n_bands; ++i) {
		long long int max_bucket_size;

		band = &meter->bands[i];
		max_bucket_size = (band->burst_size + band->rate) * 1000LL;

		band->bucket += delta_ms * band->rate;
		if (band->bucket > max_bucket_size)
			band->bucket = max_bucket_size;

		if (band->bucket >= cost) {
			band->bucket -= cost;
		} else if (band->rate > band_exceeded_rate) {
			band_exceeded_rate = band->rate;
			band_exceeded_max = i;
		}
	}

	if (band_exceeded_max >= 0) {
		/* Update band statistics. */
		band = &meter->bands[band_exceeded_max];
		band->stats.n_packets += 1;
		band->stats.n_bytes += skb->len;

		/* Drop band triggered, let the caller drop the 'skb'.  */
		if (band->type == OVS_METER_BAND_TYPE_DROP) {
			spin_unlock(&meter->lock);
			return true;
		}
	}

	spin_unlock(&meter->lock);
	return false;
}

static struct genl_ops dp_meter_genl_ops[] = {
	{ .cmd = OVS_METER_CMD_FEATURES,
#ifdef HAVE_GENL_VALIDATE_FLAGS
		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
#endif
		.flags = 0,		  /* OK for unprivileged users. */
#ifdef HAVE_GENL_OPS_POLICY
		.policy = meter_policy,
#endif
		.doit = ovs_meter_cmd_features
	},
	{ .cmd = OVS_METER_CMD_SET,
#ifdef HAVE_GENL_VALIDATE_FLAGS
		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
#endif
		.flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN
					   *  privilege.
					   */
#ifdef HAVE_GENL_OPS_POLICY
		.policy = meter_policy,
#endif
		.doit = ovs_meter_cmd_set,
	},
	{ .cmd = OVS_METER_CMD_GET,
#ifdef HAVE_GENL_VALIDATE_FLAGS
		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
#endif
		.flags = 0,		  /* OK for unprivileged users. */
#ifdef HAVE_GENL_OPS_POLICY
		.policy = meter_policy,
#endif
		.doit = ovs_meter_cmd_get,
	},
	{ .cmd = OVS_METER_CMD_DEL,
#ifdef HAVE_GENL_VALIDATE_FLAGS
		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
#endif
		.flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN
					   *  privilege.
					   */
#ifdef HAVE_GENL_OPS_POLICY
		.policy = meter_policy,
#endif
		.doit = ovs_meter_cmd_del
	},
};

static const struct genl_multicast_group ovs_meter_multicast_group = {
	.name = OVS_METER_MCGROUP,
};

struct genl_family dp_meter_genl_family __ro_after_init = {
	.hdrsize = sizeof(struct ovs_header),
	.name = OVS_METER_FAMILY,
	.version = OVS_METER_VERSION,
	.maxattr = OVS_METER_ATTR_MAX,
#ifndef HAVE_GENL_OPS_POLICY
	.policy = meter_policy,
#endif
	.netnsok = true,
	.parallel_ops = true,
	.ops = dp_meter_genl_ops,
	.n_ops = ARRAY_SIZE(dp_meter_genl_ops),
	.mcgrps = &ovs_meter_multicast_group,
	.n_mcgrps = 1,
	.module = THIS_MODULE,
};

int ovs_meters_init(struct datapath *dp)
{
	int i;

	dp->meters = kmalloc_array(METER_HASH_BUCKETS,
				   sizeof(struct hlist_head), GFP_KERNEL);

	if (!dp->meters)
		return -ENOMEM;

	for (i = 0; i < METER_HASH_BUCKETS; i++)
		INIT_HLIST_HEAD(&dp->meters[i]);

	return 0;
}

void ovs_meters_exit(struct datapath *dp)
{
	int i;

	for (i = 0; i < METER_HASH_BUCKETS; i++) {
		struct hlist_head *head = &dp->meters[i];
		struct dp_meter *meter;
		struct hlist_node *n;

		hlist_for_each_entry_safe(meter, n, head, dp_hash_node)
			kfree(meter);
	}

	kfree(dp->meters);
}
Commit	Line	Data
1cb57039 AZ	1	/*
	2	* Copyright (c) 2017 Nicira, Inc.
	3	*
	4	* This program is free software; you can redistribute it and/or
	5	* modify it under the terms of version 2 of the GNU General Public
	6	* License as published by the Free Software Foundation.
	7	*/
	8
	9	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
	10
	11	#include <linux/if.h>
	12	#include <linux/skbuff.h>
	13	#include <linux/ip.h>
	14	#include <linux/kernel.h>
	15	#include <linux/openvswitch.h>
8f6d230f	16	#include <linux/overflow.h>
1cb57039 AZ	17	#include <linux/netlink.h>
	18	#include <linux/rculist.h>
	19
	20	#include <net/netlink.h>
	21	#include <net/genetlink.h>
f7246940	22	#include <linux/mm.h>
1cb57039 AZ	23
	24	#include "datapath.h"
	25	#include "meter.h"
	26
	27	#define METER_HASH_BUCKETS 1024
	28
	29	static const struct nla_policy meter_policy[OVS_METER_ATTR_MAX + 1] = {
	30	[OVS_METER_ATTR_ID] = { .type = NLA_U32, },
	31	[OVS_METER_ATTR_KBPS] = { .type = NLA_FLAG },
	32	[OVS_METER_ATTR_STATS] = { .len = sizeof(struct ovs_flow_stats) },
	33	[OVS_METER_ATTR_BANDS] = { .type = NLA_NESTED },
	34	[OVS_METER_ATTR_USED] = { .type = NLA_U64 },
	35	[OVS_METER_ATTR_CLEAR] = { .type = NLA_FLAG },
	36	[OVS_METER_ATTR_MAX_METERS] = { .type = NLA_U32 },
	37	[OVS_METER_ATTR_MAX_BANDS] = { .type = NLA_U32 },
	38	};
	39
	40	static const struct nla_policy band_policy[OVS_BAND_ATTR_MAX + 1] = {
	41	[OVS_BAND_ATTR_TYPE] = { .type = NLA_U32, },
	42	[OVS_BAND_ATTR_RATE] = { .type = NLA_U32, },
	43	[OVS_BAND_ATTR_BURST] = { .type = NLA_U32, },
	44	[OVS_BAND_ATTR_STATS] = { .len = sizeof(struct ovs_flow_stats) },
	45	};
	46
1cb57039 AZ	47	static void ovs_meter_free(struct dp_meter *meter)
	48	{
	49	if (!meter)
	50	return;
	51
ab86f680	52	kfree_rcu(meter, rcu);
1cb57039 AZ	53	}
	54
	55	static struct hlist_head meter_hash_bucket(const struct datapath dp,
	56	u32 meter_id)
	57	{
	58	return &dp->meters[meter_id & (METER_HASH_BUCKETS - 1)];
	59	}
	60
	61	/* Call with ovs_mutex or RCU read lock. */
	62	static struct dp_meter lookup_meter(const struct datapath dp,
	63	u32 meter_id)
	64	{
	65	struct dp_meter *meter;
	66	struct hlist_head *head;
	67
	68	head = meter_hash_bucket(dp, meter_id);
	69	hlist_for_each_entry_rcu(meter, head, dp_hash_node) {
	70	if (meter->id == meter_id)
	71	return meter;
	72	}
	73	return NULL;
	74	}
	75
	76	static void attach_meter(struct datapath dp, struct dp_meter meter)
	77	{
	78	struct hlist_head *head = meter_hash_bucket(dp, meter->id);
	79
	80	hlist_add_head_rcu(&meter->dp_hash_node, head);
	81	}
	82
	83	static void detach_meter(struct dp_meter *meter)
	84	{
	85	ASSERT_OVSL();
	86	if (meter)
	87	hlist_del_rcu(&meter->dp_hash_node);
	88	}
	89
	90	static struct sk_buff *
	91	ovs_meter_cmd_reply_start(struct genl_info *info, u8 cmd,
	92	struct ovs_header **ovs_reply_header)
	93	{
	94	struct sk_buff *skb;
	95	struct ovs_header *ovs_header = info->userhdr;
	96
	97	skb = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_ATOMIC);
	98	if (!skb)
	99	return ERR_PTR(-ENOMEM);
	100
	101	*ovs_reply_header = genlmsg_put(skb, info->snd_portid,
	102	info->snd_seq,
	103	&dp_meter_genl_family, 0, cmd);
b88ec467	104	if (!*ovs_reply_header) {
1cb57039 AZ	105	nlmsg_free(skb);
	106	return ERR_PTR(-EMSGSIZE);
	107	}
	108	(*ovs_reply_header)->dp_ifindex = ovs_header->dp_ifindex;
	109
	110	return skb;
	111	}
	112
	113	static int ovs_meter_cmd_reply_stats(struct sk_buff *reply, u32 meter_id,
	114	struct dp_meter *meter)
	115	{
	116	struct nlattr *nla;
	117	struct dp_meter_band *band;
	118	u16 i;
	119
	120	if (nla_put_u32(reply, OVS_METER_ATTR_ID, meter_id))
	121	goto error;
	122
	123	if (!meter)
	124	return 0;
	125
	126	if (nla_put(reply, OVS_METER_ATTR_STATS,
	127	sizeof(struct ovs_flow_stats), &meter->stats) \|\|
	128	nla_put_u64_64bit(reply, OVS_METER_ATTR_USED, meter->used,
	129	OVS_METER_ATTR_PAD))
	130	goto error;
	131
09c33996	132	nla = nla_nest_start_noflag(reply, OVS_METER_ATTR_BANDS);
1cb57039 AZ	133	if (!nla)
	134	goto error;
	135
	136	band = meter->bands;
	137
	138	for (i = 0; i < meter->n_bands; ++i, ++band) {
	139	struct nlattr *band_nla;
	140
09c33996	141	band_nla = nla_nest_start_noflag(reply, OVS_BAND_ATTR_UNSPEC);
1cb57039 AZ	142	if (!band_nla \|\| nla_put(reply, OVS_BAND_ATTR_STATS,
	143	sizeof(struct ovs_flow_stats),
	144	&band->stats))
	145	goto error;
	146	nla_nest_end(reply, band_nla);
	147	}
	148	nla_nest_end(reply, nla);
	149
	150	return 0;
	151	error:
	152	return -EMSGSIZE;
	153	}
	154
	155	static int ovs_meter_cmd_features(struct sk_buff skb, struct genl_info info)
	156	{
	157	struct sk_buff *reply;
	158	struct ovs_header *ovs_reply_header;
	159	struct nlattr nla, band_nla;
	160	int err;
	161
	162	reply = ovs_meter_cmd_reply_start(info, OVS_METER_CMD_FEATURES,
	163	&ovs_reply_header);
720c5514	164	if (IS_ERR(reply))
1cb57039 AZ	165	return PTR_ERR(reply);
	166
	167	if (nla_put_u32(reply, OVS_METER_ATTR_MAX_METERS, U32_MAX) \|\|
	168	nla_put_u32(reply, OVS_METER_ATTR_MAX_BANDS, DP_MAX_BANDS))
	169	goto nla_put_failure;
	170
09c33996	171	nla = nla_nest_start_noflag(reply, OVS_METER_ATTR_BANDS);
1cb57039 AZ	172	if (!nla)
	173	goto nla_put_failure;
	174
09c33996	175	band_nla = nla_nest_start_noflag(reply, OVS_BAND_ATTR_UNSPEC);
1cb57039 AZ	176	if (!band_nla)
	177	goto nla_put_failure;
	178	/* Currently only DROP band type is supported. */
	179	if (nla_put_u32(reply, OVS_BAND_ATTR_TYPE, OVS_METER_BAND_TYPE_DROP))
	180	goto nla_put_failure;
	181	nla_nest_end(reply, band_nla);
	182	nla_nest_end(reply, nla);
	183
	184	genlmsg_end(reply, ovs_reply_header);
	185	return genlmsg_reply(reply, info);
	186
	187	nla_put_failure:
	188	nlmsg_free(reply);
	189	err = -EMSGSIZE;
	190	return err;
	191	}
	192
	193	#ifndef HAVE_KTIME_GET_NS
	194	#ifndef ktime_to_ns
	195	#define ktime_to_ns(kt) ((kt).tv64)
	196	#endif
	197	static inline u64 ktime_get_ns(void)
	198	{
	199	return ktime_to_ns(ktime_get());
	200	}
	201	#endif
	202
	203	static struct dp_meter dp_meter_create(struct nlattr *a)
	204	{
	205	struct nlattr *nla;
	206	int rem;
	207	u16 n_bands = 0;
	208	struct dp_meter *meter;
	209	struct dp_meter_band *band;
	210	int err;
	211
	212	/* Validate attributes, count the bands. */
	213	if (!a[OVS_METER_ATTR_BANDS])
	214	return ERR_PTR(-EINVAL);
	215
	216	nla_for_each_nested(nla, a[OVS_METER_ATTR_BANDS], rem)
	217	if (++n_bands > DP_MAX_BANDS)
	218	return ERR_PTR(-EINVAL);
	219
	220	/* Allocate and set up the meter before locking anything. */
f7246940	221	meter = kzalloc(struct_size(meter, bands, n_bands), GFP_KERNEL);
1cb57039 AZ	222	if (!meter)
	223	return ERR_PTR(-ENOMEM);
	224
52e20a3d	225	meter->id = nla_get_u32(a[OVS_METER_ATTR_ID]);
1cb57039 AZ	226	meter->used = div_u64(ktime_get_ns(), 1000 * 1000);
	227	meter->kbps = a[OVS_METER_ATTR_KBPS] ? 1 : 0;
	228	meter->keep_stats = !a[OVS_METER_ATTR_CLEAR];
	229	spin_lock_init(&meter->lock);
	230	if (meter->keep_stats && a[OVS_METER_ATTR_STATS]) {
	231	meter->stats = (struct ovs_flow_stats )
	232	nla_data(a[OVS_METER_ATTR_STATS]);
	233	}
	234	meter->n_bands = n_bands;
	235
	236	/* Set up meter bands. */
	237	band = meter->bands;
	238	nla_for_each_nested(nla, a[OVS_METER_ATTR_BANDS], rem) {
	239	struct nlattr *attr[OVS_BAND_ATTR_MAX + 1];
	240	u32 band_max_delta_t;
	241
6db0f72d JB	242	err = nla_parse_deprecated_strict((struct nlattr **)&attr,
	243	OVS_BAND_ATTR_MAX,
	244	nla_data(nla),
	245	nla_len(nla),
	246	band_policy, NULL);
1cb57039 AZ	247	if (err)
	248	goto exit_free_meter;
	249
	250	if (!attr[OVS_BAND_ATTR_TYPE] \|\|
	251	!attr[OVS_BAND_ATTR_RATE] \|\|
	252	!attr[OVS_BAND_ATTR_BURST]) {
	253	err = -EINVAL;
	254	goto exit_free_meter;
	255	}
	256
	257	band->type = nla_get_u32(attr[OVS_BAND_ATTR_TYPE]);
	258	band->rate = nla_get_u32(attr[OVS_BAND_ATTR_RATE]);
2db63edc	259	if (band->rate == 0) {
	260	err = -EINVAL;
	261	goto exit_free_meter;
	262	}
	263
1cb57039 AZ	264	band->burst_size = nla_get_u32(attr[OVS_BAND_ATTR_BURST]);
	265	/* Figure out max delta_t that is enough to fill any bucket.
	266	* Keep max_delta_t size to the bucket units:
	267	* pkts => 1/1000 packets, kilobits => bits.
2db63edc	268	*
2db63edc	269	* Start with a full bucket.
1cb57039	270	*/
2db63edc	271	band->bucket = (band->burst_size + band->rate) * 1000;
2db63edc	272	band_max_delta_t = band->bucket / band->rate;
1cb57039 AZ	273	if (band_max_delta_t > meter->max_delta_t)
	274	meter->max_delta_t = band_max_delta_t;
	275	band++;
	276	}
	277
	278	return meter;
	279
	280	exit_free_meter:
	281	kfree(meter);
	282	return ERR_PTR(err);
	283	}
	284
	285	static int ovs_meter_cmd_set(struct sk_buff skb, struct genl_info info)
	286	{
	287	struct nlattr **a = info->attrs;
	288	struct dp_meter meter, old_meter;
	289	struct sk_buff *reply;
	290	struct ovs_header *ovs_reply_header;
	291	struct ovs_header *ovs_header = info->userhdr;
	292	struct datapath *dp;
	293	int err;
	294	u32 meter_id;
	295	bool failed;
	296
52e20a3d JP	297	if (!a[OVS_METER_ATTR_ID]) {
	298	return -ENODEV;
	299	}
	300
1cb57039 AZ	301	meter = dp_meter_create(a);
	302	if (IS_ERR_OR_NULL(meter))
	303	return PTR_ERR(meter);
	304
	305	reply = ovs_meter_cmd_reply_start(info, OVS_METER_CMD_SET,
	306	&ovs_reply_header);
	307	if (IS_ERR(reply)) {
	308	err = PTR_ERR(reply);
	309	goto exit_free_meter;
	310	}
	311
	312	ovs_lock();
	313	dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
	314	if (!dp) {
	315	err = -ENODEV;
	316	goto exit_unlock;
	317	}
	318
1cb57039 AZ	319	meter_id = nla_get_u32(a[OVS_METER_ATTR_ID]);
	320
	321	/* Cannot fail after this. */
	322	old_meter = lookup_meter(dp, meter_id);
	323	detach_meter(old_meter);
	324	attach_meter(dp, meter);
	325	ovs_unlock();
	326
	327	/* Build response with the meter_id and stats from
	328	* the old meter, if any.
	329	*/
	330	failed = nla_put_u32(reply, OVS_METER_ATTR_ID, meter_id);
	331	WARN_ON(failed);
	332	if (old_meter) {
	333	spin_lock_bh(&old_meter->lock);
	334	if (old_meter->keep_stats) {
	335	err = ovs_meter_cmd_reply_stats(reply, meter_id,
	336	old_meter);
	337	WARN_ON(err);
	338	}
	339	spin_unlock_bh(&old_meter->lock);
	340	ovs_meter_free(old_meter);
	341	}
	342
	343	genlmsg_end(reply, ovs_reply_header);
	344	return genlmsg_reply(reply, info);
	345
	346	exit_unlock:
	347	ovs_unlock();
	348	nlmsg_free(reply);
	349	exit_free_meter:
	350	kfree(meter);
	351	return err;
	352	}
	353
	354	static int ovs_meter_cmd_get(struct sk_buff skb, struct genl_info info)
	355	{
	356	struct nlattr **a = info->attrs;
	357	u32 meter_id;
	358	struct ovs_header *ovs_header = info->userhdr;
	359	struct ovs_header *ovs_reply_header;
	360	struct datapath *dp;
	361	int err;
	362	struct sk_buff *reply;
	363	struct dp_meter *meter;
	364
	365	if (!a[OVS_METER_ATTR_ID])
	366	return -EINVAL;
	367
	368	meter_id = nla_get_u32(a[OVS_METER_ATTR_ID]);
	369
	370	reply = ovs_meter_cmd_reply_start(info, OVS_METER_CMD_GET,
	371	&ovs_reply_header);
	372	if (IS_ERR(reply))
	373	return PTR_ERR(reply);
	374
	375	ovs_lock();
	376
	377	dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
	378	if (!dp) {
	379	err = -ENODEV;
	380	goto exit_unlock;
	381	}
	382
383	/* Locate meter, copy stats. */
384	meter = lookup_meter(dp, meter_id);
385	if (!meter) {
386	err = -ENOENT;
387	goto exit_unlock;
388	}
389
390	spin_lock_bh(&meter->lock);
391	err = ovs_meter_cmd_reply_stats(reply, meter_id, meter);
392	spin_unlock_bh(&meter->lock);
393	if (err)
394	goto exit_unlock;
395
396	ovs_unlock();
397
398	genlmsg_end(reply, ovs_reply_header);
399	return genlmsg_reply(reply, info);
400
401	exit_unlock:
402	ovs_unlock();
403	nlmsg_free(reply);
404	return err;
405	}
406
407	static int ovs_meter_cmd_del(struct sk_buff skb, struct genl_info info)
408	{
409	struct nlattr **a = info->attrs;
410	u32 meter_id;
411	struct ovs_header *ovs_header = info->userhdr;
412	struct ovs_header *ovs_reply_header;
413	struct datapath *dp;
414	int err;
415	struct sk_buff *reply;
416	struct dp_meter *old_meter;
417
418	if (!a[OVS_METER_ATTR_ID])
419	return -EINVAL;
420	meter_id = nla_get_u32(a[OVS_METER_ATTR_ID]);
421
422	reply = ovs_meter_cmd_reply_start(info, OVS_METER_CMD_DEL,
423	&ovs_reply_header);
424	if (IS_ERR(reply))
425	return PTR_ERR(reply);
426
427	ovs_lock();
428
429	dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
430	if (!dp) {
431	err = -ENODEV;
432	goto exit_unlock;
433	}
434
435	old_meter = lookup_meter(dp, meter_id);
436	if (old_meter) {
437	spin_lock_bh(&old_meter->lock);
438	err = ovs_meter_cmd_reply_stats(reply, meter_id, old_meter);
439	WARN_ON(err);
440	spin_unlock_bh(&old_meter->lock);
441	detach_meter(old_meter);
442	}
443	ovs_unlock();
444	ovs_meter_free(old_meter);
445	genlmsg_end(reply, ovs_reply_header);
446	return genlmsg_reply(reply, info);
447
448	exit_unlock:
449	ovs_unlock();
450	nlmsg_free(reply);
451	return err;
452	}
453
454	/* Meter action execution.
455	*
456	* Return true 'meter_id' drop band is triggered. The 'skb' should be
457	* dropped by the caller'.
458	*/
459	bool ovs_meter_execute(struct datapath dp, struct sk_buff skb,
460	struct sw_flow_key *key, u32 meter_id)
461	{
462	struct dp_meter *meter;
463	struct dp_meter_band *band;
464	long long int now_ms = div_u64(ktime_get_ns(), 1000 * 1000);
465	long long int long_delta_ms;
466	u32 delta_ms;
467	u32 cost;
468	int i, band_exceeded_max = -1;
469	u32 band_exceeded_rate = 0;
470
471	meter = lookup_meter(dp, meter_id);
472	/* Do not drop the packet when there is no meter. */
473	if (!meter)
474	return false;
475
476	/* Lock the meter while using it. */
477	spin_lock(&meter->lock);
478
479	long_delta_ms = (now_ms - meter->used); /* ms */
480
481	/* Make sure delta_ms will not be too large, so that bucket will not
482	* wrap around below.
483	*/
484	delta_ms = (long_delta_ms > (long long int)meter->max_delta_t)
485	? meter->max_delta_t : (u32)long_delta_ms;
486
487	/* Update meter statistics.
488	*/
489	meter->used = now_ms;
490	meter->stats.n_packets += 1;
491	meter->stats.n_bytes += skb->len;
492
493	/* Bucket rate is either in kilobits per second, or in packets per
494	* second. We maintain the bucket in the units of either bits or
495	* 1/1000th of a packet, correspondingly.
496	* Then, when rate is multiplied with milliseconds, we get the
497	* bucket units:
498	* msec * kbps = bits, and
499	* msec * packets/sec = 1/1000 packets.
500	*
501	* 'cost' is the number of bucket units in this packet.
502	*/
503	cost = (meter->kbps) ? skb->len * 8 : 1000;
504
505	/* Update all bands and find the one hit with the highest rate. */
506	for (i = 0; i < meter->n_bands; ++i) {
507	long long int max_bucket_size;
508
509	band = &meter->bands[i];
65e64d20	510	max_bucket_size = (band->burst_size + band->rate) * 1000LL;
1cb57039 AZ	511
	512	band->bucket += delta_ms * band->rate;
	513	if (band->bucket > max_bucket_size)
	514	band->bucket = max_bucket_size;
	515
	516	if (band->bucket >= cost) {
	517	band->bucket -= cost;
	518	} else if (band->rate > band_exceeded_rate) {
	519	band_exceeded_rate = band->rate;
	520	band_exceeded_max = i;
	521	}
	522	}
	523
	524	if (band_exceeded_max >= 0) {
	525	/* Update band statistics. */
	526	band = &meter->bands[band_exceeded_max];
	527	band->stats.n_packets += 1;
	528	band->stats.n_bytes += skb->len;
	529
	530	/* Drop band triggered, let the caller drop the 'skb'. */
	531	if (band->type == OVS_METER_BAND_TYPE_DROP) {
	532	spin_unlock(&meter->lock);
	533	return true;
	534	}
	535	}
	536
	537	spin_unlock(&meter->lock);
	538	return false;
	539	}
	540
	541	static struct genl_ops dp_meter_genl_ops[] = {
	542	{ .cmd = OVS_METER_CMD_FEATURES,
f1e9590e YHW	543	#ifdef HAVE_GENL_VALIDATE_FLAGS
	544	.validate = GENL_DONT_VALIDATE_STRICT \| GENL_DONT_VALIDATE_DUMP,
	545	#endif
1cb57039	546	.flags = 0, /* OK for unprivileged users. */
2ef0f1c2	547	#ifdef HAVE_GENL_OPS_POLICY
1cb57039	548	.policy = meter_policy,
2ef0f1c2	549	#endif
1cb57039 AZ	550	.doit = ovs_meter_cmd_features
	551	},
	552	{ .cmd = OVS_METER_CMD_SET,
f1e9590e YHW	553	#ifdef HAVE_GENL_VALIDATE_FLAGS
	554	.validate = GENL_DONT_VALIDATE_STRICT \| GENL_DONT_VALIDATE_DUMP,
	555	#endif
1cb57039 AZ	556	.flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN
	557	* privilege.
	558	*/
2ef0f1c2	559	#ifdef HAVE_GENL_OPS_POLICY
1cb57039	560	.policy = meter_policy,
2ef0f1c2	561	#endif
1cb57039 AZ	562	.doit = ovs_meter_cmd_set,
	563	},
	564	{ .cmd = OVS_METER_CMD_GET,
f1e9590e YHW	565	#ifdef HAVE_GENL_VALIDATE_FLAGS
	566	.validate = GENL_DONT_VALIDATE_STRICT \| GENL_DONT_VALIDATE_DUMP,
	567	#endif
1cb57039	568	.flags = 0, /* OK for unprivileged users. */
2ef0f1c2	569	#ifdef HAVE_GENL_OPS_POLICY
1cb57039	570	.policy = meter_policy,
2ef0f1c2	571	#endif
1cb57039 AZ	572	.doit = ovs_meter_cmd_get,
	573	},
	574	{ .cmd = OVS_METER_CMD_DEL,
f1e9590e YHW	575	#ifdef HAVE_GENL_VALIDATE_FLAGS
	576	.validate = GENL_DONT_VALIDATE_STRICT \| GENL_DONT_VALIDATE_DUMP,
	577	#endif
1cb57039 AZ	578	.flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN
	579	* privilege.
	580	*/
2ef0f1c2	581	#ifdef HAVE_GENL_OPS_POLICY
1cb57039	582	.policy = meter_policy,
2ef0f1c2	583	#endif
1cb57039 AZ	584	.doit = ovs_meter_cmd_del
	585	},
	586	};
	587
	588	static const struct genl_multicast_group ovs_meter_multicast_group = {
	589	.name = OVS_METER_MCGROUP,
	590	};
	591
	592	struct genl_family dp_meter_genl_family __ro_after_init = {
	593	.hdrsize = sizeof(struct ovs_header),
	594	.name = OVS_METER_FAMILY,
	595	.version = OVS_METER_VERSION,
	596	.maxattr = OVS_METER_ATTR_MAX,
2ef0f1c2 JB	597	#ifndef HAVE_GENL_OPS_POLICY
	598	.policy = meter_policy,
	599	#endif
1cb57039 AZ	600	.netnsok = true,
	601	.parallel_ops = true,
	602	.ops = dp_meter_genl_ops,
	603	.n_ops = ARRAY_SIZE(dp_meter_genl_ops),
	604	.mcgrps = &ovs_meter_multicast_group,
	605	.n_mcgrps = 1,
	606	.module = THIS_MODULE,
	607	};
	608
	609	int ovs_meters_init(struct datapath *dp)
	610	{
	611	int i;
	612
	613	dp->meters = kmalloc_array(METER_HASH_BUCKETS,
	614	sizeof(struct hlist_head), GFP_KERNEL);
	615
	616	if (!dp->meters)
	617	return -ENOMEM;
	618
	619	for (i = 0; i < METER_HASH_BUCKETS; i++)
	620	INIT_HLIST_HEAD(&dp->meters[i]);
	621
	622	return 0;
	623	}
	624
	625	void ovs_meters_exit(struct datapath *dp)
	626	{
	627	int i;
	628
	629	for (i = 0; i < METER_HASH_BUCKETS; i++) {
	630	struct hlist_head *head = &dp->meters[i];
	631	struct dp_meter *meter;
	632	struct hlist_node *n;
	633
	634	hlist_for_each_entry_safe(meter, n, head, dp_hash_node)
	635	kfree(meter);
	636	}
	637
	638	kfree(dp->meters);
	639	}