enum qdisc_state_t {
__QDISC_STATE_SCHED,
__QDISC_STATE_DEACTIVATED,
- __QDISC_STATE_THROTTLED,
-};
-
-/*
- * following bits are only changed while qdisc lock is held
- */
-enum qdisc___state_t {
- __QDISC___STATE_RUNNING = 1,
};
struct qdisc_size_table {
};
struct Qdisc {
- int (*enqueue)(struct sk_buff *skb, struct Qdisc *dev);
- struct sk_buff * (*dequeue)(struct Qdisc *dev);
+ int (*enqueue)(struct sk_buff *skb,
+ struct Qdisc *sch,
+ struct sk_buff **to_free);
+ struct sk_buff * (*dequeue)(struct Qdisc *sch);
unsigned int flags;
#define TCQ_F_BUILTIN 1
#define TCQ_F_INGRESS 2
u32 limit;
const struct Qdisc_ops *ops;
struct qdisc_size_table __rcu *stab;
- struct list_head list;
+ struct hlist_node hash;
u32 handle;
u32 parent;
- int (*reshape_fail)(struct sk_buff *skb,
- struct Qdisc *q);
-
void *u32_node;
- /* This field is deprecated, but it is still used by CBQ
- * and it will live until better solution will be invented.
- */
- struct Qdisc *__parent;
struct netdev_queue *dev_queue;
struct gnet_stats_rate_est64 rate_est;
struct gnet_stats_basic_cpu __percpu *cpu_bstats;
struct gnet_stats_queue __percpu *cpu_qstats;
- struct Qdisc *next_sched;
- struct sk_buff *gso_skb;
/*
* For performance sake on SMP, we put highly modified fields at the end
*/
- unsigned long state;
+ struct sk_buff *gso_skb ____cacheline_aligned_in_smp;
struct sk_buff_head q;
struct gnet_stats_basic_packed bstats;
- unsigned int __state;
+ seqcount_t running;
struct gnet_stats_queue qstats;
+ unsigned long state;
+ struct Qdisc *next_sched;
+ struct sk_buff *skb_bad_txq;
struct rcu_head rcu_head;
int padded;
atomic_t refcnt;
static inline bool qdisc_is_running(const struct Qdisc *qdisc)
{
- return (qdisc->__state & __QDISC___STATE_RUNNING) ? true : false;
+ return (raw_read_seqcount(&qdisc->running) & 1) ? true : false;
}
static inline bool qdisc_run_begin(struct Qdisc *qdisc)
{
if (qdisc_is_running(qdisc))
return false;
- qdisc->__state |= __QDISC___STATE_RUNNING;
+ /* Variant of write_seqcount_begin() telling lockdep a trylock
+ * was attempted.
+ */
+ raw_write_seqcount_begin(&qdisc->running);
+ seqcount_acquire(&qdisc->running.dep_map, 0, 1, _RET_IP_);
return true;
}
static inline void qdisc_run_end(struct Qdisc *qdisc)
{
- qdisc->__state &= ~__QDISC___STATE_RUNNING;
+ write_seqcount_end(&qdisc->running);
}
static inline bool qdisc_may_bulk(const struct Qdisc *qdisc)
#endif
}
-static inline bool qdisc_is_throttled(const struct Qdisc *qdisc)
-{
- return test_bit(__QDISC_STATE_THROTTLED, &qdisc->state) ? true : false;
-}
-
-static inline void qdisc_throttled(struct Qdisc *qdisc)
-{
- set_bit(__QDISC_STATE_THROTTLED, &qdisc->state);
-}
-
-static inline void qdisc_unthrottled(struct Qdisc *qdisc)
-{
- clear_bit(__QDISC_STATE_THROTTLED, &qdisc->state);
-}
-
struct Qdisc_class_ops {
/* Child qdisc manipulation */
struct netdev_queue * (*select_queue)(struct Qdisc *, struct tcmsg *);
char id[IFNAMSIZ];
int priv_size;
- int (*enqueue)(struct sk_buff *, struct Qdisc *);
+ int (*enqueue)(struct sk_buff *skb,
+ struct Qdisc *sch,
+ struct sk_buff **to_free);
struct sk_buff * (*dequeue)(struct Qdisc *);
struct sk_buff * (*peek)(struct Qdisc *);
- unsigned int (*drop)(struct Qdisc *);
int (*init)(struct Qdisc *, struct nlattr *arg);
void (*reset)(struct Qdisc *);
return qdisc_lock(root);
}
+static inline seqcount_t *qdisc_root_sleeping_running(const struct Qdisc *qdisc)
+{
+ struct Qdisc *root = qdisc_root_sleeping(qdisc);
+
+ ASSERT_RTNL();
+ return &root->running;
+}
+
static inline struct net_device *qdisc_dev(const struct Qdisc *qdisc)
{
return qdisc->dev_queue->dev;
#endif
}
-static inline int qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch)
+static inline int qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch,
+ struct sk_buff **to_free)
{
qdisc_calculate_pkt_len(skb, sch);
- return sch->enqueue(skb, sch);
+ return sch->enqueue(skb, sch, to_free);
}
static inline bool qdisc_is_percpu_stats(const struct Qdisc *q)
return __qdisc_dequeue_head(sch, &sch->q);
}
+/* Instead of calling kfree_skb() while root qdisc lock is held,
+ * queue the skb for future freeing at end of __dev_xmit_skb()
+ */
+static inline void __qdisc_drop(struct sk_buff *skb, struct sk_buff **to_free)
+{
+ skb->next = *to_free;
+ *to_free = skb;
+}
+
static inline unsigned int __qdisc_queue_drop_head(struct Qdisc *sch,
- struct sk_buff_head *list)
+ struct sk_buff_head *list,
+ struct sk_buff **to_free)
{
struct sk_buff *skb = __skb_dequeue(list);
if (likely(skb != NULL)) {
unsigned int len = qdisc_pkt_len(skb);
+
qdisc_qstats_backlog_dec(sch, skb);
- kfree_skb(skb);
+ __qdisc_drop(skb, to_free);
return len;
}
return 0;
}
-static inline unsigned int qdisc_queue_drop_head(struct Qdisc *sch)
+static inline unsigned int qdisc_queue_drop_head(struct Qdisc *sch,
+ struct sk_buff **to_free)
{
- return __qdisc_queue_drop_head(sch, &sch->q);
-}
-
-static inline struct sk_buff *__qdisc_dequeue_tail(struct Qdisc *sch,
- struct sk_buff_head *list)
-{
- struct sk_buff *skb = __skb_dequeue_tail(list);
-
- if (likely(skb != NULL))
- qdisc_qstats_backlog_dec(sch, skb);
-
- return skb;
-}
-
-static inline struct sk_buff *qdisc_dequeue_tail(struct Qdisc *sch)
-{
- return __qdisc_dequeue_tail(sch, &sch->q);
+ return __qdisc_queue_drop_head(sch, &sch->q, to_free);
}
static inline struct sk_buff *qdisc_peek_head(struct Qdisc *sch)
return skb;
}
-static inline void __qdisc_reset_queue(struct Qdisc *sch,
- struct sk_buff_head *list)
+static inline void __qdisc_reset_queue(struct sk_buff_head *list)
{
/*
* We do not know the backlog in bytes of this list, it
* is up to the caller to correct it
*/
- __skb_queue_purge(list);
+ if (!skb_queue_empty(list)) {
+ rtnl_kfree_skbs(list->next, list->prev);
+ __skb_queue_head_init(list);
+ }
}
static inline void qdisc_reset_queue(struct Qdisc *sch)
{
- __qdisc_reset_queue(sch, &sch->q);
+ __qdisc_reset_queue(&sch->q);
sch->qstats.backlog = 0;
}
return old;
}
-static inline unsigned int __qdisc_queue_drop(struct Qdisc *sch,
- struct sk_buff_head *list)
+static inline void rtnl_qdisc_drop(struct sk_buff *skb, struct Qdisc *sch)
{
- struct sk_buff *skb = __qdisc_dequeue_tail(sch, list);
-
- if (likely(skb != NULL)) {
- unsigned int len = qdisc_pkt_len(skb);
- kfree_skb(skb);
- return len;
- }
-
- return 0;
-}
-
-static inline unsigned int qdisc_queue_drop(struct Qdisc *sch)
-{
- return __qdisc_queue_drop(sch, &sch->q);
-}
-
-static inline int qdisc_drop(struct sk_buff *skb, struct Qdisc *sch)
-{
- kfree_skb(skb);
+ rtnl_kfree_skbs(skb, skb);
qdisc_qstats_drop(sch);
-
- return NET_XMIT_DROP;
}
-static inline int qdisc_reshape_fail(struct sk_buff *skb, struct Qdisc *sch)
+
+static inline int qdisc_drop(struct sk_buff *skb, struct Qdisc *sch,
+ struct sk_buff **to_free)
{
+ __qdisc_drop(skb, to_free);
qdisc_qstats_drop(sch);
-#ifdef CONFIG_NET_CLS_ACT
- if (sch->reshape_fail == NULL || sch->reshape_fail(skb, sch))
- goto drop;
-
- return NET_XMIT_SUCCESS;
-
-drop:
-#endif
- kfree_skb(skb);
return NET_XMIT_DROP;
}
projects / mirror_ubuntu-artful-kernel.git / blobdiff