linux/net/sched/sch_fq_pie.c

570 lines
14 KiB
C
Raw Normal View History

// SPDX-License-Identifier: GPL-2.0-only
/* Flow Queue PIE discipline
*
* Copyright (C) 2019 Mohit P. Tahiliani <tahiliani@nitk.edu.in>
* Copyright (C) 2019 Sachin D. Patil <sdp.sachin@gmail.com>
* Copyright (C) 2019 V. Saicharan <vsaicharan1998@gmail.com>
* Copyright (C) 2019 Mohit Bhasi <mohitbhasi1998@gmail.com>
* Copyright (C) 2019 Leslie Monis <lesliemonis@gmail.com>
* Copyright (C) 2019 Gautam Ramakrishnan <gautamramk@gmail.com>
*/
#include <linux/jhash.h>
#include <linux/sizes.h>
#include <linux/vmalloc.h>
#include <net/pkt_cls.h>
#include <net/pie.h>
/* Flow Queue PIE
*
* Principles:
* - Packets are classified on flows.
* - This is a Stochastic model (as we use a hash, several flows might
* be hashed to the same slot)
* - Each flow has a PIE managed queue.
* - Flows are linked onto two (Round Robin) lists,
* so that new flows have priority on old ones.
* - For a given flow, packets are not reordered.
* - Drops during enqueue only.
* - ECN capability is off by default.
* - ECN threshold (if ECN is enabled) is at 10% by default.
* - Uses timestamps to calculate queue delay by default.
*/
/**
* struct fq_pie_flow - contains data for each flow
* @vars: pie vars associated with the flow
* @deficit: number of remaining byte credits
* @backlog: size of data in the flow
* @qlen: number of packets in the flow
* @flowchain: flowchain for the flow
* @head: first packet in the flow
* @tail: last packet in the flow
*/
struct fq_pie_flow {
struct pie_vars vars;
s32 deficit;
u32 backlog;
u32 qlen;
struct list_head flowchain;
struct sk_buff *head;
struct sk_buff *tail;
};
struct fq_pie_sched_data {
struct tcf_proto __rcu *filter_list; /* optional external classifier */
struct tcf_block *block;
struct fq_pie_flow *flows;
struct Qdisc *sch;
struct list_head old_flows;
struct list_head new_flows;
struct pie_params p_params;
u32 ecn_prob;
u32 flows_cnt;
u32 quantum;
u32 memory_limit;
u32 new_flow_count;
u32 memory_usage;
u32 overmemory;
struct pie_stats stats;
struct timer_list adapt_timer;
};
static unsigned int fq_pie_hash(const struct fq_pie_sched_data *q,
struct sk_buff *skb)
{
return reciprocal_scale(skb_get_hash(skb), q->flows_cnt);
}
static unsigned int fq_pie_classify(struct sk_buff *skb, struct Qdisc *sch,
int *qerr)
{
struct fq_pie_sched_data *q = qdisc_priv(sch);
struct tcf_proto *filter;
struct tcf_result res;
int result;
if (TC_H_MAJ(skb->priority) == sch->handle &&
TC_H_MIN(skb->priority) > 0 &&
TC_H_MIN(skb->priority) <= q->flows_cnt)
return TC_H_MIN(skb->priority);
filter = rcu_dereference_bh(q->filter_list);
if (!filter)
return fq_pie_hash(q, skb) + 1;
*qerr = NET_XMIT_SUCCESS | __NET_XMIT_BYPASS;
result = tcf_classify(skb, NULL, filter, &res, false);
if (result >= 0) {
#ifdef CONFIG_NET_CLS_ACT
switch (result) {
case TC_ACT_STOLEN:
case TC_ACT_QUEUED:
case TC_ACT_TRAP:
*qerr = NET_XMIT_SUCCESS | __NET_XMIT_STOLEN;
fallthrough;
case TC_ACT_SHOT:
return 0;
}
#endif
if (TC_H_MIN(res.classid) <= q->flows_cnt)
return TC_H_MIN(res.classid);
}
return 0;
}
/* add skb to flow queue (tail add) */
static inline void flow_queue_add(struct fq_pie_flow *flow,
struct sk_buff *skb)
{
if (!flow->head)
flow->head = skb;
else
flow->tail->next = skb;
flow->tail = skb;
skb->next = NULL;
}
static int fq_pie_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch,
struct sk_buff **to_free)
{
struct fq_pie_sched_data *q = qdisc_priv(sch);
struct fq_pie_flow *sel_flow;
treewide: Remove uninitialized_var() usage Using uninitialized_var() is dangerous as it papers over real bugs[1] (or can in the future), and suppresses unrelated compiler warnings (e.g. "unused variable"). If the compiler thinks it is uninitialized, either simply initialize the variable or make compiler changes. In preparation for removing[2] the[3] macro[4], remove all remaining needless uses with the following script: git grep '\buninitialized_var\b' | cut -d: -f1 | sort -u | \ xargs perl -pi -e \ 's/\buninitialized_var\(([^\)]+)\)/\1/g; s:\s*/\* (GCC be quiet|to make compiler happy) \*/$::g;' drivers/video/fbdev/riva/riva_hw.c was manually tweaked to avoid pathological white-space. No outstanding warnings were found building allmodconfig with GCC 9.3.0 for x86_64, i386, arm64, arm, powerpc, powerpc64le, s390x, mips, sparc64, alpha, and m68k. [1] https://lore.kernel.org/lkml/20200603174714.192027-1-glider@google.com/ [2] https://lore.kernel.org/lkml/CA+55aFw+Vbj0i=1TGqCR5vQkCzWJ0QxK6CernOU6eedsudAixw@mail.gmail.com/ [3] https://lore.kernel.org/lkml/CA+55aFwgbgqhbp1fkxvRKEpzyR5J8n1vKT1VZdz9knmPuXhOeg@mail.gmail.com/ [4] https://lore.kernel.org/lkml/CA+55aFz2500WfbKXAx8s67wrm9=yVJu65TpLgN_ybYNv0VEOKA@mail.gmail.com/ Reviewed-by: Leon Romanovsky <leonro@mellanox.com> # drivers/infiniband and mlx4/mlx5 Acked-by: Jason Gunthorpe <jgg@mellanox.com> # IB Acked-by: Kalle Valo <kvalo@codeaurora.org> # wireless drivers Reviewed-by: Chao Yu <yuchao0@huawei.com> # erofs Signed-off-by: Kees Cook <keescook@chromium.org>
2020-06-03 20:09:38 +00:00
int ret;
u8 memory_limited = false;
u8 enqueue = false;
u32 pkt_len;
u32 idx;
/* Classifies packet into corresponding flow */
idx = fq_pie_classify(skb, sch, &ret);
net/sched: fq_pie: fix OOB access in the traffic path the following script: # tc qdisc add dev eth0 handle 0x1 root fq_pie flows 2 # tc qdisc add dev eth0 clsact # tc filter add dev eth0 egress matchall action skbedit priority 0x10002 # ping 192.0.2.2 -I eth0 -c2 -w1 -q produces the following splat: BUG: KASAN: slab-out-of-bounds in fq_pie_qdisc_enqueue+0x1314/0x19d0 [sch_fq_pie] Read of size 4 at addr ffff888171306924 by task ping/942 CPU: 3 PID: 942 Comm: ping Not tainted 5.12.0+ #441 Hardware name: Red Hat KVM, BIOS 1.11.1-4.module+el8.1.0+4066+0f1aadab 04/01/2014 Call Trace: dump_stack+0x92/0xc1 print_address_description.constprop.7+0x1a/0x150 kasan_report.cold.13+0x7f/0x111 fq_pie_qdisc_enqueue+0x1314/0x19d0 [sch_fq_pie] __dev_queue_xmit+0x1034/0x2b10 ip_finish_output2+0xc62/0x2120 __ip_finish_output+0x553/0xea0 ip_output+0x1ca/0x4d0 ip_send_skb+0x37/0xa0 raw_sendmsg+0x1c4b/0x2d00 sock_sendmsg+0xdb/0x110 __sys_sendto+0x1d7/0x2b0 __x64_sys_sendto+0xdd/0x1b0 do_syscall_64+0x3c/0x80 entry_SYSCALL_64_after_hwframe+0x44/0xae RIP: 0033:0x7fe69735c3eb Code: 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 f3 0f 1e fa 48 8d 05 75 42 2c 00 41 89 ca 8b 00 85 c0 75 14 b8 2c 00 00 00 0f 05 <48> 3d 00 f0 ff ff 77 75 c3 0f 1f 40 00 41 57 4d 89 c7 41 56 41 89 RSP: 002b:00007fff06d7fb38 EFLAGS: 00000246 ORIG_RAX: 000000000000002c RAX: ffffffffffffffda RBX: 000055e961413700 RCX: 00007fe69735c3eb RDX: 0000000000000040 RSI: 000055e961413700 RDI: 0000000000000003 RBP: 0000000000000040 R08: 000055e961410500 R09: 0000000000000010 R10: 0000000000000000 R11: 0000000000000246 R12: 00007fff06d81260 R13: 00007fff06d7fb40 R14: 00007fff06d7fc30 R15: 000055e96140f0a0 Allocated by task 917: kasan_save_stack+0x19/0x40 __kasan_kmalloc+0x7f/0xa0 __kmalloc_node+0x139/0x280 fq_pie_init+0x555/0x8e8 [sch_fq_pie] qdisc_create+0x407/0x11b0 tc_modify_qdisc+0x3c2/0x17e0 rtnetlink_rcv_msg+0x346/0x8e0 netlink_rcv_skb+0x120/0x380 netlink_unicast+0x439/0x630 netlink_sendmsg+0x719/0xbf0 sock_sendmsg+0xe2/0x110 ____sys_sendmsg+0x5ba/0x890 ___sys_sendmsg+0xe9/0x160 __sys_sendmsg+0xd3/0x170 do_syscall_64+0x3c/0x80 entry_SYSCALL_64_after_hwframe+0x44/0xae The buggy address belongs to the object at ffff888171306800 which belongs to the cache kmalloc-256 of size 256 The buggy address is located 36 bytes to the right of 256-byte region [ffff888171306800, ffff888171306900) The buggy address belongs to the page: page:00000000bcfb624e refcount:1 mapcount:0 mapping:0000000000000000 index:0x0 pfn:0x171306 head:00000000bcfb624e order:1 compound_mapcount:0 flags: 0x17ffffc0010200(slab|head|node=0|zone=2|lastcpupid=0x1fffff) raw: 0017ffffc0010200 dead000000000100 dead000000000122 ffff888100042b40 raw: 0000000000000000 0000000000100010 00000001ffffffff 0000000000000000 page dumped because: kasan: bad access detected Memory state around the buggy address: ffff888171306800: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ffff888171306880: 00 00 00 00 00 00 00 00 00 00 00 00 fc fc fc fc >ffff888171306900: fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc ^ ffff888171306980: fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc ffff888171306a00: fa fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb fix fq_pie traffic path to avoid selecting 'q->flows + q->flows_cnt' as a valid flow: it's an address beyond the allocated memory. Fixes: ec97ecf1ebe4 ("net: sched: add Flow Queue PIE packet scheduler") CC: stable@vger.kernel.org Signed-off-by: Davide Caratti <dcaratti@redhat.com> Signed-off-by: David S. Miller <davem@davemloft.net>
2021-05-22 13:15:13 +00:00
if (idx == 0) {
if (ret & __NET_XMIT_BYPASS)
qdisc_qstats_drop(sch);
__qdisc_drop(skb, to_free);
return ret;
}
idx--;
net/sched: fq_pie: fix OOB access in the traffic path the following script: # tc qdisc add dev eth0 handle 0x1 root fq_pie flows 2 # tc qdisc add dev eth0 clsact # tc filter add dev eth0 egress matchall action skbedit priority 0x10002 # ping 192.0.2.2 -I eth0 -c2 -w1 -q produces the following splat: BUG: KASAN: slab-out-of-bounds in fq_pie_qdisc_enqueue+0x1314/0x19d0 [sch_fq_pie] Read of size 4 at addr ffff888171306924 by task ping/942 CPU: 3 PID: 942 Comm: ping Not tainted 5.12.0+ #441 Hardware name: Red Hat KVM, BIOS 1.11.1-4.module+el8.1.0+4066+0f1aadab 04/01/2014 Call Trace: dump_stack+0x92/0xc1 print_address_description.constprop.7+0x1a/0x150 kasan_report.cold.13+0x7f/0x111 fq_pie_qdisc_enqueue+0x1314/0x19d0 [sch_fq_pie] __dev_queue_xmit+0x1034/0x2b10 ip_finish_output2+0xc62/0x2120 __ip_finish_output+0x553/0xea0 ip_output+0x1ca/0x4d0 ip_send_skb+0x37/0xa0 raw_sendmsg+0x1c4b/0x2d00 sock_sendmsg+0xdb/0x110 __sys_sendto+0x1d7/0x2b0 __x64_sys_sendto+0xdd/0x1b0 do_syscall_64+0x3c/0x80 entry_SYSCALL_64_after_hwframe+0x44/0xae RIP: 0033:0x7fe69735c3eb Code: 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 f3 0f 1e fa 48 8d 05 75 42 2c 00 41 89 ca 8b 00 85 c0 75 14 b8 2c 00 00 00 0f 05 <48> 3d 00 f0 ff ff 77 75 c3 0f 1f 40 00 41 57 4d 89 c7 41 56 41 89 RSP: 002b:00007fff06d7fb38 EFLAGS: 00000246 ORIG_RAX: 000000000000002c RAX: ffffffffffffffda RBX: 000055e961413700 RCX: 00007fe69735c3eb RDX: 0000000000000040 RSI: 000055e961413700 RDI: 0000000000000003 RBP: 0000000000000040 R08: 000055e961410500 R09: 0000000000000010 R10: 0000000000000000 R11: 0000000000000246 R12: 00007fff06d81260 R13: 00007fff06d7fb40 R14: 00007fff06d7fc30 R15: 000055e96140f0a0 Allocated by task 917: kasan_save_stack+0x19/0x40 __kasan_kmalloc+0x7f/0xa0 __kmalloc_node+0x139/0x280 fq_pie_init+0x555/0x8e8 [sch_fq_pie] qdisc_create+0x407/0x11b0 tc_modify_qdisc+0x3c2/0x17e0 rtnetlink_rcv_msg+0x346/0x8e0 netlink_rcv_skb+0x120/0x380 netlink_unicast+0x439/0x630 netlink_sendmsg+0x719/0xbf0 sock_sendmsg+0xe2/0x110 ____sys_sendmsg+0x5ba/0x890 ___sys_sendmsg+0xe9/0x160 __sys_sendmsg+0xd3/0x170 do_syscall_64+0x3c/0x80 entry_SYSCALL_64_after_hwframe+0x44/0xae The buggy address belongs to the object at ffff888171306800 which belongs to the cache kmalloc-256 of size 256 The buggy address is located 36 bytes to the right of 256-byte region [ffff888171306800, ffff888171306900) The buggy address belongs to the page: page:00000000bcfb624e refcount:1 mapcount:0 mapping:0000000000000000 index:0x0 pfn:0x171306 head:00000000bcfb624e order:1 compound_mapcount:0 flags: 0x17ffffc0010200(slab|head|node=0|zone=2|lastcpupid=0x1fffff) raw: 0017ffffc0010200 dead000000000100 dead000000000122 ffff888100042b40 raw: 0000000000000000 0000000000100010 00000001ffffffff 0000000000000000 page dumped because: kasan: bad access detected Memory state around the buggy address: ffff888171306800: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ffff888171306880: 00 00 00 00 00 00 00 00 00 00 00 00 fc fc fc fc >ffff888171306900: fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc ^ ffff888171306980: fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc ffff888171306a00: fa fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb fix fq_pie traffic path to avoid selecting 'q->flows + q->flows_cnt' as a valid flow: it's an address beyond the allocated memory. Fixes: ec97ecf1ebe4 ("net: sched: add Flow Queue PIE packet scheduler") CC: stable@vger.kernel.org Signed-off-by: Davide Caratti <dcaratti@redhat.com> Signed-off-by: David S. Miller <davem@davemloft.net>
2021-05-22 13:15:13 +00:00
sel_flow = &q->flows[idx];
/* Checks whether adding a new packet would exceed memory limit */
get_pie_cb(skb)->mem_usage = skb->truesize;
memory_limited = q->memory_usage > q->memory_limit + skb->truesize;
/* Checks if the qdisc is full */
if (unlikely(qdisc_qlen(sch) >= sch->limit)) {
q->stats.overlimit++;
goto out;
} else if (unlikely(memory_limited)) {
q->overmemory++;
}
if (!pie_drop_early(sch, &q->p_params, &sel_flow->vars,
sel_flow->backlog, skb->len)) {
enqueue = true;
} else if (q->p_params.ecn &&
sel_flow->vars.prob <= (MAX_PROB / 100) * q->ecn_prob &&
INET_ECN_set_ce(skb)) {
/* If packet is ecn capable, mark it if drop probability
* is lower than the parameter ecn_prob, else drop it.
*/
q->stats.ecn_mark++;
enqueue = true;
}
if (enqueue) {
/* Set enqueue time only when dq_rate_estimator is disabled. */
if (!q->p_params.dq_rate_estimator)
pie_set_enqueue_time(skb);
pkt_len = qdisc_pkt_len(skb);
q->stats.packets_in++;
q->memory_usage += skb->truesize;
sch->qstats.backlog += pkt_len;
sch->q.qlen++;
flow_queue_add(sel_flow, skb);
if (list_empty(&sel_flow->flowchain)) {
list_add_tail(&sel_flow->flowchain, &q->new_flows);
q->new_flow_count++;
sel_flow->deficit = q->quantum;
sel_flow->qlen = 0;
sel_flow->backlog = 0;
}
sel_flow->qlen++;
sel_flow->backlog += pkt_len;
return NET_XMIT_SUCCESS;
}
out:
q->stats.dropped++;
sel_flow->vars.accu_prob = 0;
__qdisc_drop(skb, to_free);
qdisc_qstats_drop(sch);
return NET_XMIT_CN;
}
static const struct nla_policy fq_pie_policy[TCA_FQ_PIE_MAX + 1] = {
[TCA_FQ_PIE_LIMIT] = {.type = NLA_U32},
[TCA_FQ_PIE_FLOWS] = {.type = NLA_U32},
[TCA_FQ_PIE_TARGET] = {.type = NLA_U32},
[TCA_FQ_PIE_TUPDATE] = {.type = NLA_U32},
[TCA_FQ_PIE_ALPHA] = {.type = NLA_U32},
[TCA_FQ_PIE_BETA] = {.type = NLA_U32},
[TCA_FQ_PIE_QUANTUM] = {.type = NLA_U32},
[TCA_FQ_PIE_MEMORY_LIMIT] = {.type = NLA_U32},
[TCA_FQ_PIE_ECN_PROB] = {.type = NLA_U32},
[TCA_FQ_PIE_ECN] = {.type = NLA_U32},
[TCA_FQ_PIE_BYTEMODE] = {.type = NLA_U32},
[TCA_FQ_PIE_DQ_RATE_ESTIMATOR] = {.type = NLA_U32},
};
static inline struct sk_buff *dequeue_head(struct fq_pie_flow *flow)
{
struct sk_buff *skb = flow->head;
flow->head = skb->next;
skb->next = NULL;
return skb;
}
static struct sk_buff *fq_pie_qdisc_dequeue(struct Qdisc *sch)
{
struct fq_pie_sched_data *q = qdisc_priv(sch);
struct sk_buff *skb = NULL;
struct fq_pie_flow *flow;
struct list_head *head;
u32 pkt_len;
begin:
head = &q->new_flows;
if (list_empty(head)) {
head = &q->old_flows;
if (list_empty(head))
return NULL;
}
flow = list_first_entry(head, struct fq_pie_flow, flowchain);
/* Flow has exhausted all its credits */
if (flow->deficit <= 0) {
flow->deficit += q->quantum;
list_move_tail(&flow->flowchain, &q->old_flows);
goto begin;
}
if (flow->head) {
skb = dequeue_head(flow);
pkt_len = qdisc_pkt_len(skb);
sch->qstats.backlog -= pkt_len;
sch->q.qlen--;
qdisc_bstats_update(sch, skb);
}
if (!skb) {
/* force a pass through old_flows to prevent starvation */
if (head == &q->new_flows && !list_empty(&q->old_flows))
list_move_tail(&flow->flowchain, &q->old_flows);
else
list_del_init(&flow->flowchain);
goto begin;
}
flow->qlen--;
flow->deficit -= pkt_len;
flow->backlog -= pkt_len;
q->memory_usage -= get_pie_cb(skb)->mem_usage;
pie_process_dequeue(skb, &q->p_params, &flow->vars, flow->backlog);
return skb;
}
static int fq_pie_change(struct Qdisc *sch, struct nlattr *opt,
struct netlink_ext_ack *extack)
{
struct fq_pie_sched_data *q = qdisc_priv(sch);
struct nlattr *tb[TCA_FQ_PIE_MAX + 1];
unsigned int len_dropped = 0;
unsigned int num_dropped = 0;
int err;
if (!opt)
return -EINVAL;
err = nla_parse_nested(tb, TCA_FQ_PIE_MAX, opt, fq_pie_policy, extack);
if (err < 0)
return err;
sch_tree_lock(sch);
if (tb[TCA_FQ_PIE_LIMIT]) {
u32 limit = nla_get_u32(tb[TCA_FQ_PIE_LIMIT]);
q->p_params.limit = limit;
sch->limit = limit;
}
if (tb[TCA_FQ_PIE_FLOWS]) {
if (q->flows) {
NL_SET_ERR_MSG_MOD(extack,
"Number of flows cannot be changed");
goto flow_error;
}
q->flows_cnt = nla_get_u32(tb[TCA_FQ_PIE_FLOWS]);
if (!q->flows_cnt || q->flows_cnt > 65536) {
NL_SET_ERR_MSG_MOD(extack,
"Number of flows must range in [1..65536]");
goto flow_error;
}
}
/* convert from microseconds to pschedtime */
if (tb[TCA_FQ_PIE_TARGET]) {
/* target is in us */
u32 target = nla_get_u32(tb[TCA_FQ_PIE_TARGET]);
/* convert to pschedtime */
q->p_params.target =
PSCHED_NS2TICKS((u64)target * NSEC_PER_USEC);
}
/* tupdate is in jiffies */
if (tb[TCA_FQ_PIE_TUPDATE])
q->p_params.tupdate =
usecs_to_jiffies(nla_get_u32(tb[TCA_FQ_PIE_TUPDATE]));
if (tb[TCA_FQ_PIE_ALPHA])
q->p_params.alpha = nla_get_u32(tb[TCA_FQ_PIE_ALPHA]);
if (tb[TCA_FQ_PIE_BETA])
q->p_params.beta = nla_get_u32(tb[TCA_FQ_PIE_BETA]);
if (tb[TCA_FQ_PIE_QUANTUM])
q->quantum = nla_get_u32(tb[TCA_FQ_PIE_QUANTUM]);
if (tb[TCA_FQ_PIE_MEMORY_LIMIT])
q->memory_limit = nla_get_u32(tb[TCA_FQ_PIE_MEMORY_LIMIT]);
if (tb[TCA_FQ_PIE_ECN_PROB])
q->ecn_prob = nla_get_u32(tb[TCA_FQ_PIE_ECN_PROB]);
if (tb[TCA_FQ_PIE_ECN])
q->p_params.ecn = nla_get_u32(tb[TCA_FQ_PIE_ECN]);
if (tb[TCA_FQ_PIE_BYTEMODE])
q->p_params.bytemode = nla_get_u32(tb[TCA_FQ_PIE_BYTEMODE]);
if (tb[TCA_FQ_PIE_DQ_RATE_ESTIMATOR])
q->p_params.dq_rate_estimator =
nla_get_u32(tb[TCA_FQ_PIE_DQ_RATE_ESTIMATOR]);
/* Drop excess packets if new limit is lower */
while (sch->q.qlen > sch->limit) {
struct sk_buff *skb = fq_pie_qdisc_dequeue(sch);
len_dropped += qdisc_pkt_len(skb);
num_dropped += 1;
rtnl_kfree_skbs(skb, skb);
}
qdisc_tree_reduce_backlog(sch, num_dropped, len_dropped);
sch_tree_unlock(sch);
return 0;
flow_error:
sch_tree_unlock(sch);
return -EINVAL;
}
static void fq_pie_timer(struct timer_list *t)
{
struct fq_pie_sched_data *q = from_timer(q, t, adapt_timer);
struct Qdisc *sch = q->sch;
spinlock_t *root_lock; /* to lock qdisc for probability calculations */
u32 idx;
root_lock = qdisc_lock(qdisc_root_sleeping(sch));
spin_lock(root_lock);
for (idx = 0; idx < q->flows_cnt; idx++)
pie_calculate_probability(&q->p_params, &q->flows[idx].vars,
q->flows[idx].backlog);
/* reset the timer to fire after 'tupdate' jiffies. */
if (q->p_params.tupdate)
mod_timer(&q->adapt_timer, jiffies + q->p_params.tupdate);
spin_unlock(root_lock);
}
static int fq_pie_init(struct Qdisc *sch, struct nlattr *opt,
struct netlink_ext_ack *extack)
{
struct fq_pie_sched_data *q = qdisc_priv(sch);
int err;
u32 idx;
pie_params_init(&q->p_params);
sch->limit = 10 * 1024;
q->p_params.limit = sch->limit;
q->quantum = psched_mtu(qdisc_dev(sch));
q->sch = sch;
q->ecn_prob = 10;
q->flows_cnt = 1024;
q->memory_limit = SZ_32M;
INIT_LIST_HEAD(&q->new_flows);
INIT_LIST_HEAD(&q->old_flows);
net/sched: fq_pie: initialize timer earlier in fq_pie_init() with the following tdc testcase: 83be: (qdisc, fq_pie) Create FQ-PIE with invalid number of flows as fq_pie_init() fails, fq_pie_destroy() is called to clean up. Since the timer is not yet initialized, it's possible to observe a splat like this: INFO: trying to register non-static key. the code is fine but needs lockdep annotation. turning off the locking correctness validator. CPU: 0 PID: 975 Comm: tc Not tainted 5.10.0-rc4+ #298 Hardware name: Red Hat KVM, BIOS 1.11.1-4.module+el8.1.0+4066+0f1aadab 04/01/2014 Call Trace: dump_stack+0x99/0xcb register_lock_class+0x12dd/0x1750 __lock_acquire+0xfe/0x3970 lock_acquire+0x1c8/0x7f0 del_timer_sync+0x49/0xd0 fq_pie_destroy+0x3f/0x80 [sch_fq_pie] qdisc_create+0x916/0x1160 tc_modify_qdisc+0x3c4/0x1630 rtnetlink_rcv_msg+0x346/0x8e0 netlink_unicast+0x439/0x630 netlink_sendmsg+0x719/0xbf0 sock_sendmsg+0xe2/0x110 ____sys_sendmsg+0x5ba/0x890 ___sys_sendmsg+0xe9/0x160 __sys_sendmsg+0xd3/0x170 do_syscall_64+0x33/0x40 entry_SYSCALL_64_after_hwframe+0x44/0xa9 [...] ODEBUG: assert_init not available (active state 0) object type: timer_list hint: 0x0 WARNING: CPU: 0 PID: 975 at lib/debugobjects.c:508 debug_print_object+0x162/0x210 [...] Call Trace: debug_object_assert_init+0x268/0x380 try_to_del_timer_sync+0x6a/0x100 del_timer_sync+0x9e/0xd0 fq_pie_destroy+0x3f/0x80 [sch_fq_pie] qdisc_create+0x916/0x1160 tc_modify_qdisc+0x3c4/0x1630 rtnetlink_rcv_msg+0x346/0x8e0 netlink_rcv_skb+0x120/0x380 netlink_unicast+0x439/0x630 netlink_sendmsg+0x719/0xbf0 sock_sendmsg+0xe2/0x110 ____sys_sendmsg+0x5ba/0x890 ___sys_sendmsg+0xe9/0x160 __sys_sendmsg+0xd3/0x170 do_syscall_64+0x33/0x40 entry_SYSCALL_64_after_hwframe+0x44/0xa9 fix it moving timer_setup() before any failure, like it was done on 'red' with former commit 608b4adab178 ("net_sched: initialize timer earlier in red_init()"). Fixes: ec97ecf1ebe4 ("net: sched: add Flow Queue PIE packet scheduler") Signed-off-by: Davide Caratti <dcaratti@redhat.com> Reviewed-by: Cong Wang <cong.wang@bytedance.com> Link: https://lore.kernel.org/r/2e78e01c504c633ebdff18d041833cf2e079a3a4.1607020450.git.dcaratti@redhat.com Signed-off-by: Jakub Kicinski <kuba@kernel.org>
2020-12-03 18:40:47 +00:00
timer_setup(&q->adapt_timer, fq_pie_timer, 0);
if (opt) {
err = fq_pie_change(sch, opt, extack);
if (err)
return err;
}
err = tcf_block_get(&q->block, &q->filter_list, sch, extack);
if (err)
goto init_failure;
q->flows = kvcalloc(q->flows_cnt, sizeof(struct fq_pie_flow),
GFP_KERNEL);
if (!q->flows) {
err = -ENOMEM;
goto init_failure;
}
for (idx = 0; idx < q->flows_cnt; idx++) {
struct fq_pie_flow *flow = q->flows + idx;
INIT_LIST_HEAD(&flow->flowchain);
pie_vars_init(&flow->vars);
}
mod_timer(&q->adapt_timer, jiffies + HZ / 2);
return 0;
init_failure:
q->flows_cnt = 0;
return err;
}
static int fq_pie_dump(struct Qdisc *sch, struct sk_buff *skb)
{
struct fq_pie_sched_data *q = qdisc_priv(sch);
struct nlattr *opts;
opts = nla_nest_start(skb, TCA_OPTIONS);
if (!opts)
return -EMSGSIZE;
/* convert target from pschedtime to us */
if (nla_put_u32(skb, TCA_FQ_PIE_LIMIT, sch->limit) ||
nla_put_u32(skb, TCA_FQ_PIE_FLOWS, q->flows_cnt) ||
nla_put_u32(skb, TCA_FQ_PIE_TARGET,
((u32)PSCHED_TICKS2NS(q->p_params.target)) /
NSEC_PER_USEC) ||
nla_put_u32(skb, TCA_FQ_PIE_TUPDATE,
jiffies_to_usecs(q->p_params.tupdate)) ||
nla_put_u32(skb, TCA_FQ_PIE_ALPHA, q->p_params.alpha) ||
nla_put_u32(skb, TCA_FQ_PIE_BETA, q->p_params.beta) ||
nla_put_u32(skb, TCA_FQ_PIE_QUANTUM, q->quantum) ||
nla_put_u32(skb, TCA_FQ_PIE_MEMORY_LIMIT, q->memory_limit) ||
nla_put_u32(skb, TCA_FQ_PIE_ECN_PROB, q->ecn_prob) ||
nla_put_u32(skb, TCA_FQ_PIE_ECN, q->p_params.ecn) ||
nla_put_u32(skb, TCA_FQ_PIE_BYTEMODE, q->p_params.bytemode) ||
nla_put_u32(skb, TCA_FQ_PIE_DQ_RATE_ESTIMATOR,
q->p_params.dq_rate_estimator))
goto nla_put_failure;
return nla_nest_end(skb, opts);
nla_put_failure:
nla_nest_cancel(skb, opts);
return -EMSGSIZE;
}
static int fq_pie_dump_stats(struct Qdisc *sch, struct gnet_dump *d)
{
struct fq_pie_sched_data *q = qdisc_priv(sch);
struct tc_fq_pie_xstats st = {
.packets_in = q->stats.packets_in,
.overlimit = q->stats.overlimit,
.overmemory = q->overmemory,
.dropped = q->stats.dropped,
.ecn_mark = q->stats.ecn_mark,
.new_flow_count = q->new_flow_count,
.memory_usage = q->memory_usage,
};
struct list_head *pos;
sch_tree_lock(sch);
list_for_each(pos, &q->new_flows)
st.new_flows_len++;
list_for_each(pos, &q->old_flows)
st.old_flows_len++;
sch_tree_unlock(sch);
return gnet_stats_copy_app(d, &st, sizeof(st));
}
static void fq_pie_reset(struct Qdisc *sch)
{
struct fq_pie_sched_data *q = qdisc_priv(sch);
u32 idx;
INIT_LIST_HEAD(&q->new_flows);
INIT_LIST_HEAD(&q->old_flows);
for (idx = 0; idx < q->flows_cnt; idx++) {
struct fq_pie_flow *flow = q->flows + idx;
/* Removes all packets from flow */
rtnl_kfree_skbs(flow->head, flow->tail);
flow->head = NULL;
INIT_LIST_HEAD(&flow->flowchain);
pie_vars_init(&flow->vars);
}
sch->q.qlen = 0;
sch->qstats.backlog = 0;
}
static void fq_pie_destroy(struct Qdisc *sch)
{
struct fq_pie_sched_data *q = qdisc_priv(sch);
tcf_block_put(q->block);
net/sched: fq_pie: prevent dismantle issue For some reason, fq_pie_destroy() did not copy working code from pie_destroy() and other qdiscs, thus causing elusive bug. Before calling del_timer_sync(&q->adapt_timer), we need to ensure timer will not rearm itself. rcu: INFO: rcu_preempt self-detected stall on CPU rcu: 0-....: (4416 ticks this GP) idle=60d/1/0x4000000000000000 softirq=10433/10434 fqs=2579 (t=10501 jiffies g=13085 q=3989) NMI backtrace for cpu 0 CPU: 0 PID: 13 Comm: ksoftirqd/0 Not tainted 5.16.0-rc4-syzkaller #0 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 Call Trace: <IRQ> __dump_stack lib/dump_stack.c:88 [inline] dump_stack_lvl+0xcd/0x134 lib/dump_stack.c:106 nmi_cpu_backtrace.cold+0x47/0x144 lib/nmi_backtrace.c:111 nmi_trigger_cpumask_backtrace+0x1b3/0x230 lib/nmi_backtrace.c:62 trigger_single_cpu_backtrace include/linux/nmi.h:164 [inline] rcu_dump_cpu_stacks+0x25e/0x3f0 kernel/rcu/tree_stall.h:343 print_cpu_stall kernel/rcu/tree_stall.h:627 [inline] check_cpu_stall kernel/rcu/tree_stall.h:711 [inline] rcu_pending kernel/rcu/tree.c:3878 [inline] rcu_sched_clock_irq.cold+0x9d/0x746 kernel/rcu/tree.c:2597 update_process_times+0x16d/0x200 kernel/time/timer.c:1785 tick_sched_handle+0x9b/0x180 kernel/time/tick-sched.c:226 tick_sched_timer+0x1b0/0x2d0 kernel/time/tick-sched.c:1428 __run_hrtimer kernel/time/hrtimer.c:1685 [inline] __hrtimer_run_queues+0x1c0/0xe50 kernel/time/hrtimer.c:1749 hrtimer_interrupt+0x31c/0x790 kernel/time/hrtimer.c:1811 local_apic_timer_interrupt arch/x86/kernel/apic/apic.c:1086 [inline] __sysvec_apic_timer_interrupt+0x146/0x530 arch/x86/kernel/apic/apic.c:1103 sysvec_apic_timer_interrupt+0x8e/0xc0 arch/x86/kernel/apic/apic.c:1097 </IRQ> <TASK> asm_sysvec_apic_timer_interrupt+0x12/0x20 arch/x86/include/asm/idtentry.h:638 RIP: 0010:write_comp_data kernel/kcov.c:221 [inline] RIP: 0010:__sanitizer_cov_trace_const_cmp1+0x1d/0x80 kernel/kcov.c:273 Code: 54 c8 20 48 89 10 c3 66 0f 1f 44 00 00 53 41 89 fb 41 89 f1 bf 03 00 00 00 65 48 8b 0c 25 40 70 02 00 48 89 ce 4c 8b 54 24 08 <e8> 4e f7 ff ff 84 c0 74 51 48 8b 81 88 15 00 00 44 8b 81 84 15 00 RSP: 0018:ffffc90000d27b28 EFLAGS: 00000246 RAX: 0000000000000000 RBX: ffff888064bf1bf0 RCX: ffff888011928000 RDX: ffff888011928000 RSI: ffff888011928000 RDI: 0000000000000003 RBP: ffff888064bf1c28 R08: 0000000000000000 R09: 0000000000000000 R10: ffffffff875d8295 R11: 0000000000000000 R12: 0000000000000000 R13: ffff8880783dd300 R14: 0000000000000000 R15: 0000000000000000 pie_calculate_probability+0x405/0x7c0 net/sched/sch_pie.c:418 fq_pie_timer+0x170/0x2a0 net/sched/sch_fq_pie.c:383 call_timer_fn+0x1a5/0x6b0 kernel/time/timer.c:1421 expire_timers kernel/time/timer.c:1466 [inline] __run_timers.part.0+0x675/0xa20 kernel/time/timer.c:1734 __run_timers kernel/time/timer.c:1715 [inline] run_timer_softirq+0xb3/0x1d0 kernel/time/timer.c:1747 __do_softirq+0x29b/0x9c2 kernel/softirq.c:558 run_ksoftirqd kernel/softirq.c:921 [inline] run_ksoftirqd+0x2d/0x60 kernel/softirq.c:913 smpboot_thread_fn+0x645/0x9c0 kernel/smpboot.c:164 kthread+0x405/0x4f0 kernel/kthread.c:327 ret_from_fork+0x1f/0x30 arch/x86/entry/entry_64.S:295 </TASK> Fixes: ec97ecf1ebe4 ("net: sched: add Flow Queue PIE packet scheduler") Signed-off-by: Eric Dumazet <edumazet@google.com> Reported-by: syzbot <syzkaller@googlegroups.com> Cc: Mohit P. Tahiliani <tahiliani@nitk.edu.in> Cc: Sachin D. Patil <sdp.sachin@gmail.com> Cc: V. Saicharan <vsaicharan1998@gmail.com> Cc: Mohit Bhasi <mohitbhasi1998@gmail.com> Cc: Leslie Monis <lesliemonis@gmail.com> Cc: Gautam Ramakrishnan <gautamramk@gmail.com> Link: https://lore.kernel.org/r/20211209084937.3500020-1-eric.dumazet@gmail.com Signed-off-by: Jakub Kicinski <kuba@kernel.org>
2021-12-09 08:49:37 +00:00
q->p_params.tupdate = 0;
del_timer_sync(&q->adapt_timer);
kvfree(q->flows);
}
static struct Qdisc_ops fq_pie_qdisc_ops __read_mostly = {
.id = "fq_pie",
.priv_size = sizeof(struct fq_pie_sched_data),
.enqueue = fq_pie_qdisc_enqueue,
.dequeue = fq_pie_qdisc_dequeue,
.peek = qdisc_peek_dequeued,
.init = fq_pie_init,
.destroy = fq_pie_destroy,
.reset = fq_pie_reset,
.change = fq_pie_change,
.dump = fq_pie_dump,
.dump_stats = fq_pie_dump_stats,
.owner = THIS_MODULE,
};
static int __init fq_pie_module_init(void)
{
return register_qdisc(&fq_pie_qdisc_ops);
}
static void __exit fq_pie_module_exit(void)
{
unregister_qdisc(&fq_pie_qdisc_ops);
}
module_init(fq_pie_module_init);
module_exit(fq_pie_module_exit);
MODULE_DESCRIPTION("Flow Queue Proportional Integral controller Enhanced (FQ-PIE)");
MODULE_AUTHOR("Mohit P. Tahiliani");
MODULE_LICENSE("GPL");