From c204fef97ee62ca3310f43d12bbda4eb10266c7c Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Mon, 29 Apr 2024 13:40:25 +0000 Subject: [PATCH] net: move sysctl_mem_pcpu_rsv to net_hotdata sysctl_mem_pcpu_rsv is used in TCP fast path, move it to net_hodata for better cache locality. Signed-off-by: Eric Dumazet Reviewed-by: David Ahern Link: https://lore.kernel.org/r/20240429134025.1233626-6-edumazet@google.com Signed-off-by: Jakub Kicinski --- include/net/hotdata.h | 1 + include/net/proto_memory.h | 6 +++--- net/core/hotdata.c | 5 +++-- net/core/sock.c | 1 - net/core/sysctl_net_core.c | 2 +- 5 files changed, 8 insertions(+), 7 deletions(-) diff --git a/include/net/hotdata.h b/include/net/hotdata.h index 290499f72e18..30e9570beb2a 100644 --- a/include/net/hotdata.h +++ b/include/net/hotdata.h @@ -40,6 +40,7 @@ struct net_hotdata { int dev_rx_weight; int sysctl_max_skb_frags; int sysctl_skb_defer_max; + int sysctl_mem_pcpu_rsv; }; #define inet_ehash_secret net_hotdata.tcp_protocol.secret diff --git a/include/net/proto_memory.h b/include/net/proto_memory.h index 41404d4bb6f0..a6ab2f4f5e28 100644 --- a/include/net/proto_memory.h +++ b/include/net/proto_memory.h @@ -3,10 +3,10 @@ #define _PROTO_MEMORY_H #include +#include /* 1 MB per cpu, in page units */ #define SK_MEMORY_PCPU_RESERVE (1 << (20 - PAGE_SHIFT)) -extern int sysctl_mem_pcpu_rsv; static inline bool sk_has_memory_pressure(const struct sock *sk) { @@ -65,7 +65,7 @@ sk_memory_allocated_add(const struct sock *sk, int val) val = this_cpu_add_return(*proto->per_cpu_fw_alloc, val); - if (unlikely(val >= READ_ONCE(sysctl_mem_pcpu_rsv))) + if (unlikely(val >= READ_ONCE(net_hotdata.sysctl_mem_pcpu_rsv))) proto_memory_pcpu_drain(proto); } @@ -76,7 +76,7 @@ sk_memory_allocated_sub(const struct sock *sk, int val) val = this_cpu_sub_return(*proto->per_cpu_fw_alloc, val); - if (unlikely(val <= -READ_ONCE(sysctl_mem_pcpu_rsv))) + if (unlikely(val <= -READ_ONCE(net_hotdata.sysctl_mem_pcpu_rsv))) proto_memory_pcpu_drain(proto); } diff --git a/net/core/hotdata.c b/net/core/hotdata.c index a359ff160d54..d0aaaaa556f2 100644 --- a/net/core/hotdata.c +++ b/net/core/hotdata.c @@ -1,9 +1,9 @@ // SPDX-License-Identifier: GPL-2.0-or-later -#include #include #include #include - +#include +#include struct net_hotdata net_hotdata __cacheline_aligned = { .offload_base = LIST_HEAD_INIT(net_hotdata.offload_base), @@ -20,5 +20,6 @@ struct net_hotdata net_hotdata __cacheline_aligned = { .dev_rx_weight = 64, .sysctl_max_skb_frags = MAX_SKB_FRAGS, .sysctl_skb_defer_max = 64, + .sysctl_mem_pcpu_rsv = SK_MEMORY_PCPU_RESERVE }; EXPORT_SYMBOL(net_hotdata); diff --git a/net/core/sock.c b/net/core/sock.c index e0692b752369..8d6e638b5426 100644 --- a/net/core/sock.c +++ b/net/core/sock.c @@ -284,7 +284,6 @@ __u32 sysctl_rmem_max __read_mostly = SK_RMEM_MAX; EXPORT_SYMBOL(sysctl_rmem_max); __u32 sysctl_wmem_default __read_mostly = SK_WMEM_MAX; __u32 sysctl_rmem_default __read_mostly = SK_RMEM_MAX; -int sysctl_mem_pcpu_rsv __read_mostly = SK_MEMORY_PCPU_RESERVE; int sysctl_tstamp_allow_data __read_mostly = 1; diff --git a/net/core/sysctl_net_core.c b/net/core/sysctl_net_core.c index a452a330d0ed..6da5995ac86a 100644 --- a/net/core/sysctl_net_core.c +++ b/net/core/sysctl_net_core.c @@ -416,7 +416,7 @@ static struct ctl_table net_core_table[] = { }, { .procname = "mem_pcpu_rsv", - .data = &sysctl_mem_pcpu_rsv, + .data = &net_hotdata.sysctl_mem_pcpu_rsv, .maxlen = sizeof(int), .mode = 0644, .proc_handler = proc_dointvec_minmax,