linux/net/ipv6/xfrm6_policy.c
Eric Dumazet fc66f95c68 net dst: use a percpu_counter to track entries
struct dst_ops tracks number of allocated dst in an atomic_t field,
subject to high cache line contention in stress workload.

Switch to a percpu_counter, to reduce number of time we need to dirty a
central location. Place it on a separate cache line to avoid dirtying
read only fields.

Stress test :

(Sending 160.000.000 UDP frames,
IP route cache disabled, dual E5540 @2.53GHz,
32bit kernel, FIB_TRIE, SLUB/NUMA)

Before:

real    0m51.179s
user    0m15.329s
sys     10m15.942s

After:

real	0m45.570s
user	0m15.525s
sys	9m56.669s

With a small reordering of struct neighbour fields, subject of a
following patch, (to separate refcnt from other read mostly fields)

real	0m41.841s
user	0m15.261s
sys	8m45.949s

Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2010-10-11 13:06:53 -07:00

347 lines
8.1 KiB
C

/*
* xfrm6_policy.c: based on xfrm4_policy.c
*
* Authors:
* Mitsuru KANDA @USAGI
* Kazunori MIYAZAWA @USAGI
* Kunihiro Ishiguro <kunihiro@ipinfusion.com>
* IPv6 support
* YOSHIFUJI Hideaki
* Split up af-specific portion
*
*/
#include <linux/err.h>
#include <linux/kernel.h>
#include <linux/netdevice.h>
#include <net/addrconf.h>
#include <net/dst.h>
#include <net/xfrm.h>
#include <net/ip.h>
#include <net/ipv6.h>
#include <net/ip6_route.h>
#if defined(CONFIG_IPV6_MIP6) || defined(CONFIG_IPV6_MIP6_MODULE)
#include <net/mip6.h>
#endif
static struct xfrm_policy_afinfo xfrm6_policy_afinfo;
static struct dst_entry *xfrm6_dst_lookup(struct net *net, int tos,
xfrm_address_t *saddr,
xfrm_address_t *daddr)
{
struct flowi fl = {};
struct dst_entry *dst;
int err;
memcpy(&fl.fl6_dst, daddr, sizeof(fl.fl6_dst));
if (saddr)
memcpy(&fl.fl6_src, saddr, sizeof(fl.fl6_src));
dst = ip6_route_output(net, NULL, &fl);
err = dst->error;
if (dst->error) {
dst_release(dst);
dst = ERR_PTR(err);
}
return dst;
}
static int xfrm6_get_saddr(struct net *net,
xfrm_address_t *saddr, xfrm_address_t *daddr)
{
struct dst_entry *dst;
struct net_device *dev;
dst = xfrm6_dst_lookup(net, 0, NULL, daddr);
if (IS_ERR(dst))
return -EHOSTUNREACH;
dev = ip6_dst_idev(dst)->dev;
ipv6_dev_get_saddr(dev_net(dev), dev,
(struct in6_addr *)&daddr->a6, 0,
(struct in6_addr *)&saddr->a6);
dst_release(dst);
return 0;
}
static int xfrm6_get_tos(struct flowi *fl)
{
return 0;
}
static int xfrm6_init_path(struct xfrm_dst *path, struct dst_entry *dst,
int nfheader_len)
{
if (dst->ops->family == AF_INET6) {
struct rt6_info *rt = (struct rt6_info*)dst;
if (rt->rt6i_node)
path->path_cookie = rt->rt6i_node->fn_sernum;
}
path->u.rt6.rt6i_nfheader_len = nfheader_len;
return 0;
}
static int xfrm6_fill_dst(struct xfrm_dst *xdst, struct net_device *dev,
struct flowi *fl)
{
struct rt6_info *rt = (struct rt6_info*)xdst->route;
xdst->u.dst.dev = dev;
dev_hold(dev);
xdst->u.rt6.rt6i_idev = in6_dev_get(dev);
if (!xdst->u.rt6.rt6i_idev)
return -ENODEV;
/* Sheit... I remember I did this right. Apparently,
* it was magically lost, so this code needs audit */
xdst->u.rt6.rt6i_flags = rt->rt6i_flags & (RTF_ANYCAST |
RTF_LOCAL);
xdst->u.rt6.rt6i_metric = rt->rt6i_metric;
xdst->u.rt6.rt6i_node = rt->rt6i_node;
if (rt->rt6i_node)
xdst->route_cookie = rt->rt6i_node->fn_sernum;
xdst->u.rt6.rt6i_gateway = rt->rt6i_gateway;
xdst->u.rt6.rt6i_dst = rt->rt6i_dst;
xdst->u.rt6.rt6i_src = rt->rt6i_src;
return 0;
}
static inline void
_decode_session6(struct sk_buff *skb, struct flowi *fl, int reverse)
{
int onlyproto = 0;
u16 offset = skb_network_header_len(skb);
struct ipv6hdr *hdr = ipv6_hdr(skb);
struct ipv6_opt_hdr *exthdr;
const unsigned char *nh = skb_network_header(skb);
u8 nexthdr = nh[IP6CB(skb)->nhoff];
memset(fl, 0, sizeof(struct flowi));
fl->mark = skb->mark;
ipv6_addr_copy(&fl->fl6_dst, reverse ? &hdr->saddr : &hdr->daddr);
ipv6_addr_copy(&fl->fl6_src, reverse ? &hdr->daddr : &hdr->saddr);
while (nh + offset + 1 < skb->data ||
pskb_may_pull(skb, nh + offset + 1 - skb->data)) {
nh = skb_network_header(skb);
exthdr = (struct ipv6_opt_hdr *)(nh + offset);
switch (nexthdr) {
case NEXTHDR_FRAGMENT:
onlyproto = 1;
case NEXTHDR_ROUTING:
case NEXTHDR_HOP:
case NEXTHDR_DEST:
offset += ipv6_optlen(exthdr);
nexthdr = exthdr->nexthdr;
exthdr = (struct ipv6_opt_hdr *)(nh + offset);
break;
case IPPROTO_UDP:
case IPPROTO_UDPLITE:
case IPPROTO_TCP:
case IPPROTO_SCTP:
case IPPROTO_DCCP:
if (!onlyproto && (nh + offset + 4 < skb->data ||
pskb_may_pull(skb, nh + offset + 4 - skb->data))) {
__be16 *ports = (__be16 *)exthdr;
fl->fl_ip_sport = ports[!!reverse];
fl->fl_ip_dport = ports[!reverse];
}
fl->proto = nexthdr;
return;
case IPPROTO_ICMPV6:
if (!onlyproto && pskb_may_pull(skb, nh + offset + 2 - skb->data)) {
u8 *icmp = (u8 *)exthdr;
fl->fl_icmp_type = icmp[0];
fl->fl_icmp_code = icmp[1];
}
fl->proto = nexthdr;
return;
#if defined(CONFIG_IPV6_MIP6) || defined(CONFIG_IPV6_MIP6_MODULE)
case IPPROTO_MH:
if (!onlyproto && pskb_may_pull(skb, nh + offset + 3 - skb->data)) {
struct ip6_mh *mh;
mh = (struct ip6_mh *)exthdr;
fl->fl_mh_type = mh->ip6mh_type;
}
fl->proto = nexthdr;
return;
#endif
/* XXX Why are there these headers? */
case IPPROTO_AH:
case IPPROTO_ESP:
case IPPROTO_COMP:
default:
fl->fl_ipsec_spi = 0;
fl->proto = nexthdr;
return;
}
}
}
static inline int xfrm6_garbage_collect(struct dst_ops *ops)
{
struct net *net = container_of(ops, struct net, xfrm.xfrm6_dst_ops);
xfrm6_policy_afinfo.garbage_collect(net);
return dst_entries_get_fast(ops) > ops->gc_thresh * 2;
}
static void xfrm6_update_pmtu(struct dst_entry *dst, u32 mtu)
{
struct xfrm_dst *xdst = (struct xfrm_dst *)dst;
struct dst_entry *path = xdst->route;
path->ops->update_pmtu(path, mtu);
}
static void xfrm6_dst_destroy(struct dst_entry *dst)
{
struct xfrm_dst *xdst = (struct xfrm_dst *)dst;
if (likely(xdst->u.rt6.rt6i_idev))
in6_dev_put(xdst->u.rt6.rt6i_idev);
xfrm_dst_destroy(xdst);
}
static void xfrm6_dst_ifdown(struct dst_entry *dst, struct net_device *dev,
int unregister)
{
struct xfrm_dst *xdst;
if (!unregister)
return;
xdst = (struct xfrm_dst *)dst;
if (xdst->u.rt6.rt6i_idev->dev == dev) {
struct inet6_dev *loopback_idev =
in6_dev_get(dev_net(dev)->loopback_dev);
BUG_ON(!loopback_idev);
do {
in6_dev_put(xdst->u.rt6.rt6i_idev);
xdst->u.rt6.rt6i_idev = loopback_idev;
in6_dev_hold(loopback_idev);
xdst = (struct xfrm_dst *)xdst->u.dst.child;
} while (xdst->u.dst.xfrm);
__in6_dev_put(loopback_idev);
}
xfrm_dst_ifdown(dst, dev);
}
static struct dst_ops xfrm6_dst_ops = {
.family = AF_INET6,
.protocol = cpu_to_be16(ETH_P_IPV6),
.gc = xfrm6_garbage_collect,
.update_pmtu = xfrm6_update_pmtu,
.destroy = xfrm6_dst_destroy,
.ifdown = xfrm6_dst_ifdown,
.local_out = __ip6_local_out,
.gc_thresh = 1024,
};
static struct xfrm_policy_afinfo xfrm6_policy_afinfo = {
.family = AF_INET6,
.dst_ops = &xfrm6_dst_ops,
.dst_lookup = xfrm6_dst_lookup,
.get_saddr = xfrm6_get_saddr,
.decode_session = _decode_session6,
.get_tos = xfrm6_get_tos,
.init_path = xfrm6_init_path,
.fill_dst = xfrm6_fill_dst,
};
static int __init xfrm6_policy_init(void)
{
return xfrm_policy_register_afinfo(&xfrm6_policy_afinfo);
}
static void xfrm6_policy_fini(void)
{
xfrm_policy_unregister_afinfo(&xfrm6_policy_afinfo);
}
#ifdef CONFIG_SYSCTL
static struct ctl_table xfrm6_policy_table[] = {
{
.procname = "xfrm6_gc_thresh",
.data = &init_net.xfrm.xfrm6_dst_ops.gc_thresh,
.maxlen = sizeof(int),
.mode = 0644,
.proc_handler = proc_dointvec,
},
{ }
};
static struct ctl_table_header *sysctl_hdr;
#endif
int __init xfrm6_init(void)
{
int ret;
unsigned int gc_thresh;
/*
* We need a good default value for the xfrm6 gc threshold.
* In ipv4 we set it to the route hash table size * 8, which
* is half the size of the maximaum route cache for ipv4. It
* would be good to do the same thing for v6, except the table is
* constructed differently here. Here each table for a net namespace
* can have FIB_TABLE_HASHSZ entries, so lets go with the same
* computation that we used for ipv4 here. Also, lets keep the initial
* gc_thresh to a minimum of 1024, since, the ipv6 route cache defaults
* to that as a minimum as well
*/
gc_thresh = FIB6_TABLE_HASHSZ * 8;
xfrm6_dst_ops.gc_thresh = (gc_thresh < 1024) ? 1024 : gc_thresh;
dst_entries_init(&xfrm6_dst_ops);
ret = xfrm6_policy_init();
if (ret) {
dst_entries_destroy(&xfrm6_dst_ops);
goto out;
}
ret = xfrm6_state_init();
if (ret)
goto out_policy;
#ifdef CONFIG_SYSCTL
sysctl_hdr = register_net_sysctl_table(&init_net, net_ipv6_ctl_path,
xfrm6_policy_table);
#endif
out:
return ret;
out_policy:
xfrm6_policy_fini();
goto out;
}
void xfrm6_fini(void)
{
#ifdef CONFIG_SYSCTL
if (sysctl_hdr)
unregister_net_sysctl_table(sysctl_hdr);
#endif
//xfrm6_input_fini();
xfrm6_policy_fini();
xfrm6_state_fini();
dst_entries_destroy(&xfrm6_dst_ops);
}