linux/net/xfrm/xfrm_output.c
David S. Miller 1806c13dc2 Merge git://git.kernel.org/pub/scm/linux/kernel/git/netdev/net
xdp_umem.c had overlapping changes between the 64-bit math fix
for the calculation of npgs and the removal of the zerocopy
memory type which got rid of the chunk_size_nohdr member.

The mlx5 Kconfig conflict is a case where we just take the
net-next copy of the Kconfig entry dependency as it takes on
the ESWITCH dependency by one level of indirection which is
what the 'net' conflicting change is trying to ensure.

Signed-off-by: David S. Miller <davem@davemloft.net>
2020-05-31 17:48:46 -07:00

771 lines
18 KiB
C

// SPDX-License-Identifier: GPL-2.0-or-later
/*
* xfrm_output.c - Common IPsec encapsulation code.
*
* Copyright (c) 2007 Herbert Xu <herbert@gondor.apana.org.au>
*/
#include <linux/errno.h>
#include <linux/module.h>
#include <linux/netdevice.h>
#include <linux/netfilter.h>
#include <linux/skbuff.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <net/dst.h>
#include <net/icmp.h>
#include <net/inet_ecn.h>
#include <net/xfrm.h>
#if IS_ENABLED(CONFIG_IPV6)
#include <net/ip6_route.h>
#include <net/ipv6_stubs.h>
#endif
#include "xfrm_inout.h"
static int xfrm_output2(struct net *net, struct sock *sk, struct sk_buff *skb);
static int xfrm_inner_extract_output(struct xfrm_state *x, struct sk_buff *skb);
static int xfrm_skb_check_space(struct sk_buff *skb)
{
struct dst_entry *dst = skb_dst(skb);
int nhead = dst->header_len + LL_RESERVED_SPACE(dst->dev)
- skb_headroom(skb);
int ntail = dst->dev->needed_tailroom - skb_tailroom(skb);
if (nhead <= 0) {
if (ntail <= 0)
return 0;
nhead = 0;
} else if (ntail < 0)
ntail = 0;
return pskb_expand_head(skb, nhead, ntail, GFP_ATOMIC);
}
/* Children define the path of the packet through the
* Linux networking. Thus, destinations are stackable.
*/
static struct dst_entry *skb_dst_pop(struct sk_buff *skb)
{
struct dst_entry *child = dst_clone(xfrm_dst_child(skb_dst(skb)));
skb_dst_drop(skb);
return child;
}
/* Add encapsulation header.
*
* The IP header will be moved forward to make space for the encapsulation
* header.
*/
static int xfrm4_transport_output(struct xfrm_state *x, struct sk_buff *skb)
{
struct iphdr *iph = ip_hdr(skb);
int ihl = iph->ihl * 4;
skb_set_inner_transport_header(skb, skb_transport_offset(skb));
skb_set_network_header(skb, -x->props.header_len);
skb->mac_header = skb->network_header +
offsetof(struct iphdr, protocol);
skb->transport_header = skb->network_header + ihl;
__skb_pull(skb, ihl);
memmove(skb_network_header(skb), iph, ihl);
return 0;
}
/* Add encapsulation header.
*
* The IP header and mutable extension headers will be moved forward to make
* space for the encapsulation header.
*/
static int xfrm6_transport_output(struct xfrm_state *x, struct sk_buff *skb)
{
#if IS_ENABLED(CONFIG_IPV6)
struct ipv6hdr *iph;
u8 *prevhdr;
int hdr_len;
iph = ipv6_hdr(skb);
skb_set_inner_transport_header(skb, skb_transport_offset(skb));
hdr_len = x->type->hdr_offset(x, skb, &prevhdr);
if (hdr_len < 0)
return hdr_len;
skb_set_mac_header(skb,
(prevhdr - x->props.header_len) - skb->data);
skb_set_network_header(skb, -x->props.header_len);
skb->transport_header = skb->network_header + hdr_len;
__skb_pull(skb, hdr_len);
memmove(ipv6_hdr(skb), iph, hdr_len);
return 0;
#else
WARN_ON_ONCE(1);
return -EAFNOSUPPORT;
#endif
}
/* Add route optimization header space.
*
* The IP header and mutable extension headers will be moved forward to make
* space for the route optimization header.
*/
static int xfrm6_ro_output(struct xfrm_state *x, struct sk_buff *skb)
{
#if IS_ENABLED(CONFIG_IPV6)
struct ipv6hdr *iph;
u8 *prevhdr;
int hdr_len;
iph = ipv6_hdr(skb);
hdr_len = x->type->hdr_offset(x, skb, &prevhdr);
if (hdr_len < 0)
return hdr_len;
skb_set_mac_header(skb,
(prevhdr - x->props.header_len) - skb->data);
skb_set_network_header(skb, -x->props.header_len);
skb->transport_header = skb->network_header + hdr_len;
__skb_pull(skb, hdr_len);
memmove(ipv6_hdr(skb), iph, hdr_len);
x->lastused = ktime_get_real_seconds();
return 0;
#else
WARN_ON_ONCE(1);
return -EAFNOSUPPORT;
#endif
}
/* Add encapsulation header.
*
* The top IP header will be constructed per draft-nikander-esp-beet-mode-06.txt.
*/
static int xfrm4_beet_encap_add(struct xfrm_state *x, struct sk_buff *skb)
{
struct ip_beet_phdr *ph;
struct iphdr *top_iph;
int hdrlen, optlen;
hdrlen = 0;
optlen = XFRM_MODE_SKB_CB(skb)->optlen;
if (unlikely(optlen))
hdrlen += IPV4_BEET_PHMAXLEN - (optlen & 4);
skb_set_network_header(skb, -x->props.header_len - hdrlen +
(XFRM_MODE_SKB_CB(skb)->ihl - sizeof(*top_iph)));
if (x->sel.family != AF_INET6)
skb->network_header += IPV4_BEET_PHMAXLEN;
skb->mac_header = skb->network_header +
offsetof(struct iphdr, protocol);
skb->transport_header = skb->network_header + sizeof(*top_iph);
xfrm4_beet_make_header(skb);
ph = __skb_pull(skb, XFRM_MODE_SKB_CB(skb)->ihl - hdrlen);
top_iph = ip_hdr(skb);
if (unlikely(optlen)) {
if (WARN_ON(optlen < 0))
return -EINVAL;
ph->padlen = 4 - (optlen & 4);
ph->hdrlen = optlen / 8;
ph->nexthdr = top_iph->protocol;
if (ph->padlen)
memset(ph + 1, IPOPT_NOP, ph->padlen);
top_iph->protocol = IPPROTO_BEETPH;
top_iph->ihl = sizeof(struct iphdr) / 4;
}
top_iph->saddr = x->props.saddr.a4;
top_iph->daddr = x->id.daddr.a4;
return 0;
}
/* Add encapsulation header.
*
* The top IP header will be constructed per RFC 2401.
*/
static int xfrm4_tunnel_encap_add(struct xfrm_state *x, struct sk_buff *skb)
{
struct dst_entry *dst = skb_dst(skb);
struct iphdr *top_iph;
int flags;
skb_set_inner_network_header(skb, skb_network_offset(skb));
skb_set_inner_transport_header(skb, skb_transport_offset(skb));
skb_set_network_header(skb, -x->props.header_len);
skb->mac_header = skb->network_header +
offsetof(struct iphdr, protocol);
skb->transport_header = skb->network_header + sizeof(*top_iph);
top_iph = ip_hdr(skb);
top_iph->ihl = 5;
top_iph->version = 4;
top_iph->protocol = xfrm_af2proto(skb_dst(skb)->ops->family);
/* DS disclosing depends on XFRM_SA_XFLAG_DONT_ENCAP_DSCP */
if (x->props.extra_flags & XFRM_SA_XFLAG_DONT_ENCAP_DSCP)
top_iph->tos = 0;
else
top_iph->tos = XFRM_MODE_SKB_CB(skb)->tos;
top_iph->tos = INET_ECN_encapsulate(top_iph->tos,
XFRM_MODE_SKB_CB(skb)->tos);
flags = x->props.flags;
if (flags & XFRM_STATE_NOECN)
IP_ECN_clear(top_iph);
top_iph->frag_off = (flags & XFRM_STATE_NOPMTUDISC) ?
0 : (XFRM_MODE_SKB_CB(skb)->frag_off & htons(IP_DF));
top_iph->ttl = ip4_dst_hoplimit(xfrm_dst_child(dst));
top_iph->saddr = x->props.saddr.a4;
top_iph->daddr = x->id.daddr.a4;
ip_select_ident(dev_net(dst->dev), skb, NULL);
return 0;
}
#if IS_ENABLED(CONFIG_IPV6)
static int xfrm6_tunnel_encap_add(struct xfrm_state *x, struct sk_buff *skb)
{
struct dst_entry *dst = skb_dst(skb);
struct ipv6hdr *top_iph;
int dsfield;
skb_set_inner_network_header(skb, skb_network_offset(skb));
skb_set_inner_transport_header(skb, skb_transport_offset(skb));
skb_set_network_header(skb, -x->props.header_len);
skb->mac_header = skb->network_header +
offsetof(struct ipv6hdr, nexthdr);
skb->transport_header = skb->network_header + sizeof(*top_iph);
top_iph = ipv6_hdr(skb);
top_iph->version = 6;
memcpy(top_iph->flow_lbl, XFRM_MODE_SKB_CB(skb)->flow_lbl,
sizeof(top_iph->flow_lbl));
top_iph->nexthdr = xfrm_af2proto(skb_dst(skb)->ops->family);
if (x->props.extra_flags & XFRM_SA_XFLAG_DONT_ENCAP_DSCP)
dsfield = 0;
else
dsfield = XFRM_MODE_SKB_CB(skb)->tos;
dsfield = INET_ECN_encapsulate(dsfield, XFRM_MODE_SKB_CB(skb)->tos);
if (x->props.flags & XFRM_STATE_NOECN)
dsfield &= ~INET_ECN_MASK;
ipv6_change_dsfield(top_iph, 0, dsfield);
top_iph->hop_limit = ip6_dst_hoplimit(xfrm_dst_child(dst));
top_iph->saddr = *(struct in6_addr *)&x->props.saddr;
top_iph->daddr = *(struct in6_addr *)&x->id.daddr;
return 0;
}
static int xfrm6_beet_encap_add(struct xfrm_state *x, struct sk_buff *skb)
{
struct ipv6hdr *top_iph;
struct ip_beet_phdr *ph;
int optlen, hdr_len;
hdr_len = 0;
optlen = XFRM_MODE_SKB_CB(skb)->optlen;
if (unlikely(optlen))
hdr_len += IPV4_BEET_PHMAXLEN - (optlen & 4);
skb_set_network_header(skb, -x->props.header_len - hdr_len);
if (x->sel.family != AF_INET6)
skb->network_header += IPV4_BEET_PHMAXLEN;
skb->mac_header = skb->network_header +
offsetof(struct ipv6hdr, nexthdr);
skb->transport_header = skb->network_header + sizeof(*top_iph);
ph = __skb_pull(skb, XFRM_MODE_SKB_CB(skb)->ihl - hdr_len);
xfrm6_beet_make_header(skb);
top_iph = ipv6_hdr(skb);
if (unlikely(optlen)) {
if (WARN_ON(optlen < 0))
return -EINVAL;
ph->padlen = 4 - (optlen & 4);
ph->hdrlen = optlen / 8;
ph->nexthdr = top_iph->nexthdr;
if (ph->padlen)
memset(ph + 1, IPOPT_NOP, ph->padlen);
top_iph->nexthdr = IPPROTO_BEETPH;
}
top_iph->saddr = *(struct in6_addr *)&x->props.saddr;
top_iph->daddr = *(struct in6_addr *)&x->id.daddr;
return 0;
}
#endif
/* Add encapsulation header.
*
* On exit, the transport header will be set to the start of the
* encapsulation header to be filled in by x->type->output and the mac
* header will be set to the nextheader (protocol for IPv4) field of the
* extension header directly preceding the encapsulation header, or in
* its absence, that of the top IP header.
* The value of the network header will always point to the top IP header
* while skb->data will point to the payload.
*/
static int xfrm4_prepare_output(struct xfrm_state *x, struct sk_buff *skb)
{
int err;
err = xfrm_inner_extract_output(x, skb);
if (err)
return err;
IPCB(skb)->flags |= IPSKB_XFRM_TUNNEL_SIZE;
skb->protocol = htons(ETH_P_IP);
switch (x->outer_mode.encap) {
case XFRM_MODE_BEET:
return xfrm4_beet_encap_add(x, skb);
case XFRM_MODE_TUNNEL:
return xfrm4_tunnel_encap_add(x, skb);
}
WARN_ON_ONCE(1);
return -EOPNOTSUPP;
}
static int xfrm6_prepare_output(struct xfrm_state *x, struct sk_buff *skb)
{
#if IS_ENABLED(CONFIG_IPV6)
int err;
err = xfrm_inner_extract_output(x, skb);
if (err)
return err;
skb->ignore_df = 1;
skb->protocol = htons(ETH_P_IPV6);
switch (x->outer_mode.encap) {
case XFRM_MODE_BEET:
return xfrm6_beet_encap_add(x, skb);
case XFRM_MODE_TUNNEL:
return xfrm6_tunnel_encap_add(x, skb);
default:
WARN_ON_ONCE(1);
return -EOPNOTSUPP;
}
#endif
WARN_ON_ONCE(1);
return -EAFNOSUPPORT;
}
static int xfrm_outer_mode_output(struct xfrm_state *x, struct sk_buff *skb)
{
switch (x->outer_mode.encap) {
case XFRM_MODE_BEET:
case XFRM_MODE_TUNNEL:
if (x->outer_mode.family == AF_INET)
return xfrm4_prepare_output(x, skb);
if (x->outer_mode.family == AF_INET6)
return xfrm6_prepare_output(x, skb);
break;
case XFRM_MODE_TRANSPORT:
if (x->outer_mode.family == AF_INET)
return xfrm4_transport_output(x, skb);
if (x->outer_mode.family == AF_INET6)
return xfrm6_transport_output(x, skb);
break;
case XFRM_MODE_ROUTEOPTIMIZATION:
if (x->outer_mode.family == AF_INET6)
return xfrm6_ro_output(x, skb);
WARN_ON_ONCE(1);
break;
default:
WARN_ON_ONCE(1);
break;
}
return -EOPNOTSUPP;
}
#if IS_ENABLED(CONFIG_NET_PKTGEN)
int pktgen_xfrm_outer_mode_output(struct xfrm_state *x, struct sk_buff *skb)
{
return xfrm_outer_mode_output(x, skb);
}
EXPORT_SYMBOL_GPL(pktgen_xfrm_outer_mode_output);
#endif
static int xfrm_output_one(struct sk_buff *skb, int err)
{
struct dst_entry *dst = skb_dst(skb);
struct xfrm_state *x = dst->xfrm;
struct net *net = xs_net(x);
if (err <= 0)
goto resume;
do {
err = xfrm_skb_check_space(skb);
if (err) {
XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTERROR);
goto error_nolock;
}
skb->mark = xfrm_smark_get(skb->mark, x);
err = xfrm_outer_mode_output(x, skb);
if (err) {
XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTSTATEMODEERROR);
goto error_nolock;
}
spin_lock_bh(&x->lock);
if (unlikely(x->km.state != XFRM_STATE_VALID)) {
XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTSTATEINVALID);
err = -EINVAL;
goto error;
}
err = xfrm_state_check_expire(x);
if (err) {
XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTSTATEEXPIRED);
goto error;
}
err = x->repl->overflow(x, skb);
if (err) {
XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTSTATESEQERROR);
goto error;
}
x->curlft.bytes += skb->len;
x->curlft.packets++;
spin_unlock_bh(&x->lock);
skb_dst_force(skb);
if (!skb_dst(skb)) {
XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTERROR);
err = -EHOSTUNREACH;
goto error_nolock;
}
if (xfrm_offload(skb)) {
x->type_offload->encap(x, skb);
} else {
/* Inner headers are invalid now. */
skb->encapsulation = 0;
err = x->type->output(x, skb);
if (err == -EINPROGRESS)
goto out;
}
resume:
if (err) {
XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTSTATEPROTOERROR);
goto error_nolock;
}
dst = skb_dst_pop(skb);
if (!dst) {
XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTERROR);
err = -EHOSTUNREACH;
goto error_nolock;
}
skb_dst_set(skb, dst);
x = dst->xfrm;
} while (x && !(x->outer_mode.flags & XFRM_MODE_FLAG_TUNNEL));
return 0;
error:
spin_unlock_bh(&x->lock);
error_nolock:
kfree_skb(skb);
out:
return err;
}
int xfrm_output_resume(struct sk_buff *skb, int err)
{
struct net *net = xs_net(skb_dst(skb)->xfrm);
while (likely((err = xfrm_output_one(skb, err)) == 0)) {
nf_reset_ct(skb);
err = skb_dst(skb)->ops->local_out(net, skb->sk, skb);
if (unlikely(err != 1))
goto out;
if (!skb_dst(skb)->xfrm)
return dst_output(net, skb->sk, skb);
err = nf_hook(skb_dst(skb)->ops->family,
NF_INET_POST_ROUTING, net, skb->sk, skb,
NULL, skb_dst(skb)->dev, xfrm_output2);
if (unlikely(err != 1))
goto out;
}
if (err == -EINPROGRESS)
err = 0;
out:
return err;
}
EXPORT_SYMBOL_GPL(xfrm_output_resume);
static int xfrm_output2(struct net *net, struct sock *sk, struct sk_buff *skb)
{
return xfrm_output_resume(skb, 1);
}
static int xfrm_output_gso(struct net *net, struct sock *sk, struct sk_buff *skb)
{
struct sk_buff *segs, *nskb;
BUILD_BUG_ON(sizeof(*IPCB(skb)) > SKB_GSO_CB_OFFSET);
BUILD_BUG_ON(sizeof(*IP6CB(skb)) > SKB_GSO_CB_OFFSET);
segs = skb_gso_segment(skb, 0);
kfree_skb(skb);
if (IS_ERR(segs))
return PTR_ERR(segs);
if (segs == NULL)
return -EINVAL;
skb_list_walk_safe(segs, segs, nskb) {
int err;
skb_mark_not_on_list(segs);
err = xfrm_output2(net, sk, segs);
if (unlikely(err)) {
kfree_skb_list(nskb);
return err;
}
}
return 0;
}
int xfrm_output(struct sock *sk, struct sk_buff *skb)
{
struct net *net = dev_net(skb_dst(skb)->dev);
struct xfrm_state *x = skb_dst(skb)->xfrm;
int err;
switch (x->outer_mode.family) {
case AF_INET:
memset(IPCB(skb), 0, sizeof(*IPCB(skb)));
#ifdef CONFIG_NETFILTER
IPCB(skb)->flags |= IPSKB_XFRM_TRANSFORMED;
#endif
break;
case AF_INET6:
memset(IP6CB(skb), 0, sizeof(*IP6CB(skb)));
#ifdef CONFIG_NETFILTER
IP6CB(skb)->flags |= IP6SKB_XFRM_TRANSFORMED;
#endif
break;
}
secpath_reset(skb);
if (xfrm_dev_offload_ok(skb, x)) {
struct sec_path *sp;
sp = secpath_set(skb);
if (!sp) {
XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTERROR);
kfree_skb(skb);
return -ENOMEM;
}
skb->encapsulation = 1;
sp->olen++;
sp->xvec[sp->len++] = x;
xfrm_state_hold(x);
if (skb_is_gso(skb)) {
if (skb->inner_protocol)
return xfrm_output_gso(net, sk, skb);
skb_shinfo(skb)->gso_type |= SKB_GSO_ESP;
goto out;
}
if (x->xso.dev && x->xso.dev->features & NETIF_F_HW_ESP_TX_CSUM)
goto out;
} else {
if (skb_is_gso(skb))
return xfrm_output_gso(net, sk, skb);
}
if (skb->ip_summed == CHECKSUM_PARTIAL) {
err = skb_checksum_help(skb);
if (err) {
XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTERROR);
kfree_skb(skb);
return err;
}
}
out:
return xfrm_output2(net, sk, skb);
}
EXPORT_SYMBOL_GPL(xfrm_output);
static int xfrm4_tunnel_check_size(struct sk_buff *skb)
{
int mtu, ret = 0;
if (IPCB(skb)->flags & IPSKB_XFRM_TUNNEL_SIZE)
goto out;
if (!(ip_hdr(skb)->frag_off & htons(IP_DF)) || skb->ignore_df)
goto out;
mtu = dst_mtu(skb_dst(skb));
if ((!skb_is_gso(skb) && skb->len > mtu) ||
(skb_is_gso(skb) &&
!skb_gso_validate_network_len(skb, ip_skb_dst_mtu(skb->sk, skb)))) {
skb->protocol = htons(ETH_P_IP);
if (skb->sk)
xfrm_local_error(skb, mtu);
else
icmp_send(skb, ICMP_DEST_UNREACH,
ICMP_FRAG_NEEDED, htonl(mtu));
ret = -EMSGSIZE;
}
out:
return ret;
}
static int xfrm4_extract_output(struct xfrm_state *x, struct sk_buff *skb)
{
int err;
err = xfrm4_tunnel_check_size(skb);
if (err)
return err;
XFRM_MODE_SKB_CB(skb)->protocol = ip_hdr(skb)->protocol;
xfrm4_extract_header(skb);
return 0;
}
#if IS_ENABLED(CONFIG_IPV6)
static int xfrm6_tunnel_check_size(struct sk_buff *skb)
{
int mtu, ret = 0;
struct dst_entry *dst = skb_dst(skb);
if (skb->ignore_df)
goto out;
mtu = dst_mtu(dst);
if (mtu < IPV6_MIN_MTU)
mtu = IPV6_MIN_MTU;
if ((!skb_is_gso(skb) && skb->len > mtu) ||
(skb_is_gso(skb) &&
!skb_gso_validate_network_len(skb, ip6_skb_dst_mtu(skb)))) {
skb->dev = dst->dev;
skb->protocol = htons(ETH_P_IPV6);
if (xfrm6_local_dontfrag(skb->sk))
ipv6_stub->xfrm6_local_rxpmtu(skb, mtu);
else if (skb->sk)
xfrm_local_error(skb, mtu);
else
icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu);
ret = -EMSGSIZE;
}
out:
return ret;
}
#endif
static int xfrm6_extract_output(struct xfrm_state *x, struct sk_buff *skb)
{
#if IS_ENABLED(CONFIG_IPV6)
int err;
err = xfrm6_tunnel_check_size(skb);
if (err)
return err;
XFRM_MODE_SKB_CB(skb)->protocol = ipv6_hdr(skb)->nexthdr;
xfrm6_extract_header(skb);
return 0;
#else
WARN_ON_ONCE(1);
return -EAFNOSUPPORT;
#endif
}
static int xfrm_inner_extract_output(struct xfrm_state *x, struct sk_buff *skb)
{
const struct xfrm_mode *inner_mode;
if (x->sel.family == AF_UNSPEC)
inner_mode = xfrm_ip2inner_mode(x,
xfrm_af2proto(skb_dst(skb)->ops->family));
else
inner_mode = &x->inner_mode;
if (inner_mode == NULL)
return -EAFNOSUPPORT;
switch (inner_mode->family) {
case AF_INET:
return xfrm4_extract_output(x, skb);
case AF_INET6:
return xfrm6_extract_output(x, skb);
}
return -EAFNOSUPPORT;
}
void xfrm_local_error(struct sk_buff *skb, int mtu)
{
unsigned int proto;
struct xfrm_state_afinfo *afinfo;
if (skb->protocol == htons(ETH_P_IP))
proto = AF_INET;
else if (skb->protocol == htons(ETH_P_IPV6) &&
skb->sk->sk_family == AF_INET6)
proto = AF_INET6;
else
return;
afinfo = xfrm_state_get_afinfo(proto);
if (afinfo) {
afinfo->local_error(skb, mtu);
rcu_read_unlock();
}
}
EXPORT_SYMBOL_GPL(xfrm_local_error);