system/freebsd-src
system/freebsd-src
1
0
Fork 0
mirror of https://github.com/freebsd/freebsd-src synced 2024-07-22 02:37:15 +00:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

UDP: further performance improvements on tx

Browse source 
Cumulative throughput while running 64
  netperf -H $DUT -t UDP_STREAM -- -m 1
on a 2x8x2 SKL went from 1.1Mpps to 2.5Mpps

Single stream throughput increases from 910kpps to 1.18Mpps

Baseline:
https://people.freebsd.org/~mmacy/2018.05.11/udpsender2.svg

- Protect read access to global ifnet list with epoch
https://people.freebsd.org/~mmacy/2018.05.11/udpsender3.svg

- Protect short lived ifaddr references with epoch
https://people.freebsd.org/~mmacy/2018.05.11/udpsender4.svg

- Convert if_afdata read lock path to epoch
https://people.freebsd.org/~mmacy/2018.05.11/udpsender5.svg

A fix for the inpcbhash contention is pending sufficient time
on a canary at LLNW.

Reviewed by:	gallatin
Sponsored by:	Limelight Networks
Differential Revision:	https://reviews.freebsd.org/D15409
This commit is contained in:
Matt Macy 2018-05-23 21:02:14 +00:00
parent 630ba2c514
commit 4f6c66cc9c
Notes: svn2git 2020-12-20 02:59:44 +00:00 
svn path=/head/; revision=334118
38 changed files with 312 additions and 313 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
sys/compat/linprocfs/linprocfs.c
View file

@ -1138,7 +1138,7 @@ linux_ifname(struct ifnet *ifp, char *buffer, size_t buflen)
/* Determine the (relative) unit number for ethernet interfaces */
ethno = 0;
TAILQ_FOREACH(ifscan, &V_ifnet, if_link) {
CK_STAILQ_FOREACH(ifscan, &V_ifnet, if_link) {
if (ifscan == ifp)
return (snprintf(buffer, buflen, "eth%d", ethno));
if (IFP_IS_ETH(ifscan))
@ -1166,7 +1166,7 @@ linprocfs_donetdev(PFS_FILL_ARGS)
CURVNET_SET(TD_TO_VNET(curthread));
IFNET_RLOCK();
TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) {
linux_ifname(ifp, ifname, sizeof ifname);
sbuf_printf(sb, "%6.6s: ", ifname);
sbuf_printf(sb, "%7ju %7ju %4ju %4ju %4lu %5lu %10lu %9ju ",

8
sys/compat/linux/linux_ioctl.c
View file

@ -2150,7 +2150,7 @@ ifname_linux_to_bsd(struct thread *td, const char *lxname, char *bsdname)
is_eth = (len == 3 && !strncmp(lxname, "eth", len)) ? 1 : 0;
CURVNET_SET(TD_TO_VNET(td));
IFNET_RLOCK();
TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) {
/*
* Allow Linux programs to use FreeBSD names. Don't presume
* we never have an interface named "eth", so don't make
@ -2188,7 +2188,7 @@ linux_ioctl_ifname(struct thread *td, struct l_ifreq *uifr)
index = 1; /* ifr.ifr_ifindex starts from 1 */
ethno = 0;
error = ENODEV;
TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) {
if (ifr.ifr_ifindex == index) {
if (IFP_IS_ETH(ifp))
snprintf(ifr.ifr_name, LINUX_IFNAMSIZ,
@ -2240,7 +2240,7 @@ linux_ifconf(struct thread *td, struct ifconf *uifc)
if ((l_uintptr_t)ifc.ifc_buf == PTROUT(NULL)) {
ifc.ifc_len = 0;
IFNET_RLOCK();
TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) {
CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
struct sockaddr *sa = ifa->ifa_addr;
if (sa->sa_family == AF_INET)
@ -2271,7 +2271,7 @@ linux_ifconf(struct thread *td, struct ifconf *uifc)
/* Return all AF_INET addresses of all interfaces */
IFNET_RLOCK();
TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) {
int addrs = 0;
bzero(&ifr, sizeof(ifr));

6
sys/compat/linuxkpi/common/include/linux/inetdevice.h
View file

@ -44,16 +44,17 @@ ip_dev_find(struct vnet *vnet, uint32_t addr)
sin.sin_addr.s_addr = addr;
sin.sin_len = sizeof(sin);
sin.sin_family = AF_INET;
NET_EPOCH_ENTER();
CURVNET_SET_QUIET(vnet);
ifa = ifa_ifwithaddr((struct sockaddr *)&sin);
CURVNET_RESTORE();
if (ifa) {
ifp = ifa->ifa_ifp;
if_ref(ifp);
ifa_free(ifa);
} else {
ifp = NULL;
}
NET_EPOCH_EXIT();
return (ifp);
}
@ -69,6 +70,7 @@ ip6_dev_find(struct vnet *vnet, struct in6_addr addr)
sin6.sin6_addr = addr;
sin6.sin6_len = sizeof(sin6);
sin6.sin6_family = AF_INET6;
NET_EPOCH_ENTER();
CURVNET_SET_QUIET(vnet);
if (IN6_IS_SCOPE_LINKLOCAL(&addr) ||
IN6_IS_ADDR_MC_INTFACELOCAL(&addr)) {
@ -85,8 +87,8 @@ ip6_dev_find(struct vnet *vnet, struct in6_addr addr)
if (ifa != NULL) {
ifp = ifa->ifa_ifp;
if_ref(ifp);
ifa_free(ifa);
}
NET_EPOCH_EXIT();
CURVNET_RESTORE();
return (ifp);
}

2
sys/dev/mlx5/mlx5_ib/mlx5_ib_main.c
View file

@ -2771,7 +2771,7 @@ static int mlx5_enable_roce(struct mlx5_ib_dev *dev)
VNET_FOREACH(vnet_iter) {
IFNET_RLOCK();
CURVNET_SET_QUIET(vnet_iter);
TAILQ_FOREACH(idev, &V_ifnet, if_link) {
CK_STAILQ_FOREACH(idev, &V_ifnet, if_link) {
/* check if network interface belongs to mlx5en */
if (!mlx5_netdev_match(idev, dev->mdev, "mce"))
continue;

2
sys/dev/wtap/if_wtap.c
View file

@ -108,7 +108,7 @@ wtap_node_write(struct cdev *dev, struct uio *uio, int ioflag)
CURVNET_SET(TD_TO_VNET(curthread));
IFNET_RLOCK_NOSLEEP();
TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) {
printf("ifp->if_xname = %s\n", ifp->if_xname);
if(strcmp(devtoname(dev), ifp->if_xname) == 0){
printf("found match, correspoding wtap = %s\n",

4
sys/net/altq/altq_subr.c
View file

@ -434,8 +434,8 @@ tbr_timeout(arg)
VNET_LIST_RLOCK_NOSLEEP();
VNET_FOREACH(vnet_iter) {
CURVNET_SET(vnet_iter);
for (ifp = TAILQ_FIRST(&V_ifnet); ifp;
ifp = TAILQ_NEXT(ifp, if_link)) {
for (ifp = CK_STAILQ_FIRST(&V_ifnet); ifp;
ifp = CK_STAILQ_NEXT(ifp, if_link)) {
/* read from if_snd unlocked */
if (!TBR_IS_ENABLED(&ifp->if_snd))
continue;

2
sys/net/bridgestp.c
View file

@ -2043,7 +2043,7 @@ bstp_reinit(struct bstp_state *bs)
* bridges in the same STP domain.
*/
IFNET_RLOCK_NOSLEEP();
TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) {
if (ifp->if_type != IFT_ETHER)
continue; /* Not Ethernet */

288
sys/net/if.c
View file

@ -256,7 +256,7 @@ struct mbuf *(*tbr_dequeue_ptr)(struct ifaltq *, int) = NULL;
static void if_attachdomain(void *);
static void if_attachdomain1(struct ifnet *);
static int ifconf(u_long, caddr_t);
static void if_grow(void);
static void *if_grow(void);
static void if_input_default(struct ifnet *, struct mbuf *);
static int if_requestencap_default(struct ifnet *, struct if_encap_req *);
static void if_route(struct ifnet *, int flag, int fam);
@ -345,9 +345,7 @@ ifnet_byindex(u_short idx)
{
struct ifnet *ifp;
IFNET_RLOCK_NOSLEEP();
ifp = ifnet_byindex_locked(idx);
IFNET_RUNLOCK_NOSLEEP();
return (ifp);
}
@ -372,12 +370,11 @@ ifnet_byindex_ref(u_short idx)
* failure.
*/
static u_short
ifindex_alloc(void)
ifindex_alloc(void **old)
{
u_short idx;
IFNET_WLOCK_ASSERT();
retry:
/*
* Try to find an empty slot below V_if_index. If we fail, take the
* next slot.
@ -389,8 +386,8 @@ ifindex_alloc(void)
/* Catch if_index overflow. */
if (idx >= V_if_indexlim) {
if_grow();
goto retry;
*old = if_grow();
return (USHRT_MAX);
}
if (idx > V_if_index)
V_if_index = idx;
@ -418,22 +415,11 @@ ifindex_free(u_short idx)
IFNET_WUNLOCK();
}
static void
ifnet_setbyindex_locked(u_short idx, struct ifnet *ifp)
{
IFNET_WLOCK_ASSERT();
V_ifindex_table[idx] = ifp;
}
static void
ifnet_setbyindex(u_short idx, struct ifnet *ifp)
{
IFNET_WLOCK();
ifnet_setbyindex_locked(idx, ifp);
IFNET_WUNLOCK();
V_ifindex_table[idx] = ifp;
}
struct ifaddr *
@ -460,12 +446,15 @@ ifaddr_byindex(u_short idx)
static void
vnet_if_init(const void *unused __unused)
{
void *old;
TAILQ_INIT(&V_ifnet);
TAILQ_INIT(&V_ifg_head);
CK_STAILQ_INIT(&V_ifnet);
CK_STAILQ_INIT(&V_ifg_head);
IFNET_WLOCK();
if_grow(); /* create initial table */
old = if_grow(); /* create initial table */
IFNET_WUNLOCK();
epoch_wait_preempt(net_epoch_preempt);
free(old, M_IFNET);
vnet_if_clone_init();
}
VNET_SYSINIT(vnet_if_init, SI_SUB_INIT_IF, SI_ORDER_SECOND, vnet_if_init,
@ -476,9 +465,9 @@ static void
vnet_if_uninit(const void *unused __unused)
{
VNET_ASSERT(TAILQ_EMPTY(&V_ifnet), ("%s:%d tailq &V_ifnet=%p "
VNET_ASSERT(CK_STAILQ_EMPTY(&V_ifnet), ("%s:%d tailq &V_ifnet=%p "
"not empty", __func__, __LINE__, &V_ifnet));
VNET_ASSERT(TAILQ_EMPTY(&V_ifg_head), ("%s:%d tailq &V_ifg_head=%p "
VNET_ASSERT(CK_STAILQ_EMPTY(&V_ifg_head), ("%s:%d tailq &V_ifg_head=%p "
"not empty", __func__, __LINE__, &V_ifg_head));
free((caddr_t)V_ifindex_table, M_IFNET);
@ -492,7 +481,7 @@ vnet_if_return(const void *unused __unused)
struct ifnet *ifp, *nifp;
/* Return all inherited interfaces to their parent vnets. */
TAILQ_FOREACH_SAFE(ifp, &V_ifnet, if_link, nifp) {
CK_STAILQ_FOREACH_SAFE(ifp, &V_ifnet, if_link, nifp) {
if (ifp->if_home_vnet != ifp->if_vnet)
if_vmove(ifp, ifp->if_home_vnet);
}
@ -501,13 +490,16 @@ VNET_SYSUNINIT(vnet_if_return, SI_SUB_VNET_DONE, SI_ORDER_ANY,
vnet_if_return, NULL);
#endif
static void
static void *
if_grow(void)
{
int oldlim;
u_int n;
struct ifnet **e;
void *old;
old = NULL;
IFNET_WLOCK_ASSERT();
oldlim = V_if_indexlim;
IFNET_WUNLOCK();
@ -516,14 +508,15 @@ if_grow(void)
IFNET_WLOCK();
if (V_if_indexlim != oldlim) {
free(e, M_IFNET);
return;
return (NULL);
}
if (V_ifindex_table != NULL) {
memcpy((caddr_t)e, (caddr_t)V_ifindex_table, n/2);
free((caddr_t)V_ifindex_table, M_IFNET);
old = V_ifindex_table;
}
V_if_indexlim <<= 1;
V_ifindex_table = e;
return (old);
}
/*
@ -536,11 +529,19 @@ if_alloc(u_char type)
{
struct ifnet *ifp;
u_short idx;
void *old;
ifp = malloc(sizeof(struct ifnet), M_IFNET, M_WAITOK|M_ZERO);
restart:
IFNET_WLOCK();
idx = ifindex_alloc();
ifnet_setbyindex_locked(idx, IFNET_HOLD);
idx = ifindex_alloc(&old);
if (__predict_false(idx == USHRT_MAX)) {
IFNET_WUNLOCK();
epoch_wait_preempt(net_epoch_preempt);
free(old, M_IFNET);
goto restart;
}
ifnet_setbyindex(idx, IFNET_HOLD);
IFNET_WUNLOCK();
ifp->if_index = idx;
ifp->if_type = type;
@ -563,7 +564,7 @@ if_alloc(u_char type)
IF_AFDATA_LOCK_INIT(ifp);
CK_STAILQ_INIT(&ifp->if_addrhead);
CK_STAILQ_INIT(&ifp->if_multiaddrs);
TAILQ_INIT(&ifp->if_groups);
CK_STAILQ_INIT(&ifp->if_groups);
#ifdef MAC
mac_ifnet_init(ifp);
#endif
@ -609,6 +610,15 @@ if_free_internal(struct ifnet *ifp)
free(ifp, M_IFNET);
}
static void
if_destroy(epoch_context_t ctx)
{
struct ifnet *ifp;
ifp = __containerof(ctx, struct ifnet, if_epoch_ctx);
if_free_internal(ifp);
}
/*
* Deregister an interface and free the associated storage.
*/
@ -627,7 +637,7 @@ if_free(struct ifnet *ifp)
IFNET_WUNLOCK();
if (refcount_release(&ifp->if_refcount))
if_free_internal(ifp);
epoch_call(net_epoch_preempt, &ifp->if_epoch_ctx, if_destroy);
CURVNET_RESTORE();
}
@ -650,7 +660,7 @@ if_rele(struct ifnet *ifp)
if (!refcount_release(&ifp->if_refcount))
return;
if_free_internal(ifp);
epoch_call(net_epoch_preempt, &ifp->if_epoch_ctx, if_destroy);
}
void
@ -883,7 +893,7 @@ if_attach_internal(struct ifnet *ifp, int vmove, struct if_clone *ifc)
#endif
IFNET_WLOCK();
TAILQ_INSERT_TAIL(&V_ifnet, ifp, if_link);
CK_STAILQ_INSERT_TAIL(&V_ifnet, ifp, if_link);
#ifdef VIMAGE
curvnet->vnet_ifcnt++;
#endif
@ -915,7 +925,7 @@ if_attachdomain(void *dummy)
{
struct ifnet *ifp;
TAILQ_FOREACH(ifp, &V_ifnet, if_link)
CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link)
if_attachdomain1(ifp);
}
SYSINIT(domainifattach, SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_SECOND,
@ -1050,9 +1060,9 @@ if_detach_internal(struct ifnet *ifp, int vmove, struct if_clone **ifcp)
ifp->if_vnet->vnet_state < SI_SUB_VNET_DONE) ? 1 : 0;
#endif
IFNET_WLOCK();
TAILQ_FOREACH(iter, &V_ifnet, if_link)
CK_STAILQ_FOREACH(iter, &V_ifnet, if_link)
if (iter == ifp) {
TAILQ_REMOVE(&V_ifnet, ifp, if_link);
CK_STAILQ_REMOVE(&V_ifnet, ifp, ifnet, if_link);
found = 1;
break;
}
@ -1080,7 +1090,7 @@ if_detach_internal(struct ifnet *ifp, int vmove, struct if_clone **ifcp)
#ifdef VIMAGE
curvnet->vnet_ifcnt--;
#endif
epoch_wait_preempt(net_epoch_preempt);
/*
* In any case (destroy or vmove) detach us from the groups
* and remove/wait for pending events on the taskq.
@ -1220,6 +1230,7 @@ if_vmove(struct ifnet *ifp, struct vnet *new_vnet)
{
struct if_clone *ifc;
u_int bif_dlt, bif_hdrlen;
void *old;
int rc;
/*
@ -1260,10 +1271,16 @@ if_vmove(struct ifnet *ifp, struct vnet *new_vnet)
* Switch to the context of the target vnet.
*/
CURVNET_SET_QUIET(new_vnet);
restart:
IFNET_WLOCK();
ifp->if_index = ifindex_alloc();
ifnet_setbyindex_locked(ifp->if_index, ifp);
ifp->if_index = ifindex_alloc(&old);
if (__predict_false(ifp->if_index == USHRT_MAX)) {
IFNET_WUNLOCK();
epoch_wait_preempt(net_epoch_preempt);
free(old, M_IFNET);
goto restart;
}
ifnet_setbyindex(ifp->if_index, ifp);
IFNET_WUNLOCK();
if_attach_internal(ifp, 1, ifc);
@ -1400,7 +1417,7 @@ if_addgroup(struct ifnet *ifp, const char *groupname)
return (EINVAL);
IFNET_WLOCK();
TAILQ_FOREACH(ifgl, &ifp->if_groups, ifgl_next)
CK_STAILQ_FOREACH(ifgl, &ifp->if_groups, ifgl_next)
if (!strcmp(ifgl->ifgl_group->ifg_group, groupname)) {
IFNET_WUNLOCK();
return (EEXIST);
@ -1419,7 +1436,7 @@ if_addgroup(struct ifnet *ifp, const char *groupname)
return (ENOMEM);
}
TAILQ_FOREACH(ifg, &V_ifg_head, ifg_next)
CK_STAILQ_FOREACH(ifg, &V_ifg_head, ifg_next)
if (!strcmp(ifg->ifg_group, groupname))
break;
@ -1433,8 +1450,8 @@ if_addgroup(struct ifnet *ifp, const char *groupname)
}
strlcpy(ifg->ifg_group, groupname, sizeof(ifg->ifg_group));
ifg->ifg_refcnt = 0;
TAILQ_INIT(&ifg->ifg_members);
TAILQ_INSERT_TAIL(&V_ifg_head, ifg, ifg_next);
CK_STAILQ_INIT(&ifg->ifg_members);
CK_STAILQ_INSERT_TAIL(&V_ifg_head, ifg, ifg_next);
new = 1;
}
@ -1443,8 +1460,8 @@ if_addgroup(struct ifnet *ifp, const char *groupname)
ifgm->ifgm_ifp = ifp;
IF_ADDR_WLOCK(ifp);
TAILQ_INSERT_TAIL(&ifg->ifg_members, ifgm, ifgm_next);
TAILQ_INSERT_TAIL(&ifp->if_groups, ifgl, ifgl_next);
CK_STAILQ_INSERT_TAIL(&ifg->ifg_members, ifgm, ifgm_next);
CK_STAILQ_INSERT_TAIL(&ifp->if_groups, ifgl, ifgl_next);
IF_ADDR_WUNLOCK(ifp);
IFNET_WUNLOCK();
@ -1464,9 +1481,10 @@ if_delgroup(struct ifnet *ifp, const char *groupname)
{
struct ifg_list *ifgl;
struct ifg_member *ifgm;
int freeifgl;
IFNET_WLOCK();
TAILQ_FOREACH(ifgl, &ifp->if_groups, ifgl_next)
CK_STAILQ_FOREACH(ifgl, &ifp->if_groups, ifgl_next)
if (!strcmp(ifgl->ifgl_group->ifg_group, groupname))
break;
if (ifgl == NULL) {
@ -1474,27 +1492,30 @@ if_delgroup(struct ifnet *ifp, const char *groupname)
return (ENOENT);
}
freeifgl = 0;
IF_ADDR_WLOCK(ifp);
TAILQ_REMOVE(&ifp->if_groups, ifgl, ifgl_next);
CK_STAILQ_REMOVE(&ifp->if_groups, ifgl, ifg_list, ifgl_next);
IF_ADDR_WUNLOCK(ifp);
TAILQ_FOREACH(ifgm, &ifgl->ifgl_group->ifg_members, ifgm_next)
CK_STAILQ_FOREACH(ifgm, &ifgl->ifgl_group->ifg_members, ifgm_next)
if (ifgm->ifgm_ifp == ifp)
break;
if (ifgm != NULL) {
TAILQ_REMOVE(&ifgl->ifgl_group->ifg_members, ifgm, ifgm_next);
free(ifgm, M_TEMP);
}
if (ifgm != NULL)
CK_STAILQ_REMOVE(&ifgl->ifgl_group->ifg_members, ifgm, ifg_member, ifgm_next);
if (--ifgl->ifgl_group->ifg_refcnt == 0) {
TAILQ_REMOVE(&V_ifg_head, ifgl->ifgl_group, ifg_next);
IFNET_WUNLOCK();
CK_STAILQ_REMOVE(&V_ifg_head, ifgl->ifgl_group, ifg_group, ifg_next);
freeifgl = 1;
}
IFNET_WUNLOCK();
epoch_wait_preempt(net_epoch_preempt);
if (freeifgl) {
EVENTHANDLER_INVOKE(group_detach_event, ifgl->ifgl_group);
free(ifgl->ifgl_group, M_TEMP);
} else
IFNET_WUNLOCK();
}
free(ifgm, M_TEMP);
free(ifgl, M_TEMP);
EVENTHANDLER_INVOKE(group_change_event, groupname);
@ -1511,38 +1532,38 @@ if_delgroups(struct ifnet *ifp)
struct ifg_list *ifgl;
struct ifg_member *ifgm;
char groupname[IFNAMSIZ];
int ifglfree;
IFNET_WLOCK();
while (!TAILQ_EMPTY(&ifp->if_groups)) {
ifgl = TAILQ_FIRST(&ifp->if_groups);
while (!CK_STAILQ_EMPTY(&ifp->if_groups)) {
ifgl = CK_STAILQ_FIRST(&ifp->if_groups);
strlcpy(groupname, ifgl->ifgl_group->ifg_group, IFNAMSIZ);
IF_ADDR_WLOCK(ifp);
TAILQ_REMOVE(&ifp->if_groups, ifgl, ifgl_next);
CK_STAILQ_REMOVE(&ifp->if_groups, ifgl, ifg_list, ifgl_next);
IF_ADDR_WUNLOCK(ifp);
TAILQ_FOREACH(ifgm, &ifgl->ifgl_group->ifg_members, ifgm_next)
CK_STAILQ_FOREACH(ifgm, &ifgl->ifgl_group->ifg_members, ifgm_next)
if (ifgm->ifgm_ifp == ifp)
break;
if (ifgm != NULL) {
TAILQ_REMOVE(&ifgl->ifgl_group->ifg_members, ifgm,
if (ifgm != NULL)
CK_STAILQ_REMOVE(&ifgl->ifgl_group->ifg_members, ifgm, ifg_member,
ifgm_next);
free(ifgm, M_TEMP);
ifglfree = 0;
if (--ifgl->ifgl_group->ifg_refcnt == 0) {
CK_STAILQ_REMOVE(&V_ifg_head, ifgl->ifgl_group, ifg_group, ifg_next);
ifglfree = 1;
}
if (--ifgl->ifgl_group->ifg_refcnt == 0) {
TAILQ_REMOVE(&V_ifg_head, ifgl->ifgl_group, ifg_next);
IFNET_WUNLOCK();
epoch_wait_preempt(net_epoch_preempt);
free(ifgm, M_TEMP);
if (ifglfree) {
EVENTHANDLER_INVOKE(group_detach_event,
ifgl->ifgl_group);
ifgl->ifgl_group);
free(ifgl->ifgl_group, M_TEMP);
} else
IFNET_WUNLOCK();
free(ifgl, M_TEMP);
}
EVENTHANDLER_INVOKE(group_change_event, groupname);
IFNET_WLOCK();
@ -1589,7 +1610,7 @@ if_getgroup(struct ifgroupreq *ifgr, struct ifnet *ifp)
if (ifgr->ifgr_len == 0) {
IF_ADDR_RLOCK(ifp);
TAILQ_FOREACH(ifgl, &ifp->if_groups, ifgl_next)
CK_STAILQ_FOREACH(ifgl, &ifp->if_groups, ifgl_next)
ifgr->ifgr_len += sizeof(struct ifg_req);
IF_ADDR_RUNLOCK(ifp);
return (0);
@ -1599,7 +1620,7 @@ if_getgroup(struct ifgroupreq *ifgr, struct ifnet *ifp)
ifgp = ifgr_groups_get(ifgr);
/* XXX: wire */
IF_ADDR_RLOCK(ifp);
TAILQ_FOREACH(ifgl, &ifp->if_groups, ifgl_next) {
CK_STAILQ_FOREACH(ifgl, &ifp->if_groups, ifgl_next) {
if (len < sizeof(ifgrq)) {
IF_ADDR_RUNLOCK(ifp);
return (EINVAL);
@ -1631,7 +1652,7 @@ if_getgroupmembers(struct ifgroupreq *ifgr)
int len, error;
IFNET_RLOCK();
TAILQ_FOREACH(ifg, &V_ifg_head, ifg_next)
CK_STAILQ_FOREACH(ifg, &V_ifg_head, ifg_next)
if (!strcmp(ifg->ifg_group, ifgr->ifgr_name))
break;
if (ifg == NULL) {
@ -1640,7 +1661,7 @@ if_getgroupmembers(struct ifgroupreq *ifgr)
}
if (ifgr->ifgr_len == 0) {
TAILQ_FOREACH(ifgm, &ifg->ifg_members, ifgm_next)
CK_STAILQ_FOREACH(ifgm, &ifg->ifg_members, ifgm_next)
ifgr->ifgr_len += sizeof(ifgrq);
IFNET_RUNLOCK();
return (0);
@ -1648,7 +1669,7 @@ if_getgroupmembers(struct ifgroupreq *ifgr)
len = ifgr->ifgr_len;
ifgp = ifgr_groups_get(ifgr);
TAILQ_FOREACH(ifgm, &ifg->ifg_members, ifgm_next) {
CK_STAILQ_FOREACH(ifgm, &ifg->ifg_members, ifgm_next) {
if (len < sizeof(ifgrq)) {
IFNET_RUNLOCK();
return (EINVAL);
@ -1897,22 +1918,18 @@ ifa_switch_loopback_route(struct ifaddr *ifa, struct sockaddr *ia)
* Locate an interface based on a complete address.
*/
/*ARGSUSED*/
static struct ifaddr *
ifa_ifwithaddr_internal(const struct sockaddr *addr, int getref)
struct ifaddr *
ifa_ifwithaddr(const struct sockaddr *addr)
{
struct ifnet *ifp;
struct ifaddr *ifa;
IFNET_RLOCK_NOSLEEP();
TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
IF_ADDR_RLOCK(ifp);
MPASS(in_epoch());
CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) {
CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
if (ifa->ifa_addr->sa_family != addr->sa_family)
continue;
if (sa_equal(addr, ifa->ifa_addr)) {
if (getref)
ifa_ref(ifa);
IF_ADDR_RUNLOCK(ifp);
goto done;
}
/* IP6 doesn't have broadcast */
@ -1920,32 +1937,24 @@ ifa_ifwithaddr_internal(const struct sockaddr *addr, int getref)
ifa->ifa_broadaddr &&
ifa->ifa_broadaddr->sa_len != 0 &&
sa_equal(ifa->ifa_broadaddr, addr)) {
if (getref)
ifa_ref(ifa);
IF_ADDR_RUNLOCK(ifp);
goto done;
}
}
IF_ADDR_RUNLOCK(ifp);
}
ifa = NULL;
done:
IFNET_RUNLOCK_NOSLEEP();
return (ifa);
}
struct ifaddr *
ifa_ifwithaddr(const struct sockaddr *addr)
{
return (ifa_ifwithaddr_internal(addr, 1));
}
int
ifa_ifwithaddr_check(const struct sockaddr *addr)
{
int rc;
return (ifa_ifwithaddr_internal(addr, 0) != NULL);
NET_EPOCH_ENTER();
rc = (ifa_ifwithaddr(addr) != NULL);
NET_EPOCH_EXIT();
return (rc);
}
/*
@ -1958,11 +1967,10 @@ ifa_ifwithbroadaddr(const struct sockaddr *addr, int fibnum)
struct ifnet *ifp;
struct ifaddr *ifa;
IFNET_RLOCK_NOSLEEP();
TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
MPASS(in_epoch());
CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) {
if ((fibnum != RT_ALL_FIBS) && (ifp->if_fib != fibnum))
continue;
IF_ADDR_RLOCK(ifp);
CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
if (ifa->ifa_addr->sa_family != addr->sa_family)
continue;
@ -1970,16 +1978,12 @@ ifa_ifwithbroadaddr(const struct sockaddr *addr, int fibnum)
ifa->ifa_broadaddr &&
ifa->ifa_broadaddr->sa_len != 0 &&
sa_equal(ifa->ifa_broadaddr, addr)) {
ifa_ref(ifa);
IF_ADDR_RUNLOCK(ifp);
goto done;
}
}
IF_ADDR_RUNLOCK(ifp);
}
ifa = NULL;
done:
IFNET_RUNLOCK_NOSLEEP();
return (ifa);
}
@ -1993,28 +1997,23 @@ ifa_ifwithdstaddr(const struct sockaddr *addr, int fibnum)
struct ifnet *ifp;
struct ifaddr *ifa;
IFNET_RLOCK_NOSLEEP();
TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
MPASS(in_epoch());
CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) {
if ((ifp->if_flags & IFF_POINTOPOINT) == 0)
continue;
if ((fibnum != RT_ALL_FIBS) && (ifp->if_fib != fibnum))
continue;
IF_ADDR_RLOCK(ifp);
CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
if (ifa->ifa_addr->sa_family != addr->sa_family)
continue;
if (ifa->ifa_dstaddr != NULL &&
sa_equal(addr, ifa->ifa_dstaddr)) {
ifa_ref(ifa);
IF_ADDR_RUNLOCK(ifp);
goto done;
}
}
IF_ADDR_RUNLOCK(ifp);
}
ifa = NULL;
done:
IFNET_RUNLOCK_NOSLEEP();
return (ifa);
}
@ -2031,6 +2030,7 @@ ifa_ifwithnet(const struct sockaddr *addr, int ignore_ptp, int fibnum)
u_int af = addr->sa_family;
const char *addr_data = addr->sa_data, *cplim;
MPASS(in_epoch());
/*
* AF_LINK addresses can be looked up directly by their index number,
* so do that if we can.
@ -2047,11 +2047,9 @@ ifa_ifwithnet(const struct sockaddr *addr, int ignore_ptp, int fibnum)
* on ifa_maybe once we find one, as we release the IF_ADDR_RLOCK() that
* kept it stable when we move onto the next interface.
*/
IFNET_RLOCK_NOSLEEP();
TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) {
if ((fibnum != RT_ALL_FIBS) && (ifp->if_fib != fibnum))
continue;
IF_ADDR_RLOCK(ifp);
CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
const char *cp, *cp2, *cp3;
@ -2069,7 +2067,6 @@ next: continue;
*/
if (ifa->ifa_dstaddr != NULL &&
sa_equal(addr, ifa->ifa_dstaddr)) {
ifa_ref(ifa);
IF_ADDR_RUNLOCK(ifp);
goto done;
}
@ -2103,21 +2100,14 @@ next: continue;
ifa_preferred(ifa_maybe, ifa) ||
rn_refines((caddr_t)ifa->ifa_netmask,
(caddr_t)ifa_maybe->ifa_netmask)) {
if (ifa_maybe != NULL)
ifa_free(ifa_maybe);
ifa_maybe = ifa;
ifa_ref(ifa_maybe);
}
}
}
IF_ADDR_RUNLOCK(ifp);
}
ifa = ifa_maybe;
ifa_maybe = NULL;
done:
IFNET_RUNLOCK_NOSLEEP();
if (ifa_maybe != NULL)
ifa_free(ifa_maybe);
return (ifa);
}
@ -2136,7 +2126,7 @@ ifaof_ifpforaddr(const struct sockaddr *addr, struct ifnet *ifp)
if (af >= AF_MAX)
return (NULL);
IF_ADDR_RLOCK(ifp);
MPASS(in_epoch());
CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
if (ifa->ifa_addr->sa_family != af)
continue;
@ -2166,9 +2156,6 @@ ifaof_ifpforaddr(const struct sockaddr *addr, struct ifnet *ifp)
}
ifa = ifa_maybe;
done:
if (ifa != NULL)
ifa_ref(ifa);
IF_ADDR_RUNLOCK(ifp);
return (ifa);
}
@ -2204,14 +2191,15 @@ link_rtrequest(int cmd, struct rtentry *rt, struct rt_addrinfo *info)
if (cmd != RTM_ADD || ((ifa = rt->rt_ifa) == NULL) ||
((ifp = ifa->ifa_ifp) == NULL) || ((dst = rt_key(rt)) == NULL))
return;
NET_EPOCH_ENTER();
ifa = ifaof_ifpforaddr(dst, ifp);
if (ifa) {
oifa = rt->rt_ifa;
rt->rt_ifa = ifa;
ifa_free(oifa);
if (ifa->ifa_rtrequest && ifa->ifa_rtrequest != link_rtrequest)
ifa->ifa_rtrequest(cmd, rt, info);
}
NET_EPOCH_EXIT();
}
struct sockaddr_dl *
@ -2414,7 +2402,7 @@ ifunit_ref(const char *name)
struct ifnet *ifp;
IFNET_RLOCK_NOSLEEP();
TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) {
if (strncmp(name, ifp->if_xname, IFNAMSIZ) == 0 &&
!(ifp->if_flags & IFF_DYING))
break;
@ -2431,7 +2419,7 @@ ifunit(const char *name)
struct ifnet *ifp;
IFNET_RLOCK_NOSLEEP();
TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) {
if (strncmp(name, ifp->if_xname, IFNAMSIZ) == 0)
break;
}
@ -3256,7 +3244,7 @@ ifconf(u_long cmd, caddr_t data)
valid_len = 0;
IFNET_RLOCK();
TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) {
int addrs;
/*
@ -3609,7 +3597,7 @@ if_delmulti(struct ifnet *ifp, struct sockaddr *sa)
struct ifnet *oifp;
IFNET_RLOCK_NOSLEEP();
TAILQ_FOREACH(oifp, &V_ifnet, if_link)
CK_STAILQ_FOREACH(oifp, &V_ifnet, if_link)
if (ifp == oifp)
break;
if (ifp != oifp)
@ -3683,7 +3671,7 @@ if_delmulti_ifma_flags(struct ifmultiaddr *ifma, int flags)
struct ifnet *oifp;
IFNET_RLOCK_NOSLEEP();
TAILQ_FOREACH(oifp, &V_ifnet, if_link)
CK_STAILQ_FOREACH(oifp, &V_ifnet, if_link)
if (ifp == oifp)
break;
if (ifp != oifp) {
@ -3810,23 +3798,24 @@ if_setlladdr(struct ifnet *ifp, const u_char *lladdr, int len)
struct sockaddr_dl *sdl;
struct ifaddr *ifa;
struct ifreq ifr;
int rc;
IF_ADDR_RLOCK(ifp);
rc = 0;
NET_EPOCH_ENTER();
ifa = ifp->if_addr;
if (ifa == NULL) {
IF_ADDR_RUNLOCK(ifp);
return (EINVAL);
rc = EINVAL;
goto out;
}
ifa_ref(ifa);
IF_ADDR_RUNLOCK(ifp);
sdl = (struct sockaddr_dl *)ifa->ifa_addr;
if (sdl == NULL) {
ifa_free(ifa);
return (EINVAL);
rc = EINVAL;
goto out;
}
if (len != sdl->sdl_alen) { /* don't allow length to change */
ifa_free(ifa);
return (EINVAL);
rc = EINVAL;
goto out;
}
switch (ifp->if_type) {
case IFT_ETHER:
@ -3835,11 +3824,10 @@ if_setlladdr(struct ifnet *ifp, const u_char *lladdr, int len)
case IFT_BRIDGE:
case IFT_IEEE8023ADLAG:
bcopy(lladdr, LLADDR(sdl), len);
ifa_free(ifa);
break;
default:
ifa_free(ifa);
return (ENODEV);
rc = ENODEV;
goto out;
}
/*
@ -3860,7 +3848,9 @@ if_setlladdr(struct ifnet *ifp, const u_char *lladdr, int len)
}
}
EVENTHANDLER_INVOKE(iflladdr_event, ifp);
return (0);
out:
NET_EPOCH_EXIT();
return (rc);
}
/*

16
sys/net/if_llatbl.c
View file

@ -146,7 +146,7 @@ htable_foreach_lle(struct lltable *llt, llt_foreach_cb_t *f, void *farg)
error = 0;
for (i = 0; i < llt->llt_hsize; i++) {
LIST_FOREACH_SAFE(lle, &llt->lle_head[i], lle_next, next) {
CK_LIST_FOREACH_SAFE(lle, &llt->lle_head[i], lle_next, next) {
error = f(llt, lle, farg);
if (error != 0)
break;
@ -173,7 +173,7 @@ htable_link_entry(struct lltable *llt, struct llentry *lle)
lle->lle_tbl = llt;
lle->lle_head = lleh;
lle->la_flags |= LLE_LINKED;
LIST_INSERT_HEAD(lleh, lle, lle_next);
CK_LIST_INSERT_HEAD(lleh, lle, lle_next);
}
static void
@ -182,7 +182,7 @@ htable_unlink_entry(struct llentry *lle)
if ((lle->la_flags & LLE_LINKED) != 0) {
IF_AFDATA_WLOCK_ASSERT(lle->lle_tbl->llt_ifp);
LIST_REMOVE(lle, lle_next);
CK_LIST_REMOVE(lle, lle_next);
lle->la_flags &= ~(LLE_VALID | LLE_LINKED);
#if 0
lle->lle_tbl = NULL;
@ -224,7 +224,7 @@ htable_prefix_free(struct lltable *llt, const struct sockaddr *addr,
pmd.addr = addr;
pmd.mask = mask;
pmd.flags = flags;
LIST_INIT(&pmd.dchain);
CK_LIST_INIT(&pmd.dchain);
IF_AFDATA_WLOCK(llt->llt_ifp);
/* Push matching lles to chain */
@ -514,7 +514,7 @@ lltable_free(struct lltable *llt)
lltable_unlink(llt);
LIST_INIT(&dchain);
CK_LIST_INIT(&dchain);
IF_AFDATA_WLOCK(llt->llt_ifp);
/* Push all lles to @dchain */
lltable_foreach_lle(llt, lltable_free_cb, &dchain);
@ -544,7 +544,7 @@ lltable_drain(int af)
continue;
for (i=0; i < llt->llt_hsize; i++) {
LIST_FOREACH(lle, &llt->lle_head[i], lle_next) {
CK_LIST_FOREACH(lle, &llt->lle_head[i], lle_next) {
LLE_WLOCK(lle);
if (lle->la_hold) {
m_freem(lle->la_hold);
@ -620,7 +620,7 @@ lltable_allocate_htbl(uint32_t hsize)
M_LLTABLE, M_WAITOK | M_ZERO);
for (i = 0; i < llt->llt_hsize; i++)
LIST_INIT(&llt->lle_head[i]);
CK_LIST_INIT(&llt->lle_head[i]);
/* Set some default callbacks */
llt->llt_link_entry = htable_link_entry;
@ -917,7 +917,7 @@ llatbl_llt_show(struct lltable *llt)
llt, llt->llt_af, llt->llt_ifp);
for (i = 0; i < llt->llt_hsize; i++) {
LIST_FOREACH(lle, &llt->lle_head[i], lle_next) {
CK_LIST_FOREACH(lle, &llt->lle_head[i], lle_next) {
llatbl_lle_show((struct llentry_sa *)lle);
if (db_pager_quit)

2
sys/net/if_llatbl.h
View file

@ -34,6 +34,7 @@ __FBSDID("$FreeBSD$");
#include <sys/_rwlock.h>
#include <netinet/in.h>
#include <sys/epoch.h>
struct ifnet;
struct sysctl_req;
@ -80,6 +81,7 @@ struct llentry {
struct callout lle_timer;
struct rwlock lle_lock;
struct mtx req_mtx;
struct epoch_context lle_epoch_ctx;
};
#define LLE_WLOCK(lle) rw_wlock(&(lle)->lle_lock)

50
sys/net/if_var.h
View file

@ -89,10 +89,10 @@ struct netdump_methods;
#include <net/altq/if_altq.h>
TAILQ_HEAD(ifnethead, ifnet); /* we use TAILQs so that the order of */
CK_STAILQ_HEAD(ifnethead, ifnet); /* we use TAILQs so that the order of */
CK_STAILQ_HEAD(ifaddrhead, ifaddr); /* instantiation is preserved in the list */
CK_STAILQ_HEAD(ifmultihead, ifmultiaddr);
TAILQ_HEAD(ifgrouphead, ifg_group);
CK_STAILQ_HEAD(ifgrouphead, ifg_group);
#ifdef _KERNEL
VNET_DECLARE(struct pfil_head, link_pfil_hook); /* packet filter hooks */
@ -238,9 +238,9 @@ typedef void (if_snd_tag_free_t)(struct m_snd_tag *);
*/
struct ifnet {
/* General book keeping of interface lists. */
TAILQ_ENTRY(ifnet) if_link; /* all struct ifnets are chained */
STAILQ_ENTRY(ifnet) if_link; /* all struct ifnets are chained (CK_) */
LIST_ENTRY(ifnet) if_clones; /* interfaces of a cloner */
TAILQ_HEAD(, ifg_list) if_groups; /* linked list of groups per if */
STAILQ_HEAD(, ifg_list) if_groups; /* linked list of groups per if (CK_) */
/* protected by if_addr_lock */
u_char if_alloctype; /* if_type at time of allocation */
@ -297,7 +297,7 @@ struct ifnet {
struct ifaddr *if_addr; /* pointer to link-level address */
void *if_hw_addr; /* hardware link-level address */
const u_int8_t *if_broadcastaddr; /* linklevel broadcast bytestring */
struct rwlock if_afdata_lock;
struct mtx if_afdata_lock;
void *if_afdata[AF_MAX];
int if_afdata_initialized;
@ -380,6 +380,7 @@ struct ifnet {
* Netdump hooks to be called while dumping.
*/
struct netdump_methods *if_netdump_methods;
struct epoch_context if_epoch_ctx;
/*
* Spare fields to be added before branching a stable branch, so
@ -404,6 +405,9 @@ struct ifnet {
#define IF_ADDR_WUNLOCK(if) mtx_unlock(&(if)->if_addr_lock)
#define IF_ADDR_LOCK_ASSERT(if) MPASS(in_epoch() || mtx_owned(&(if)->if_addr_lock))
#define IF_ADDR_WLOCK_ASSERT(if) mtx_assert(&(if)->if_addr_lock, MA_OWNED)
#define NET_EPOCH_ENTER() epoch_enter_preempt(net_epoch_preempt)
#define NET_EPOCH_EXIT() epoch_exit_preempt(net_epoch_preempt)
/*
* Function variations on locking macros intended to be used by loadable
@ -448,18 +452,18 @@ struct ifg_group {
char ifg_group[IFNAMSIZ];
u_int ifg_refcnt;
void *ifg_pf_kif;
TAILQ_HEAD(, ifg_member) ifg_members;
TAILQ_ENTRY(ifg_group) ifg_next;
STAILQ_HEAD(, ifg_member) ifg_members; /* (CK_) */
STAILQ_ENTRY(ifg_group) ifg_next; /* (CK_) */
};
struct ifg_member {
TAILQ_ENTRY(ifg_member) ifgm_next;
STAILQ_ENTRY(ifg_member) ifgm_next; /* (CK_) */
struct ifnet *ifgm_ifp;
};
struct ifg_list {
struct ifg_group *ifgl_group;
TAILQ_ENTRY(ifg_list) ifgl_next;
STAILQ_ENTRY(ifg_list) ifgl_next; /* (CK_) */
};
#ifdef _SYS_EVENTHANDLER_H_
@ -475,21 +479,21 @@ EVENTHANDLER_DECLARE(group_change_event, group_change_event_handler_t);
#endif /* _SYS_EVENTHANDLER_H_ */
#define IF_AFDATA_LOCK_INIT(ifp) \
rw_init(&(ifp)->if_afdata_lock, "if_afdata")
mtx_init(&(ifp)->if_afdata_lock, "if_afdata", NULL, MTX_DEF)
#define IF_AFDATA_WLOCK(ifp) rw_wlock(&(ifp)->if_afdata_lock)
#define IF_AFDATA_RLOCK(ifp) rw_rlock(&(ifp)->if_afdata_lock)
#define IF_AFDATA_WUNLOCK(ifp) rw_wunlock(&(ifp)->if_afdata_lock)
#define IF_AFDATA_RUNLOCK(ifp) rw_runlock(&(ifp)->if_afdata_lock)
#define IF_AFDATA_WLOCK(ifp) mtx_lock(&(ifp)->if_afdata_lock)
#define IF_AFDATA_RLOCK(ifp) epoch_enter_preempt(net_epoch_preempt)
#define IF_AFDATA_WUNLOCK(ifp) mtx_unlock(&(ifp)->if_afdata_lock)
#define IF_AFDATA_RUNLOCK(ifp) epoch_exit_preempt(net_epoch_preempt)
#define IF_AFDATA_LOCK(ifp) IF_AFDATA_WLOCK(ifp)
#define IF_AFDATA_UNLOCK(ifp) IF_AFDATA_WUNLOCK(ifp)
#define IF_AFDATA_TRYLOCK(ifp) rw_try_wlock(&(ifp)->if_afdata_lock)
#define IF_AFDATA_DESTROY(ifp) rw_destroy(&(ifp)->if_afdata_lock)
#define IF_AFDATA_TRYLOCK(ifp) mtx_trylock(&(ifp)->if_afdata_lock)
#define IF_AFDATA_DESTROY(ifp) mtx_destroy(&(ifp)->if_afdata_lock)
#define IF_AFDATA_LOCK_ASSERT(ifp) rw_assert(&(ifp)->if_afdata_lock, RA_LOCKED)
#define IF_AFDATA_RLOCK_ASSERT(ifp) rw_assert(&(ifp)->if_afdata_lock, RA_RLOCKED)
#define IF_AFDATA_WLOCK_ASSERT(ifp) rw_assert(&(ifp)->if_afdata_lock, RA_WLOCKED)
#define IF_AFDATA_UNLOCK_ASSERT(ifp) rw_assert(&(ifp)->if_afdata_lock, RA_UNLOCKED)
#define IF_AFDATA_LOCK_ASSERT(ifp) MPASS(in_epoch() || mtx_owned(&(ifp)->if_afdata_lock))
#define IF_AFDATA_RLOCK_ASSERT(ifp) MPASS(in_epoch());
#define IF_AFDATA_WLOCK_ASSERT(ifp) mtx_assert(&(ifp)->if_afdata_lock, MA_OWNED)
#define IF_AFDATA_UNLOCK_ASSERT(ifp) mtx_assert(&(ifp)->if_afdata_lock, MA_NOTOWNED)
/*
* 72 was chosen below because it is the size of a TCP/IP
@ -569,16 +573,16 @@ extern struct sx ifnet_sxlock;
* write, but also whether it was acquired with sleep support or not.
*/
#define IFNET_RLOCK_ASSERT() sx_assert(&ifnet_sxlock, SA_SLOCKED)
#define IFNET_RLOCK_NOSLEEP_ASSERT() rw_assert(&ifnet_rwlock, RA_RLOCKED)
#define IFNET_RLOCK_NOSLEEP_ASSERT() MPASS(in_epoch())
#define IFNET_WLOCK_ASSERT() do { \
sx_assert(&ifnet_sxlock, SA_XLOCKED); \
rw_assert(&ifnet_rwlock, RA_WLOCKED); \
} while (0)
#define IFNET_RLOCK() sx_slock(&ifnet_sxlock)
#define IFNET_RLOCK_NOSLEEP() rw_rlock(&ifnet_rwlock)
#define IFNET_RLOCK_NOSLEEP() epoch_enter_preempt(net_epoch_preempt)
#define IFNET_RUNLOCK() sx_sunlock(&ifnet_sxlock)
#define IFNET_RUNLOCK_NOSLEEP() rw_runlock(&ifnet_rwlock)
#define IFNET_RUNLOCK_NOSLEEP() epoch_exit_preempt(net_epoch_preempt)
/*
* Look up an ifnet given its index; the _ref variant also acquires a

20
sys/net/route.c
View file

@ -594,12 +594,12 @@ rtredirect_fib(struct sockaddr *dst,
struct rib_head *rnh;
ifa = NULL;
NET_EPOCH_ENTER();
rnh = rt_tables_get_rnh(fibnum, dst->sa_family);
if (rnh == NULL) {
error = EAFNOSUPPORT;
goto out;
}
/* verify the gateway is directly reachable */
if ((ifa = ifa_ifwithnet(gateway, 0, fibnum)) == NULL) {
error = ENETUNREACH;
@ -687,7 +687,8 @@ rtredirect_fib(struct sockaddr *dst,
done:
if (rt)
RTFREE_LOCKED(rt);
out:
out:
NET_EPOCH_EXIT();
if (error)
V_rtstat.rts_badredirect++;
else if (stat != NULL)
@ -698,8 +699,6 @@ rtredirect_fib(struct sockaddr *dst,
info.rti_info[RTAX_NETMASK] = netmask;
info.rti_info[RTAX_AUTHOR] = src;
rt_missmsg_fib(RTM_REDIRECT, &info, flags, error, fibnum);
if (ifa != NULL)
ifa_free(ifa);
}
/*
@ -730,6 +729,7 @@ ifa_ifwithroute(int flags, const struct sockaddr *dst, struct sockaddr *gateway,
struct ifaddr *ifa;
int not_found = 0;
MPASS(in_epoch());
if ((flags & RTF_GATEWAY) == 0) {
/*
* If we are adding a route to an interface,
@ -758,7 +758,7 @@ ifa_ifwithroute(int flags, const struct sockaddr *dst, struct sockaddr *gateway,
rt = rtalloc1_fib(gateway, 0, flags, fibnum);
if (rt == NULL)
return (NULL);
goto out;
/*
* dismiss a gateway that is reachable only
* through the default router
@ -777,21 +777,19 @@ ifa_ifwithroute(int flags, const struct sockaddr *dst, struct sockaddr *gateway,
}
if (!not_found && rt->rt_ifa != NULL) {
ifa = rt->rt_ifa;
ifa_ref(ifa);
}
RT_REMREF(rt);
RT_UNLOCK(rt);
if (not_found || ifa == NULL)
return (NULL);
goto out;
}
if (ifa->ifa_addr->sa_family != dst->sa_family) {
struct ifaddr *oifa = ifa;
ifa = ifaof_ifpforaddr(dst, ifa->ifa_ifp);
if (ifa == NULL)
ifa = oifa;
else
ifa_free(oifa);
}
out:
return (ifa);
}
@ -1283,11 +1281,11 @@ rt_getifa_fib(struct rt_addrinfo *info, u_int fibnum)
* ifp may be specified by sockaddr_dl
* when protocol address is ambiguous.
*/
NET_EPOCH_ENTER();
if (info->rti_ifp == NULL && ifpaddr != NULL &&
ifpaddr->sa_family == AF_LINK &&
(ifa = ifa_ifwithnet(ifpaddr, 0, fibnum)) != NULL) {
info->rti_ifp = ifa->ifa_ifp;
ifa_free(ifa);
}
if (info->rti_ifa == NULL && ifaaddr != NULL)
info->rti_ifa = ifa_ifwithaddr(ifaaddr);
@ -1308,8 +1306,10 @@ rt_getifa_fib(struct rt_addrinfo *info, u_int fibnum)
if ((ifa = info->rti_ifa) != NULL) {
if (info->rti_ifp == NULL)
info->rti_ifp = ifa->ifa_ifp;
ifa_ref(info->rti_ifa);
} else
error = ENETUNREACH;
NET_EPOCH_EXIT();
return (error);
}

6
sys/net/rtsock.c
View file

@ -784,12 +784,14 @@ route_output(struct mbuf *m, struct socket *so, ...)
rt->rt_ifp->if_type == IFT_PROPVIRTUAL) {
struct ifaddr *ifa;
NET_EPOCH_ENTER();
ifa = ifa_ifwithnet(info.rti_info[RTAX_DST], 1,
RT_ALL_FIBS);
if (ifa != NULL)
rt_maskedcopy(ifa->ifa_addr,
&laddr,
ifa->ifa_netmask);
NET_EPOCH_EXIT();
} else
rt_maskedcopy(rt->rt_ifa->ifa_addr,
&laddr,
@ -1736,7 +1738,7 @@ sysctl_iflist(int af, struct walkarg *w)
bzero((caddr_t)&info, sizeof(info));
bzero(&ifd, sizeof(ifd));
IFNET_RLOCK_NOSLEEP();
TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) {
if (w->w_arg && w->w_arg != ifp->if_index)
continue;
if_data_copy(ifp, &ifd);
@ -1806,7 +1808,7 @@ sysctl_ifmalist(int af, struct walkarg *w)
bzero((caddr_t)&info, sizeof(info));
IFNET_RLOCK_NOSLEEP();
TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) {
if (w->w_arg && w->w_arg != ifp->if_index)
continue;
ifa = ifp->if_addr;

24
sys/netinet/igmp.c
View file

@ -1218,11 +1218,11 @@ igmp_input_v1_report(struct ifnet *ifp, /*const*/ struct ip *ip,
* Replace 0.0.0.0 with the subnet address if told to do so.
*/
if (V_igmp_recvifkludge && in_nullhost(ip->ip_src)) {
NET_EPOCH_ENTER();
IFP_TO_IA(ifp, ia, &in_ifa_tracker);
if (ia != NULL) {
if (ia != NULL)
ip->ip_src.s_addr = htonl(ia->ia_subnet);
ifa_free(&ia->ia_ifa);
}
NET_EPOCH_EXIT();
}
CTR3(KTR_IGMPV3, "process v1 report 0x%08x on ifp %p(%s)",
@ -1315,24 +1315,23 @@ igmp_input_v2_report(struct ifnet *ifp, /*const*/ struct ip *ip,
* leave requires knowing that we are the only member of a
* group.
*/
NET_EPOCH_ENTER();
IFP_TO_IA(ifp, ia, &in_ifa_tracker);
if (ia != NULL && in_hosteq(ip->ip_src, IA_SIN(ia)->sin_addr)) {
ifa_free(&ia->ia_ifa);
NET_EPOCH_EXIT();
return (0);
}
IGMPSTAT_INC(igps_rcv_reports);
if (ifp->if_flags & IFF_LOOPBACK) {
if (ia != NULL)
ifa_free(&ia->ia_ifa);
NET_EPOCH_EXIT();
return (0);
}
if (!IN_MULTICAST(ntohl(igmp->igmp_group.s_addr)) ||
!in_hosteq(igmp->igmp_group, ip->ip_dst)) {
if (ia != NULL)
ifa_free(&ia->ia_ifa);
NET_EPOCH_EXIT();
IGMPSTAT_INC(igps_rcv_badreports);
return (EINVAL);
}
@ -1348,8 +1347,7 @@ igmp_input_v2_report(struct ifnet *ifp, /*const*/ struct ip *ip,
if (ia != NULL)
ip->ip_src.s_addr = htonl(ia->ia_subnet);
}
if (ia != NULL)
ifa_free(&ia->ia_ifa);
NET_EPOCH_EXIT();
CTR3(KTR_IGMPV3, "process v2 report 0x%08x on ifp %p(%s)",
ntohl(igmp->igmp_group.s_addr), ifp, ifp->if_xname);
@ -3526,11 +3524,11 @@ igmp_v3_encap_report(struct ifnet *ifp, struct mbuf *m)
if (m->m_flags & M_IGMP_LOOP) {
struct in_ifaddr *ia;
NET_EPOCH_ENTER();
IFP_TO_IA(ifp, ia, &in_ifa_tracker);
if (ia != NULL) {
if (ia != NULL)
ip->ip_src = ia->ia_addr.sin_addr;
ifa_free(&ia->ia_ifa);
}
NET_EPOCH_EXIT();
}
ip->ip_dst.s_addr = htonl(INADDR_ALLRPTS_GROUP);

12
sys/netinet/in.c
View file

@ -912,7 +912,7 @@ in_ifscrub_all(void)
struct ifaliasreq ifr;
IFNET_RLOCK();
TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) {
/* Cannot lock here - lock recursion. */
/* IF_ADDR_RLOCK(ifp); */
CK_STAILQ_FOREACH_SAFE(ifa, &ifp->if_addrhead, ifa_link, nifa) {
@ -1055,9 +1055,11 @@ struct in_llentry {
* Do actual deallocation of @lle.
*/
static void
in_lltable_destroy_lle_unlocked(struct llentry *lle)
in_lltable_destroy_lle_unlocked(epoch_context_t ctx)
{
struct llentry *lle;
lle = __containerof(ctx, struct llentry, lle_epoch_ctx);
LLE_LOCK_DESTROY(lle);
LLE_REQ_DESTROY(lle);
free(lle, M_LLTABLE);
@ -1085,7 +1087,7 @@ in_lltable_destroy_lle(struct llentry *lle)
{
LLE_WUNLOCK(lle);
in_lltable_destroy_lle_unlocked(lle);
epoch_call(net_epoch_preempt, &lle->lle_epoch_ctx, in_lltable_destroy_lle_unlocked);
}
static struct llentry *
@ -1294,7 +1296,7 @@ in_lltable_find_dst(struct lltable *llt, struct in_addr dst)
hashidx = in_lltable_hash_dst(dst, llt->llt_hsize);
lleh = &llt->lle_head[hashidx];
LIST_FOREACH(lle, lleh, lle_next) {
CK_LIST_FOREACH(lle, lleh, lle_next) {
if (lle->la_flags & LLE_DELETED)
continue;
if (lle->r_l3addr.addr4.s_addr == dst.s_addr)
@ -1350,7 +1352,7 @@ in_lltable_alloc(struct lltable *llt, u_int flags, const struct sockaddr *l3addr
linkhdrsize = LLE_MAX_LINKHDR;
if (lltable_calc_llheader(ifp, AF_INET, IF_LLADDR(ifp),
linkhdr, &linkhdrsize, &lladdr_off) != 0) {
in_lltable_destroy_lle_unlocked(lle);
epoch_call(net_epoch_preempt, &lle->lle_epoch_ctx, in_lltable_destroy_lle_unlocked);
return (NULL);
}
lltable_set_entry_addr(ifp, lle, linkhdr, linkhdrsize,

6
sys/netinet/in_mcast.c
View file

@ -1876,12 +1876,12 @@ inp_getmoptions(struct inpcb *inp, struct sockopt *sopt)
mreqn.imr_address = imo->imo_multicast_addr;
} else if (ifp != NULL) {
mreqn.imr_ifindex = ifp->if_index;
NET_EPOCH_ENTER();
IFP_TO_IA(ifp, ia, &in_ifa_tracker);
if (ia != NULL) {
if (ia != NULL)
mreqn.imr_address =
IA_SIN(ia)->sin_addr;
ifa_free(&ia->ia_ifa);
}
NET_EPOCH_EXIT();
}
}
INP_WUNLOCK(inp);

12
sys/netinet/in_pcb.c
View file

@ -794,7 +794,6 @@ in_pcbladdr(struct inpcb *inp, struct in_addr *faddr, struct in_addr *laddr,
int error;
KASSERT(laddr != NULL, ("%s: laddr NULL", __func__));
/*
* Bypass source address selection and use the primary jail IP
* if requested.
@ -827,28 +826,30 @@ in_pcbladdr(struct inpcb *inp, struct in_addr *faddr, struct in_addr *laddr,
* network and try to find a corresponding interface to take
* the source address from.
*/
NET_EPOCH_ENTER();
if (sro.ro_rt == NULL || sro.ro_rt->rt_ifp == NULL) {
struct in_ifaddr *ia;
struct ifnet *ifp;
ia = ifatoia(ifa_ifwithdstaddr((struct sockaddr *)sin,
inp->inp_socket->so_fibnum));
if (ia == NULL)
if (ia == NULL) {
ia = ifatoia(ifa_ifwithnet((struct sockaddr *)sin, 0,
inp->inp_socket->so_fibnum));
}
if (ia == NULL) {
printf("ifa_ifwithnet failed\n");
error = ENETUNREACH;
goto done;
}
if (cred == NULL || !prison_flag(cred, PR_IP4)) {
laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
ifa_free(&ia->ia_ifa);
goto done;
}
ifp = ia->ia_ifp;
ifa_free(&ia->ia_ifa);
ia = NULL;
IF_ADDR_RLOCK(ifp);
CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
@ -964,7 +965,6 @@ in_pcbladdr(struct inpcb *inp, struct in_addr *faddr, struct in_addr *laddr,
goto done;
}
laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
ifa_free(&ia->ia_ifa);
goto done;
}
@ -973,7 +973,6 @@ in_pcbladdr(struct inpcb *inp, struct in_addr *faddr, struct in_addr *laddr,
struct ifnet *ifp;
ifp = ia->ia_ifp;
ifa_free(&ia->ia_ifa);
ia = NULL;
IF_ADDR_RLOCK(ifp);
CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
@ -1002,6 +1001,7 @@ in_pcbladdr(struct inpcb *inp, struct in_addr *faddr, struct in_addr *laddr,
}
done:
NET_EPOCH_EXIT();
if (sro.ro_rt != NULL)
RTFREE(sro.ro_rt);
return (error);

2
sys/netinet/in_var.h
View file

@ -176,8 +176,6 @@ do { \
(ia) != NULL && (ia)->ia_ifp != (ifp); \
(ia) = CK_STAILQ_NEXT((ia), ia_link)) \
continue; \
if ((ia) != NULL) \
ifa_ref(&(ia)->ia_ifa); \
IN_IFADDR_RUNLOCK((t)); \
} while (0)

4
sys/netinet/ip_divert.c
View file

@ -468,13 +468,15 @@ div_output(struct socket *so, struct mbuf *m, struct sockaddr_in *sin,
bzero(sin->sin_zero, sizeof(sin->sin_zero));
sin->sin_port = 0;
NET_EPOCH_ENTER();
ifa = ifa_ifwithaddr((struct sockaddr *) sin);
if (ifa == NULL) {
error = EADDRNOTAVAIL;
NET_EPOCH_EXIT();
goto cantsend;
}
m->m_pkthdr.rcvif = ifa->ifa_ifp;
ifa_free(ifa);
NET_EPOCH_EXIT();
}
#ifdef MAC
mac_socket_create_mbuf(so, m);

11
sys/netinet/ip_icmp.c
View file

@ -406,6 +406,7 @@ icmp_input(struct mbuf **mp, int *offp, int proto)
inet_ntoa_r(ip->ip_dst, dstbuf), icmplen);
}
#endif
NET_EPOCH_ENTER();
if (icmplen < ICMP_MINLEN) {
ICMPSTAT_INC(icps_tooshort);
goto freeit;
@ -413,6 +414,7 @@ icmp_input(struct mbuf **mp, int *offp, int proto)
i = hlen + min(icmplen, ICMP_ADVLENMIN);
if (m->m_len < i && (m = m_pullup(m, i)) == NULL) {
ICMPSTAT_INC(icps_tooshort);
NET_EPOCH_EXIT();
return (IPPROTO_DONE);
}
ip = mtod(m, struct ip *);
@ -530,6 +532,7 @@ icmp_input(struct mbuf **mp, int *offp, int proto)
if (m->m_len < i && (m = m_pullup(m, i)) == NULL) {
/* This should actually not happen */
ICMPSTAT_INC(icps_tooshort);
NET_EPOCH_EXIT();
return (IPPROTO_DONE);
}
ip = mtod(m, struct ip *);
@ -605,10 +608,8 @@ icmp_input(struct mbuf **mp, int *offp, int proto)
(struct sockaddr *)&icmpdst, m->m_pkthdr.rcvif);
if (ia == NULL)
break;
if (ia->ia_ifp == NULL) {
ifa_free(&ia->ia_ifa);
if (ia->ia_ifp == NULL)
break;
}
icp->icmp_type = ICMP_MASKREPLY;
if (V_icmpmaskfake == 0)
icp->icmp_mask = ia->ia_sockmask.sin_addr.s_addr;
@ -620,11 +621,11 @@ icmp_input(struct mbuf **mp, int *offp, int proto)
else if (ia->ia_ifp->if_flags & IFF_POINTOPOINT)
ip->ip_src = satosin(&ia->ia_dstaddr)->sin_addr;
}
ifa_free(&ia->ia_ifa);
reflect:
ICMPSTAT_INC(icps_reflect);
ICMPSTAT_INC(icps_outhist[icp->icmp_type]);
icmp_reflect(m);
NET_EPOCH_EXIT();
return (IPPROTO_DONE);
case ICMP_REDIRECT:
@ -701,11 +702,13 @@ icmp_input(struct mbuf **mp, int *offp, int proto)
}
raw:
NET_EPOCH_EXIT();
*mp = m;
rip_input(mp, offp, proto);
return (IPPROTO_DONE);
freeit:
NET_EPOCH_EXIT();
m_freem(m);
return (IPPROTO_DONE);
}

26
sys/netinet/ip_input.c
View file

@ -399,7 +399,7 @@ ip_destroy(void *unused __unused)
/* Make sure the IPv4 routes are gone as well. */
IFNET_RLOCK();
TAILQ_FOREACH(ifp, &V_ifnet, if_link)
CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link)
rt_flushifroutes_af(ifp, AF_INET);
IFNET_RUNLOCK();
@ -977,9 +977,9 @@ ip_forward(struct mbuf *m, int srcrt)
#else
in_rtalloc_ign(&ro, 0, M_GETFIB(m));
#endif
NET_EPOCH_ENTER();
if (ro.ro_rt != NULL) {
ia = ifatoia(ro.ro_rt->rt_ifa);
ifa_ref(&ia->ia_ifa);
} else
ia = NULL;
/*
@ -1025,7 +1025,7 @@ ip_forward(struct mbuf *m, int srcrt)
m_freem(mcopy);
if (error != EINPROGRESS)
IPSTAT_INC(ips_cantforward);
return;
goto out;
}
/* No IPsec processing required */
}
@ -1078,16 +1078,12 @@ ip_forward(struct mbuf *m, int srcrt)
else {
if (mcopy)
m_freem(mcopy);
if (ia != NULL)
ifa_free(&ia->ia_ifa);
return;
goto out;
}
}
if (mcopy == NULL) {
if (ia != NULL)
ifa_free(&ia->ia_ifa);
return;
}
if (mcopy == NULL)
goto out;
switch (error) {
@ -1129,13 +1125,11 @@ ip_forward(struct mbuf *m, int srcrt)
case ENOBUFS:
case EACCES: /* ipfw denied packet */
m_freem(mcopy);
if (ia != NULL)
ifa_free(&ia->ia_ifa);
return;
goto out;
}
if (ia != NULL)
ifa_free(&ia->ia_ifa);
icmp_error(mcopy, type, code, dest.s_addr, mtu);
out:
NET_EPOCH_EXIT();
}
#define CHECK_SO_CT(sp, ct) \

4
sys/netinet/ip_mroute.c
View file

@ -878,13 +878,15 @@ add_vif(struct vifctl *vifcp)
ifp = NULL;
} else {
sin.sin_addr = vifcp->vifc_lcl_addr;
NET_EPOCH_ENTER();
ifa = ifa_ifwithaddr((struct sockaddr *)&sin);
if (ifa == NULL) {
NET_EPOCH_EXIT();
VIF_UNLOCK();
return EADDRNOTAVAIL;
}
ifp = ifa->ifa_ifp;
ifa_free(ifa);
NET_EPOCH_EXIT();
}
if ((vifcp->vifc_flags & VIFF_TUNNEL) != 0) {

7
sys/netinet/ip_options.c
View file

@ -110,6 +110,7 @@ ip_dooptions(struct mbuf *m, int pass)
struct nhop4_extended nh_ext;
struct sockaddr_in ipaddr = { sizeof(ipaddr), AF_INET };
NET_EPOCH_ENTER();
/* Ignore or reject packets with IP options. */
if (V_ip_doopts == 0)
return 0;
@ -224,6 +225,7 @@ ip_dooptions(struct mbuf *m, int pass)
#endif
IPSTAT_INC(ips_cantforward);
m_freem(m);
NET_EPOCH_EXIT();
return (1);
}
}
@ -250,7 +252,6 @@ ip_dooptions(struct mbuf *m, int pass)
memcpy(cp + off, &(IA_SIN(ia)->sin_addr),
sizeof(struct in_addr));
ifa_free(&ia->ia_ifa);
} else {
/* XXX MRT 0 for routing */
if (fib4_lookup_nh_ext(M_GETFIB(m),
@ -298,7 +299,6 @@ ip_dooptions(struct mbuf *m, int pass)
if ((ia = (INA)ifa_ifwithaddr((SA)&ipaddr)) != NULL) {
memcpy(cp + off, &(IA_SIN(ia)->sin_addr),
sizeof(struct in_addr));
ifa_free(&ia->ia_ifa);
} else if (fib4_lookup_nh_ext(M_GETFIB(m),
ipaddr.sin_addr, 0, 0, &nh_ext) == 0) {
memcpy(cp + off, &nh_ext.nh_src,
@ -353,7 +353,6 @@ ip_dooptions(struct mbuf *m, int pass)
continue;
(void)memcpy(sin, &IA_SIN(ia)->sin_addr,
sizeof(struct in_addr));
ifa_free(&ia->ia_ifa);
cp[IPOPT_OFFSET] += sizeof(struct in_addr);
off += sizeof(struct in_addr);
break;
@ -381,12 +380,14 @@ ip_dooptions(struct mbuf *m, int pass)
cp[IPOPT_OFFSET] += sizeof(uint32_t);
}
}
NET_EPOCH_EXIT();
if (forward && V_ipforwarding) {
ip_forward(m, 1);
return (1);
}
return (0);
bad:
NET_EPOCH_EXIT();
icmp_error(m, type, code, 0, 0);
IPSTAT_INC(ips_badoptions);
return (1);

14
sys/netinet/ip_output.c
View file

@ -225,7 +225,6 @@ ip_output(struct mbuf *m, struct mbuf *opt, struct route *ro, int flags,
struct route iproute;
struct rtentry *rte; /* cache for ro->ro_rt */
uint32_t fibnum;
int have_ia_ref;
#if defined(IPSEC) || defined(IPSEC_SUPPORT)
int no_route_but_check_spd = 0;
#endif
@ -281,6 +280,7 @@ ip_output(struct mbuf *m, struct mbuf *opt, struct route *ro, int flags,
dst->sin_len = sizeof(*dst);
dst->sin_addr = ip->ip_dst;
}
NET_EPOCH_ENTER();
again:
/*
* Validate route against routing table additions;
@ -306,7 +306,6 @@ ip_output(struct mbuf *m, struct mbuf *opt, struct route *ro, int flags,
rte = NULL;
}
ia = NULL;
have_ia_ref = 0;
/*
* If routing to interface only, short circuit routing lookup.
* The use of an all-ones broadcast address implies this; an
@ -322,7 +321,6 @@ ip_output(struct mbuf *m, struct mbuf *opt, struct route *ro, int flags,
error = ENETUNREACH;
goto bad;
}
have_ia_ref = 1;
ip->ip_dst.s_addr = INADDR_BROADCAST;
dst->sin_addr = ip->ip_dst;
ifp = ia->ia_ifp;
@ -337,7 +335,6 @@ ip_output(struct mbuf *m, struct mbuf *opt, struct route *ro, int flags,
error = ENETUNREACH;
goto bad;
}
have_ia_ref = 1;
ifp = ia->ia_ifp;
ip->ip_ttl = 1;
isbroadcast = ifp->if_flags & IFF_BROADCAST ?
@ -350,8 +347,6 @@ ip_output(struct mbuf *m, struct mbuf *opt, struct route *ro, int flags,
*/
ifp = imo->imo_multicast_ifp;
IFP_TO_IA(ifp, ia, &in_ifa_tracker);
if (ia)
have_ia_ref = 1;
isbroadcast = 0; /* fool gcc */
} else {
/*
@ -579,8 +574,6 @@ ip_output(struct mbuf *m, struct mbuf *opt, struct route *ro, int flags,
case -1: /* Need to try again */
/* Reset everything for a new round */
RO_RTFREE(ro);
if (have_ia_ref)
ifa_free(&ia->ia_ifa);
ro->ro_prepend = NULL;
rte = NULL;
gw = dst;
@ -735,10 +728,9 @@ ip_output(struct mbuf *m, struct mbuf *opt, struct route *ro, int flags,
* calling RTFREE on it again.
*/
ro->ro_rt = NULL;
if (have_ia_ref)
ifa_free(&ia->ia_ifa);
NET_EPOCH_EXIT();
return (error);
bad:
bad:
m_freem(m);
goto done;
}

2
sys/netinet/netdump/netdump_client.c
View file

@ -1069,7 +1069,7 @@ netdump_configure(struct netdump_conf *conf, struct thread *td)
return (EINVAL);
}
IFNET_RLOCK_NOSLEEP();
TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) {
if (strcmp(ifp->if_xname, conf->ndc_iface) == 0)
break;
}

4
sys/netinet/raw_ip.c
View file

@ -929,7 +929,7 @@ rip_bind(struct socket *so, struct sockaddr *nam, struct thread *td)
inp = sotoinpcb(so);
KASSERT(inp != NULL, ("rip_bind: inp == NULL"));
if (TAILQ_EMPTY(&V_ifnet) ||
if (CK_STAILQ_EMPTY(&V_ifnet) ||
(addr->sin_family != AF_INET && addr->sin_family != AF_IMPLINK) ||
(addr->sin_addr.s_addr &&
(inp->inp_flags & INP_BINDANY) == 0 &&
@ -954,7 +954,7 @@ rip_connect(struct socket *so, struct sockaddr *nam, struct thread *td)
if (nam->sa_len != sizeof(*addr))
return (EINVAL);
if (TAILQ_EMPTY(&V_ifnet))
if (CK_STAILQ_EMPTY(&V_ifnet))
return (EADDRNOTAVAIL);
if (addr->sin_family != AF_INET && addr->sin_family != AF_IMPLINK)
return (EAFNOSUPPORT);

4
sys/netinet/sctp_bsd_addr.c
View file

@ -207,7 +207,7 @@ sctp_init_ifns_for_vrf(int vrfid)
#endif
IFNET_RLOCK();
TAILQ_FOREACH(ifn, &MODULE_GLOBAL(ifnet), if_link) {
CK_STAILQ_FOREACH(ifn, &MODULE_GLOBAL(ifnet), if_link) {
if (sctp_is_desired_interface_type(ifn) == 0) {
/* non desired type */
continue;
@ -360,7 +360,7 @@ void
struct ifaddr *ifa;
IFNET_RLOCK();
TAILQ_FOREACH(ifn, &MODULE_GLOBAL(ifnet), if_link) {
CK_STAILQ_FOREACH(ifn, &MODULE_GLOBAL(ifnet), if_link) {
if (!(*pred) (ifn)) {
continue;
}

6
sys/netinet6/icmp6.c
View file

@ -1677,7 +1677,7 @@ ni6_addrs(struct icmp6_nodeinfo *ni6, struct mbuf *m, struct ifnet **ifpp,
}
IFNET_RLOCK_NOSLEEP();
TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) {
addrsofif = 0;
IF_ADDR_RLOCK(ifp);
CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
@ -1761,10 +1761,10 @@ ni6_store_addrs(struct icmp6_nodeinfo *ni6, struct icmp6_nodeinfo *nni6,
return (0); /* needless to copy */
IFNET_RLOCK_NOSLEEP();
ifp = ifp0 ? ifp0 : TAILQ_FIRST(&V_ifnet);
ifp = ifp0 ? ifp0 : CK_STAILQ_FIRST(&V_ifnet);
again:
for (; ifp; ifp = TAILQ_NEXT(ifp, if_link)) {
for (; ifp; ifp = CK_STAILQ_NEXT(ifp, if_link)) {
IF_ADDR_RLOCK(ifp);
CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
if (ifa->ifa_addr->sa_family != AF_INET6)

16
sys/netinet6/in6.c
View file

@ -1973,7 +1973,7 @@ in6_setmaxmtu(void)
struct ifnet *ifp;
IFNET_RLOCK_NOSLEEP();
TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) {
/* this function can be called during ifnet initialization */
if (!ifp->if_afdata[AF_INET6])
continue;
@ -2045,9 +2045,11 @@ struct in6_llentry {
* Do actual deallocation of @lle.
*/
static void
in6_lltable_destroy_lle_unlocked(struct llentry *lle)
in6_lltable_destroy_lle_unlocked(epoch_context_t ctx)
{
struct llentry *lle;
lle = __containerof(ctx, struct llentry, lle_epoch_ctx);
LLE_LOCK_DESTROY(lle);
LLE_REQ_DESTROY(lle);
free(lle, M_LLTABLE);
@ -2062,7 +2064,7 @@ in6_lltable_destroy_lle(struct llentry *lle)
{
LLE_WUNLOCK(lle);
in6_lltable_destroy_lle_unlocked(lle);
epoch_call(net_epoch_preempt, &lle->lle_epoch_ctx, in6_lltable_destroy_lle_unlocked);
}
static struct llentry *
@ -2165,11 +2167,13 @@ in6_lltable_rtcheck(struct ifnet *ifp,
* Create an ND6 cache for an IPv6 neighbor
* that is not covered by our own prefix.
*/
NET_EPOCH_ENTER();
ifa = ifaof_ifpforaddr(l3addr, ifp);
if (ifa != NULL) {
ifa_free(ifa);
NET_EPOCH_EXIT();
return 0;
}
NET_EPOCH_EXIT();
log(LOG_INFO, "IPv6 address: \"%s\" is not on the network\n",
ip6_sprintf(ip6buf, &sin6->sin6_addr));
return EINVAL;
@ -2231,7 +2235,7 @@ in6_lltable_find_dst(struct lltable *llt, const struct in6_addr *dst)
hashidx = in6_lltable_hash_dst(dst, llt->llt_hsize);
lleh = &llt->lle_head[hashidx];
LIST_FOREACH(lle, lleh, lle_next) {
CK_LIST_FOREACH(lle, lleh, lle_next) {
if (lle->la_flags & LLE_DELETED)
continue;
if (IN6_ARE_ADDR_EQUAL(&lle->r_l3addr.addr6, dst))
@ -2286,7 +2290,7 @@ in6_lltable_alloc(struct lltable *llt, u_int flags,
linkhdrsize = LLE_MAX_LINKHDR;
if (lltable_calc_llheader(ifp, AF_INET6, IF_LLADDR(ifp),
linkhdr, &linkhdrsize, &lladdr_off) != 0) {
in6_lltable_destroy_lle_unlocked(lle);
epoch_call(net_epoch_preempt, &lle->lle_epoch_ctx, in6_lltable_destroy_lle_unlocked);
return (NULL);
}
lltable_set_entry_addr(ifp, lle, linkhdr, linkhdrsize,

4
sys/netinet6/in6_ifattach.c
View file

@ -384,7 +384,7 @@ get_ifid(struct ifnet *ifp0, struct ifnet *altifp,
/* next, try to get it from some other hardware interface */
IFNET_RLOCK_NOSLEEP();
TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) {
if (ifp == ifp0)
continue;
if (in6_get_hw_ifid(ifp, in6) != 0)
@ -826,7 +826,7 @@ in6_tmpaddrtimer(void *arg)
V_ip6_temp_regen_advance) * hz, in6_tmpaddrtimer, curvnet);
bzero(nullbuf, sizeof(nullbuf));
TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) {
if (ifp->if_afdata[AF_INET6] == NULL)
continue;
ndi = ND_IFINFO(ifp);

7
sys/netinet6/in6_pcb.c
View file

@ -170,9 +170,11 @@ in6_pcbbind(struct inpcb *inp, struct sockaddr *nam,
struct ifaddr *ifa;
sin6->sin6_port = 0; /* yech... */
NET_EPOCH_ENTER();
if ((ifa = ifa_ifwithaddr((struct sockaddr *)sin6)) ==
NULL &&
(inp->inp_flags & INP_BINDANY) == 0) {
NET_EPOCH_EXIT();
return (EADDRNOTAVAIL);
}
@ -185,11 +187,10 @@ in6_pcbbind(struct inpcb *inp, struct sockaddr *nam,
if (ifa != NULL &&
((struct in6_ifaddr *)ifa)->ia6_flags &
(IN6_IFF_ANYCAST|IN6_IFF_NOTREADY|IN6_IFF_DETACHED)) {
ifa_free(ifa);
NET_EPOCH_EXIT();
return (EADDRNOTAVAIL);
}
if (ifa != NULL)
ifa_free(ifa);
NET_EPOCH_EXIT();
}
if (lport) {
struct inpcb *t;

2
sys/netinet6/ip6_input.c
View file

@ -377,7 +377,7 @@ ip6_destroy(void *unused __unused)
/* Cleanup addresses. */
IFNET_RLOCK();
TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) {
/* Cannot lock here - lock recursion. */
/* IF_ADDR_LOCK(ifp); */
CK_STAILQ_FOREACH_SAFE(ifa, &ifp->if_addrhead, ifa_link, nifa) {

2
sys/netinet6/nd6.c
View file

@ -2133,7 +2133,7 @@ nd6_slowtimo(void *arg)
callout_reset(&V_nd6_slowtimo_ch, ND6_SLOWTIMER_INTERVAL * hz,
nd6_slowtimo, curvnet);
IFNET_RLOCK_NOSLEEP();
TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) {
if (ifp->if_afdata[AF_INET6] == NULL)
continue;
nd6if = ND_IFINFO(ifp);

14
sys/netinet6/raw_ip6.c
View file

@ -734,23 +734,25 @@ rip6_bind(struct socket *so, struct sockaddr *nam, struct thread *td)
return (EINVAL);
if ((error = prison_check_ip6(td->td_ucred, &addr->sin6_addr)) != 0)
return (error);
if (TAILQ_EMPTY(&V_ifnet) || addr->sin6_family != AF_INET6)
if (CK_STAILQ_EMPTY(&V_ifnet) || addr->sin6_family != AF_INET6)
return (EADDRNOTAVAIL);
if ((error = sa6_embedscope(addr, V_ip6_use_defzone)) != 0)
return (error);
NET_EPOCH_ENTER();
if (!IN6_IS_ADDR_UNSPECIFIED(&addr->sin6_addr) &&
(ifa = ifa_ifwithaddr((struct sockaddr *)addr)) == NULL)
(ifa = ifa_ifwithaddr((struct sockaddr *)addr)) == NULL) {
NET_EPOCH_EXIT();
return (EADDRNOTAVAIL);
}
if (ifa != NULL &&
((struct in6_ifaddr *)ifa)->ia6_flags &
(IN6_IFF_ANYCAST|IN6_IFF_NOTREADY|
IN6_IFF_DETACHED|IN6_IFF_DEPRECATED)) {
ifa_free(ifa);
NET_EPOCH_EXIT();
return (EADDRNOTAVAIL);
}
if (ifa != NULL)
ifa_free(ifa);
NET_EPOCH_EXIT();
INP_INFO_WLOCK(&V_ripcbinfo);
INP_WLOCK(inp);
inp->in6p_laddr = addr->sin6_addr;
@ -772,7 +774,7 @@ rip6_connect(struct socket *so, struct sockaddr *nam, struct thread *td)
if (nam->sa_len != sizeof(*addr))
return (EINVAL);
if (TAILQ_EMPTY(&V_ifnet))
if (CK_STAILQ_EMPTY(&V_ifnet))
return (EADDRNOTAVAIL);
if (addr->sin6_family != AF_INET6)
return (EAFNOSUPPORT);

18
sys/netpfil/pf/pf_if.c
View file

@ -129,9 +129,9 @@ pfi_initialize_vnet(void)
PF_RULES_WUNLOCK();
IFNET_RLOCK();
TAILQ_FOREACH(ifg, &V_ifg_head, ifg_next)
CK_STAILQ_FOREACH(ifg, &V_ifg_head, ifg_next)
pfi_attach_ifgroup(ifg);
TAILQ_FOREACH(ifp, &V_ifnet, if_link)
CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link)
pfi_attach_ifnet(ifp);
IFNET_RUNLOCK();
}
@ -300,7 +300,7 @@ pfi_kif_match(struct pfi_kif *rule_kif, struct pfi_kif *packet_kif)
if (rule_kif->pfik_group != NULL)
/* XXXGL: locking? */
TAILQ_FOREACH(p, &packet_kif->pfik_ifp->if_groups, ifgl_next)
CK_STAILQ_FOREACH(p, &packet_kif->pfik_ifp->if_groups, ifgl_next)
if (p->ifgl_group == rule_kif->pfik_group)
return (1);
@ -466,7 +466,7 @@ pfi_kif_update(struct pfi_kif *kif)
/* again for all groups kif is member of */
if (kif->pfik_ifp != NULL) {
IF_ADDR_RLOCK(kif->pfik_ifp);
TAILQ_FOREACH(ifgl, &kif->pfik_ifp->if_groups, ifgl_next)
CK_STAILQ_FOREACH(ifgl, &kif->pfik_ifp->if_groups, ifgl_next)
pfi_kif_update((struct pfi_kif *)
ifgl->ifgl_group->ifg_pf_kif);
IF_ADDR_RUNLOCK(kif->pfik_ifp);
@ -506,7 +506,7 @@ pfi_table_update(struct pfr_ktable *kt, struct pfi_kif *kif, int net, int flags)
pfi_instance_add(kif->pfik_ifp, net, flags);
else if (kif->pfik_group != NULL) {
IFNET_RLOCK_NOSLEEP();
TAILQ_FOREACH(ifgm, &kif->pfik_group->ifg_members, ifgm_next)
CK_STAILQ_FOREACH(ifgm, &kif->pfik_group->ifg_members, ifgm_next)
pfi_instance_add(ifgm->ifgm_ifp, net, flags);
IFNET_RUNLOCK_NOSLEEP();
}
@ -668,7 +668,7 @@ pfi_update_status(const char *name, struct pf_status *pfs)
struct pfi_kif *p;
struct pfi_kif_cmp key;
struct ifg_member p_member, *ifgm;
TAILQ_HEAD(, ifg_member) ifg_members;
CK_STAILQ_HEAD(, ifg_member) ifg_members;
int i, j, k;
strlcpy(key.pfik_name, name, sizeof(key.pfik_name));
@ -683,14 +683,14 @@ pfi_update_status(const char *name, struct pf_status *pfs)
/* build a temporary list for p only */
bzero(&p_member, sizeof(p_member));
p_member.ifgm_ifp = p->pfik_ifp;
TAILQ_INIT(&ifg_members);
TAILQ_INSERT_TAIL(&ifg_members, &p_member, ifgm_next);
CK_STAILQ_INIT(&ifg_members);
CK_STAILQ_INSERT_TAIL(&ifg_members, &p_member, ifgm_next);
}
if (pfs) {
bzero(pfs->pcounters, sizeof(pfs->pcounters));
bzero(pfs->bcounters, sizeof(pfs->bcounters));
}
TAILQ_FOREACH(ifgm, &ifg_members, ifgm_next) {
CK_STAILQ_FOREACH(ifgm, &ifg_members, ifgm_next) {
if (ifgm->ifgm_ifp == NULL || ifgm->ifgm_ifp->if_pf_kif == NULL)
continue;
p = (struct pfi_kif *)ifgm->ifgm_ifp->if_pf_kif;

10
sys/nfs/bootp_subr.c
View file

@ -409,12 +409,12 @@ bootpboot_p_iflist(void)
printf("Interface list:\n");
IFNET_RLOCK();
for (ifp = TAILQ_FIRST(&V_ifnet);
for (ifp = CK_STAILQ_FIRST(&V_ifnet);
ifp != NULL;
ifp = TAILQ_NEXT(ifp, if_link)) {
ifp = CK_STAILQ_NEXT(ifp, if_link)) {
for (ifa = CK_STAILQ_FIRST(&ifp->if_addrhead);
ifa != NULL;
ifa = TAILQ_NEXT(ifa, ifa_link))
ifa = CK_STAILQ_NEXT(ifa, ifa_link))
if (ifa->ifa_addr->sa_family == AF_INET)
bootpboot_p_if(ifp, ifa);
}
@ -1637,7 +1637,7 @@ bootpc_init(void)
*/
ifcnt = 0;
IFNET_RLOCK();
TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) {
if ((ifp->if_flags &
(IFF_LOOPBACK | IFF_POINTOPOINT | IFF_BROADCAST)) !=
IFF_BROADCAST)
@ -1660,7 +1660,7 @@ bootpc_init(void)
retry:
ifctx = STAILQ_FIRST(&gctx->interfaces);
IFNET_RLOCK();
TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) {
if (ifctx == NULL)
break;
#ifdef BOOTP_WIRED_TO

2
sys/nfs/nfs_diskless.c
View file

@ -224,7 +224,7 @@ nfs_setup_diskless(void)
retry:
CURVNET_SET(TD_TO_VNET(curthread));
IFNET_RLOCK();
TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) {
CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
if (ifa->ifa_addr->sa_family == AF_LINK) {
sdl = (struct sockaddr_dl *)ifa->ifa_addr;

2
sys/ofed/drivers/infiniband/core/ib_roce_gid_mgmt.c
View file

@ -191,7 +191,7 @@ roce_gid_update_addr_callback(struct ib_device *device, u8 port,
CURVNET_SET(ndev->if_vnet);
IFNET_RLOCK();
TAILQ_FOREACH(idev, &V_ifnet, if_link) {
CK_STAILQ_FOREACH(idev, &V_ifnet, if_link) {
if (idev != ndev) {
if (idev->if_type != IFT_L2VLAN)
continue;