diff --git a/sys/compat/linprocfs/linprocfs.c b/sys/compat/linprocfs/linprocfs.c index d56f23e679e8..586259f28315 100644 --- a/sys/compat/linprocfs/linprocfs.c +++ b/sys/compat/linprocfs/linprocfs.c @@ -1138,7 +1138,7 @@ linux_ifname(struct ifnet *ifp, char *buffer, size_t buflen) /* Determine the (relative) unit number for ethernet interfaces */ ethno = 0; - TAILQ_FOREACH(ifscan, &V_ifnet, if_link) { + CK_STAILQ_FOREACH(ifscan, &V_ifnet, if_link) { if (ifscan == ifp) return (snprintf(buffer, buflen, "eth%d", ethno)); if (IFP_IS_ETH(ifscan)) @@ -1166,7 +1166,7 @@ linprocfs_donetdev(PFS_FILL_ARGS) CURVNET_SET(TD_TO_VNET(curthread)); IFNET_RLOCK(); - TAILQ_FOREACH(ifp, &V_ifnet, if_link) { + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { linux_ifname(ifp, ifname, sizeof ifname); sbuf_printf(sb, "%6.6s: ", ifname); sbuf_printf(sb, "%7ju %7ju %4ju %4ju %4lu %5lu %10lu %9ju ", diff --git a/sys/compat/linux/linux_ioctl.c b/sys/compat/linux/linux_ioctl.c index d3ee9928d0ed..cea8721f096e 100644 --- a/sys/compat/linux/linux_ioctl.c +++ b/sys/compat/linux/linux_ioctl.c @@ -2150,7 +2150,7 @@ ifname_linux_to_bsd(struct thread *td, const char *lxname, char *bsdname) is_eth = (len == 3 && !strncmp(lxname, "eth", len)) ? 1 : 0; CURVNET_SET(TD_TO_VNET(td)); IFNET_RLOCK(); - TAILQ_FOREACH(ifp, &V_ifnet, if_link) { + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { /* * Allow Linux programs to use FreeBSD names. Don't presume * we never have an interface named "eth", so don't make @@ -2188,7 +2188,7 @@ linux_ioctl_ifname(struct thread *td, struct l_ifreq *uifr) index = 1; /* ifr.ifr_ifindex starts from 1 */ ethno = 0; error = ENODEV; - TAILQ_FOREACH(ifp, &V_ifnet, if_link) { + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { if (ifr.ifr_ifindex == index) { if (IFP_IS_ETH(ifp)) snprintf(ifr.ifr_name, LINUX_IFNAMSIZ, @@ -2240,7 +2240,7 @@ linux_ifconf(struct thread *td, struct ifconf *uifc) if ((l_uintptr_t)ifc.ifc_buf == PTROUT(NULL)) { ifc.ifc_len = 0; IFNET_RLOCK(); - TAILQ_FOREACH(ifp, &V_ifnet, if_link) { + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { struct sockaddr *sa = ifa->ifa_addr; if (sa->sa_family == AF_INET) @@ -2271,7 +2271,7 @@ linux_ifconf(struct thread *td, struct ifconf *uifc) /* Return all AF_INET addresses of all interfaces */ IFNET_RLOCK(); - TAILQ_FOREACH(ifp, &V_ifnet, if_link) { + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { int addrs = 0; bzero(&ifr, sizeof(ifr)); diff --git a/sys/compat/linuxkpi/common/include/linux/inetdevice.h b/sys/compat/linuxkpi/common/include/linux/inetdevice.h index 0b9783f156fd..bffb938af76c 100644 --- a/sys/compat/linuxkpi/common/include/linux/inetdevice.h +++ b/sys/compat/linuxkpi/common/include/linux/inetdevice.h @@ -44,16 +44,17 @@ ip_dev_find(struct vnet *vnet, uint32_t addr) sin.sin_addr.s_addr = addr; sin.sin_len = sizeof(sin); sin.sin_family = AF_INET; + NET_EPOCH_ENTER(); CURVNET_SET_QUIET(vnet); ifa = ifa_ifwithaddr((struct sockaddr *)&sin); CURVNET_RESTORE(); if (ifa) { ifp = ifa->ifa_ifp; if_ref(ifp); - ifa_free(ifa); } else { ifp = NULL; } + NET_EPOCH_EXIT(); return (ifp); } @@ -69,6 +70,7 @@ ip6_dev_find(struct vnet *vnet, struct in6_addr addr) sin6.sin6_addr = addr; sin6.sin6_len = sizeof(sin6); sin6.sin6_family = AF_INET6; + NET_EPOCH_ENTER(); CURVNET_SET_QUIET(vnet); if (IN6_IS_SCOPE_LINKLOCAL(&addr) || IN6_IS_ADDR_MC_INTFACELOCAL(&addr)) { @@ -85,8 +87,8 @@ ip6_dev_find(struct vnet *vnet, struct in6_addr addr) if (ifa != NULL) { ifp = ifa->ifa_ifp; if_ref(ifp); - ifa_free(ifa); } + NET_EPOCH_EXIT(); CURVNET_RESTORE(); return (ifp); } diff --git a/sys/dev/mlx5/mlx5_ib/mlx5_ib_main.c b/sys/dev/mlx5/mlx5_ib/mlx5_ib_main.c index 5ca6f6d5c27a..ef798a091e5b 100644 --- a/sys/dev/mlx5/mlx5_ib/mlx5_ib_main.c +++ b/sys/dev/mlx5/mlx5_ib/mlx5_ib_main.c @@ -2771,7 +2771,7 @@ static int mlx5_enable_roce(struct mlx5_ib_dev *dev) VNET_FOREACH(vnet_iter) { IFNET_RLOCK(); CURVNET_SET_QUIET(vnet_iter); - TAILQ_FOREACH(idev, &V_ifnet, if_link) { + CK_STAILQ_FOREACH(idev, &V_ifnet, if_link) { /* check if network interface belongs to mlx5en */ if (!mlx5_netdev_match(idev, dev->mdev, "mce")) continue; diff --git a/sys/dev/wtap/if_wtap.c b/sys/dev/wtap/if_wtap.c index e9cb3d418fe8..198d21399cd7 100644 --- a/sys/dev/wtap/if_wtap.c +++ b/sys/dev/wtap/if_wtap.c @@ -108,7 +108,7 @@ wtap_node_write(struct cdev *dev, struct uio *uio, int ioflag) CURVNET_SET(TD_TO_VNET(curthread)); IFNET_RLOCK_NOSLEEP(); - TAILQ_FOREACH(ifp, &V_ifnet, if_link) { + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { printf("ifp->if_xname = %s\n", ifp->if_xname); if(strcmp(devtoname(dev), ifp->if_xname) == 0){ printf("found match, correspoding wtap = %s\n", diff --git a/sys/net/altq/altq_subr.c b/sys/net/altq/altq_subr.c index ae5959ad7d78..356f79f84b9a 100644 --- a/sys/net/altq/altq_subr.c +++ b/sys/net/altq/altq_subr.c @@ -434,8 +434,8 @@ tbr_timeout(arg) VNET_LIST_RLOCK_NOSLEEP(); VNET_FOREACH(vnet_iter) { CURVNET_SET(vnet_iter); - for (ifp = TAILQ_FIRST(&V_ifnet); ifp; - ifp = TAILQ_NEXT(ifp, if_link)) { + for (ifp = CK_STAILQ_FIRST(&V_ifnet); ifp; + ifp = CK_STAILQ_NEXT(ifp, if_link)) { /* read from if_snd unlocked */ if (!TBR_IS_ENABLED(&ifp->if_snd)) continue; diff --git a/sys/net/bridgestp.c b/sys/net/bridgestp.c index 36f084882c20..b89fde031938 100644 --- a/sys/net/bridgestp.c +++ b/sys/net/bridgestp.c @@ -2043,7 +2043,7 @@ bstp_reinit(struct bstp_state *bs) * bridges in the same STP domain. */ IFNET_RLOCK_NOSLEEP(); - TAILQ_FOREACH(ifp, &V_ifnet, if_link) { + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { if (ifp->if_type != IFT_ETHER) continue; /* Not Ethernet */ diff --git a/sys/net/if.c b/sys/net/if.c index 980940b268ab..596074b2b124 100644 --- a/sys/net/if.c +++ b/sys/net/if.c @@ -256,7 +256,7 @@ struct mbuf *(*tbr_dequeue_ptr)(struct ifaltq *, int) = NULL; static void if_attachdomain(void *); static void if_attachdomain1(struct ifnet *); static int ifconf(u_long, caddr_t); -static void if_grow(void); +static void *if_grow(void); static void if_input_default(struct ifnet *, struct mbuf *); static int if_requestencap_default(struct ifnet *, struct if_encap_req *); static void if_route(struct ifnet *, int flag, int fam); @@ -345,9 +345,7 @@ ifnet_byindex(u_short idx) { struct ifnet *ifp; - IFNET_RLOCK_NOSLEEP(); ifp = ifnet_byindex_locked(idx); - IFNET_RUNLOCK_NOSLEEP(); return (ifp); } @@ -372,12 +370,11 @@ ifnet_byindex_ref(u_short idx) * failure. */ static u_short -ifindex_alloc(void) +ifindex_alloc(void **old) { u_short idx; IFNET_WLOCK_ASSERT(); -retry: /* * Try to find an empty slot below V_if_index. If we fail, take the * next slot. @@ -389,8 +386,8 @@ ifindex_alloc(void) /* Catch if_index overflow. */ if (idx >= V_if_indexlim) { - if_grow(); - goto retry; + *old = if_grow(); + return (USHRT_MAX); } if (idx > V_if_index) V_if_index = idx; @@ -418,22 +415,11 @@ ifindex_free(u_short idx) IFNET_WUNLOCK(); } -static void -ifnet_setbyindex_locked(u_short idx, struct ifnet *ifp) -{ - - IFNET_WLOCK_ASSERT(); - - V_ifindex_table[idx] = ifp; -} - static void ifnet_setbyindex(u_short idx, struct ifnet *ifp) { - IFNET_WLOCK(); - ifnet_setbyindex_locked(idx, ifp); - IFNET_WUNLOCK(); + V_ifindex_table[idx] = ifp; } struct ifaddr * @@ -460,12 +446,15 @@ ifaddr_byindex(u_short idx) static void vnet_if_init(const void *unused __unused) { + void *old; - TAILQ_INIT(&V_ifnet); - TAILQ_INIT(&V_ifg_head); + CK_STAILQ_INIT(&V_ifnet); + CK_STAILQ_INIT(&V_ifg_head); IFNET_WLOCK(); - if_grow(); /* create initial table */ + old = if_grow(); /* create initial table */ IFNET_WUNLOCK(); + epoch_wait_preempt(net_epoch_preempt); + free(old, M_IFNET); vnet_if_clone_init(); } VNET_SYSINIT(vnet_if_init, SI_SUB_INIT_IF, SI_ORDER_SECOND, vnet_if_init, @@ -476,9 +465,9 @@ static void vnet_if_uninit(const void *unused __unused) { - VNET_ASSERT(TAILQ_EMPTY(&V_ifnet), ("%s:%d tailq &V_ifnet=%p " + VNET_ASSERT(CK_STAILQ_EMPTY(&V_ifnet), ("%s:%d tailq &V_ifnet=%p " "not empty", __func__, __LINE__, &V_ifnet)); - VNET_ASSERT(TAILQ_EMPTY(&V_ifg_head), ("%s:%d tailq &V_ifg_head=%p " + VNET_ASSERT(CK_STAILQ_EMPTY(&V_ifg_head), ("%s:%d tailq &V_ifg_head=%p " "not empty", __func__, __LINE__, &V_ifg_head)); free((caddr_t)V_ifindex_table, M_IFNET); @@ -492,7 +481,7 @@ vnet_if_return(const void *unused __unused) struct ifnet *ifp, *nifp; /* Return all inherited interfaces to their parent vnets. */ - TAILQ_FOREACH_SAFE(ifp, &V_ifnet, if_link, nifp) { + CK_STAILQ_FOREACH_SAFE(ifp, &V_ifnet, if_link, nifp) { if (ifp->if_home_vnet != ifp->if_vnet) if_vmove(ifp, ifp->if_home_vnet); } @@ -501,13 +490,16 @@ VNET_SYSUNINIT(vnet_if_return, SI_SUB_VNET_DONE, SI_ORDER_ANY, vnet_if_return, NULL); #endif -static void + +static void * if_grow(void) { int oldlim; u_int n; struct ifnet **e; + void *old; + old = NULL; IFNET_WLOCK_ASSERT(); oldlim = V_if_indexlim; IFNET_WUNLOCK(); @@ -516,14 +508,15 @@ if_grow(void) IFNET_WLOCK(); if (V_if_indexlim != oldlim) { free(e, M_IFNET); - return; + return (NULL); } if (V_ifindex_table != NULL) { memcpy((caddr_t)e, (caddr_t)V_ifindex_table, n/2); - free((caddr_t)V_ifindex_table, M_IFNET); + old = V_ifindex_table; } V_if_indexlim <<= 1; V_ifindex_table = e; + return (old); } /* @@ -536,11 +529,19 @@ if_alloc(u_char type) { struct ifnet *ifp; u_short idx; + void *old; ifp = malloc(sizeof(struct ifnet), M_IFNET, M_WAITOK|M_ZERO); + restart: IFNET_WLOCK(); - idx = ifindex_alloc(); - ifnet_setbyindex_locked(idx, IFNET_HOLD); + idx = ifindex_alloc(&old); + if (__predict_false(idx == USHRT_MAX)) { + IFNET_WUNLOCK(); + epoch_wait_preempt(net_epoch_preempt); + free(old, M_IFNET); + goto restart; + } + ifnet_setbyindex(idx, IFNET_HOLD); IFNET_WUNLOCK(); ifp->if_index = idx; ifp->if_type = type; @@ -563,7 +564,7 @@ if_alloc(u_char type) IF_AFDATA_LOCK_INIT(ifp); CK_STAILQ_INIT(&ifp->if_addrhead); CK_STAILQ_INIT(&ifp->if_multiaddrs); - TAILQ_INIT(&ifp->if_groups); + CK_STAILQ_INIT(&ifp->if_groups); #ifdef MAC mac_ifnet_init(ifp); #endif @@ -609,6 +610,15 @@ if_free_internal(struct ifnet *ifp) free(ifp, M_IFNET); } +static void +if_destroy(epoch_context_t ctx) +{ + struct ifnet *ifp; + + ifp = __containerof(ctx, struct ifnet, if_epoch_ctx); + if_free_internal(ifp); +} + /* * Deregister an interface and free the associated storage. */ @@ -627,7 +637,7 @@ if_free(struct ifnet *ifp) IFNET_WUNLOCK(); if (refcount_release(&ifp->if_refcount)) - if_free_internal(ifp); + epoch_call(net_epoch_preempt, &ifp->if_epoch_ctx, if_destroy); CURVNET_RESTORE(); } @@ -650,7 +660,7 @@ if_rele(struct ifnet *ifp) if (!refcount_release(&ifp->if_refcount)) return; - if_free_internal(ifp); + epoch_call(net_epoch_preempt, &ifp->if_epoch_ctx, if_destroy); } void @@ -883,7 +893,7 @@ if_attach_internal(struct ifnet *ifp, int vmove, struct if_clone *ifc) #endif IFNET_WLOCK(); - TAILQ_INSERT_TAIL(&V_ifnet, ifp, if_link); + CK_STAILQ_INSERT_TAIL(&V_ifnet, ifp, if_link); #ifdef VIMAGE curvnet->vnet_ifcnt++; #endif @@ -915,7 +925,7 @@ if_attachdomain(void *dummy) { struct ifnet *ifp; - TAILQ_FOREACH(ifp, &V_ifnet, if_link) + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) if_attachdomain1(ifp); } SYSINIT(domainifattach, SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_SECOND, @@ -1050,9 +1060,9 @@ if_detach_internal(struct ifnet *ifp, int vmove, struct if_clone **ifcp) ifp->if_vnet->vnet_state < SI_SUB_VNET_DONE) ? 1 : 0; #endif IFNET_WLOCK(); - TAILQ_FOREACH(iter, &V_ifnet, if_link) + CK_STAILQ_FOREACH(iter, &V_ifnet, if_link) if (iter == ifp) { - TAILQ_REMOVE(&V_ifnet, ifp, if_link); + CK_STAILQ_REMOVE(&V_ifnet, ifp, ifnet, if_link); found = 1; break; } @@ -1080,7 +1090,7 @@ if_detach_internal(struct ifnet *ifp, int vmove, struct if_clone **ifcp) #ifdef VIMAGE curvnet->vnet_ifcnt--; #endif - + epoch_wait_preempt(net_epoch_preempt); /* * In any case (destroy or vmove) detach us from the groups * and remove/wait for pending events on the taskq. @@ -1220,6 +1230,7 @@ if_vmove(struct ifnet *ifp, struct vnet *new_vnet) { struct if_clone *ifc; u_int bif_dlt, bif_hdrlen; + void *old; int rc; /* @@ -1260,10 +1271,16 @@ if_vmove(struct ifnet *ifp, struct vnet *new_vnet) * Switch to the context of the target vnet. */ CURVNET_SET_QUIET(new_vnet); - + restart: IFNET_WLOCK(); - ifp->if_index = ifindex_alloc(); - ifnet_setbyindex_locked(ifp->if_index, ifp); + ifp->if_index = ifindex_alloc(&old); + if (__predict_false(ifp->if_index == USHRT_MAX)) { + IFNET_WUNLOCK(); + epoch_wait_preempt(net_epoch_preempt); + free(old, M_IFNET); + goto restart; + } + ifnet_setbyindex(ifp->if_index, ifp); IFNET_WUNLOCK(); if_attach_internal(ifp, 1, ifc); @@ -1400,7 +1417,7 @@ if_addgroup(struct ifnet *ifp, const char *groupname) return (EINVAL); IFNET_WLOCK(); - TAILQ_FOREACH(ifgl, &ifp->if_groups, ifgl_next) + CK_STAILQ_FOREACH(ifgl, &ifp->if_groups, ifgl_next) if (!strcmp(ifgl->ifgl_group->ifg_group, groupname)) { IFNET_WUNLOCK(); return (EEXIST); @@ -1419,7 +1436,7 @@ if_addgroup(struct ifnet *ifp, const char *groupname) return (ENOMEM); } - TAILQ_FOREACH(ifg, &V_ifg_head, ifg_next) + CK_STAILQ_FOREACH(ifg, &V_ifg_head, ifg_next) if (!strcmp(ifg->ifg_group, groupname)) break; @@ -1433,8 +1450,8 @@ if_addgroup(struct ifnet *ifp, const char *groupname) } strlcpy(ifg->ifg_group, groupname, sizeof(ifg->ifg_group)); ifg->ifg_refcnt = 0; - TAILQ_INIT(&ifg->ifg_members); - TAILQ_INSERT_TAIL(&V_ifg_head, ifg, ifg_next); + CK_STAILQ_INIT(&ifg->ifg_members); + CK_STAILQ_INSERT_TAIL(&V_ifg_head, ifg, ifg_next); new = 1; } @@ -1443,8 +1460,8 @@ if_addgroup(struct ifnet *ifp, const char *groupname) ifgm->ifgm_ifp = ifp; IF_ADDR_WLOCK(ifp); - TAILQ_INSERT_TAIL(&ifg->ifg_members, ifgm, ifgm_next); - TAILQ_INSERT_TAIL(&ifp->if_groups, ifgl, ifgl_next); + CK_STAILQ_INSERT_TAIL(&ifg->ifg_members, ifgm, ifgm_next); + CK_STAILQ_INSERT_TAIL(&ifp->if_groups, ifgl, ifgl_next); IF_ADDR_WUNLOCK(ifp); IFNET_WUNLOCK(); @@ -1464,9 +1481,10 @@ if_delgroup(struct ifnet *ifp, const char *groupname) { struct ifg_list *ifgl; struct ifg_member *ifgm; + int freeifgl; IFNET_WLOCK(); - TAILQ_FOREACH(ifgl, &ifp->if_groups, ifgl_next) + CK_STAILQ_FOREACH(ifgl, &ifp->if_groups, ifgl_next) if (!strcmp(ifgl->ifgl_group->ifg_group, groupname)) break; if (ifgl == NULL) { @@ -1474,27 +1492,30 @@ if_delgroup(struct ifnet *ifp, const char *groupname) return (ENOENT); } + freeifgl = 0; IF_ADDR_WLOCK(ifp); - TAILQ_REMOVE(&ifp->if_groups, ifgl, ifgl_next); + CK_STAILQ_REMOVE(&ifp->if_groups, ifgl, ifg_list, ifgl_next); IF_ADDR_WUNLOCK(ifp); - TAILQ_FOREACH(ifgm, &ifgl->ifgl_group->ifg_members, ifgm_next) + CK_STAILQ_FOREACH(ifgm, &ifgl->ifgl_group->ifg_members, ifgm_next) if (ifgm->ifgm_ifp == ifp) break; - if (ifgm != NULL) { - TAILQ_REMOVE(&ifgl->ifgl_group->ifg_members, ifgm, ifgm_next); - free(ifgm, M_TEMP); - } + if (ifgm != NULL) + CK_STAILQ_REMOVE(&ifgl->ifgl_group->ifg_members, ifgm, ifg_member, ifgm_next); if (--ifgl->ifgl_group->ifg_refcnt == 0) { - TAILQ_REMOVE(&V_ifg_head, ifgl->ifgl_group, ifg_next); - IFNET_WUNLOCK(); + CK_STAILQ_REMOVE(&V_ifg_head, ifgl->ifgl_group, ifg_group, ifg_next); + freeifgl = 1; + } + IFNET_WUNLOCK(); + + epoch_wait_preempt(net_epoch_preempt); + if (freeifgl) { EVENTHANDLER_INVOKE(group_detach_event, ifgl->ifgl_group); free(ifgl->ifgl_group, M_TEMP); - } else - IFNET_WUNLOCK(); - + } + free(ifgm, M_TEMP); free(ifgl, M_TEMP); EVENTHANDLER_INVOKE(group_change_event, groupname); @@ -1511,38 +1532,38 @@ if_delgroups(struct ifnet *ifp) struct ifg_list *ifgl; struct ifg_member *ifgm; char groupname[IFNAMSIZ]; + int ifglfree; IFNET_WLOCK(); - while (!TAILQ_EMPTY(&ifp->if_groups)) { - ifgl = TAILQ_FIRST(&ifp->if_groups); + while (!CK_STAILQ_EMPTY(&ifp->if_groups)) { + ifgl = CK_STAILQ_FIRST(&ifp->if_groups); strlcpy(groupname, ifgl->ifgl_group->ifg_group, IFNAMSIZ); IF_ADDR_WLOCK(ifp); - TAILQ_REMOVE(&ifp->if_groups, ifgl, ifgl_next); + CK_STAILQ_REMOVE(&ifp->if_groups, ifgl, ifg_list, ifgl_next); IF_ADDR_WUNLOCK(ifp); - TAILQ_FOREACH(ifgm, &ifgl->ifgl_group->ifg_members, ifgm_next) + CK_STAILQ_FOREACH(ifgm, &ifgl->ifgl_group->ifg_members, ifgm_next) if (ifgm->ifgm_ifp == ifp) break; - if (ifgm != NULL) { - TAILQ_REMOVE(&ifgl->ifgl_group->ifg_members, ifgm, + if (ifgm != NULL) + CK_STAILQ_REMOVE(&ifgl->ifgl_group->ifg_members, ifgm, ifg_member, ifgm_next); - free(ifgm, M_TEMP); + ifglfree = 0; + if (--ifgl->ifgl_group->ifg_refcnt == 0) { + CK_STAILQ_REMOVE(&V_ifg_head, ifgl->ifgl_group, ifg_group, ifg_next); + ifglfree = 1; } - if (--ifgl->ifgl_group->ifg_refcnt == 0) { - TAILQ_REMOVE(&V_ifg_head, ifgl->ifgl_group, ifg_next); - IFNET_WUNLOCK(); + epoch_wait_preempt(net_epoch_preempt); + free(ifgm, M_TEMP); + if (ifglfree) { EVENTHANDLER_INVOKE(group_detach_event, - ifgl->ifgl_group); + ifgl->ifgl_group); free(ifgl->ifgl_group, M_TEMP); - } else - IFNET_WUNLOCK(); - - free(ifgl, M_TEMP); - + } EVENTHANDLER_INVOKE(group_change_event, groupname); IFNET_WLOCK(); @@ -1589,7 +1610,7 @@ if_getgroup(struct ifgroupreq *ifgr, struct ifnet *ifp) if (ifgr->ifgr_len == 0) { IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifgl, &ifp->if_groups, ifgl_next) + CK_STAILQ_FOREACH(ifgl, &ifp->if_groups, ifgl_next) ifgr->ifgr_len += sizeof(struct ifg_req); IF_ADDR_RUNLOCK(ifp); return (0); @@ -1599,7 +1620,7 @@ if_getgroup(struct ifgroupreq *ifgr, struct ifnet *ifp) ifgp = ifgr_groups_get(ifgr); /* XXX: wire */ IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifgl, &ifp->if_groups, ifgl_next) { + CK_STAILQ_FOREACH(ifgl, &ifp->if_groups, ifgl_next) { if (len < sizeof(ifgrq)) { IF_ADDR_RUNLOCK(ifp); return (EINVAL); @@ -1631,7 +1652,7 @@ if_getgroupmembers(struct ifgroupreq *ifgr) int len, error; IFNET_RLOCK(); - TAILQ_FOREACH(ifg, &V_ifg_head, ifg_next) + CK_STAILQ_FOREACH(ifg, &V_ifg_head, ifg_next) if (!strcmp(ifg->ifg_group, ifgr->ifgr_name)) break; if (ifg == NULL) { @@ -1640,7 +1661,7 @@ if_getgroupmembers(struct ifgroupreq *ifgr) } if (ifgr->ifgr_len == 0) { - TAILQ_FOREACH(ifgm, &ifg->ifg_members, ifgm_next) + CK_STAILQ_FOREACH(ifgm, &ifg->ifg_members, ifgm_next) ifgr->ifgr_len += sizeof(ifgrq); IFNET_RUNLOCK(); return (0); @@ -1648,7 +1669,7 @@ if_getgroupmembers(struct ifgroupreq *ifgr) len = ifgr->ifgr_len; ifgp = ifgr_groups_get(ifgr); - TAILQ_FOREACH(ifgm, &ifg->ifg_members, ifgm_next) { + CK_STAILQ_FOREACH(ifgm, &ifg->ifg_members, ifgm_next) { if (len < sizeof(ifgrq)) { IFNET_RUNLOCK(); return (EINVAL); @@ -1897,22 +1918,18 @@ ifa_switch_loopback_route(struct ifaddr *ifa, struct sockaddr *ia) * Locate an interface based on a complete address. */ /*ARGSUSED*/ -static struct ifaddr * -ifa_ifwithaddr_internal(const struct sockaddr *addr, int getref) +struct ifaddr * +ifa_ifwithaddr(const struct sockaddr *addr) { struct ifnet *ifp; struct ifaddr *ifa; - IFNET_RLOCK_NOSLEEP(); - TAILQ_FOREACH(ifp, &V_ifnet, if_link) { - IF_ADDR_RLOCK(ifp); + MPASS(in_epoch()); + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != addr->sa_family) continue; if (sa_equal(addr, ifa->ifa_addr)) { - if (getref) - ifa_ref(ifa); - IF_ADDR_RUNLOCK(ifp); goto done; } /* IP6 doesn't have broadcast */ @@ -1920,32 +1937,24 @@ ifa_ifwithaddr_internal(const struct sockaddr *addr, int getref) ifa->ifa_broadaddr && ifa->ifa_broadaddr->sa_len != 0 && sa_equal(ifa->ifa_broadaddr, addr)) { - if (getref) - ifa_ref(ifa); - IF_ADDR_RUNLOCK(ifp); goto done; } } - IF_ADDR_RUNLOCK(ifp); } ifa = NULL; done: - IFNET_RUNLOCK_NOSLEEP(); return (ifa); } -struct ifaddr * -ifa_ifwithaddr(const struct sockaddr *addr) -{ - - return (ifa_ifwithaddr_internal(addr, 1)); -} - int ifa_ifwithaddr_check(const struct sockaddr *addr) { + int rc; - return (ifa_ifwithaddr_internal(addr, 0) != NULL); + NET_EPOCH_ENTER(); + rc = (ifa_ifwithaddr(addr) != NULL); + NET_EPOCH_EXIT(); + return (rc); } /* @@ -1958,11 +1967,10 @@ ifa_ifwithbroadaddr(const struct sockaddr *addr, int fibnum) struct ifnet *ifp; struct ifaddr *ifa; - IFNET_RLOCK_NOSLEEP(); - TAILQ_FOREACH(ifp, &V_ifnet, if_link) { + MPASS(in_epoch()); + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { if ((fibnum != RT_ALL_FIBS) && (ifp->if_fib != fibnum)) continue; - IF_ADDR_RLOCK(ifp); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != addr->sa_family) continue; @@ -1970,16 +1978,12 @@ ifa_ifwithbroadaddr(const struct sockaddr *addr, int fibnum) ifa->ifa_broadaddr && ifa->ifa_broadaddr->sa_len != 0 && sa_equal(ifa->ifa_broadaddr, addr)) { - ifa_ref(ifa); - IF_ADDR_RUNLOCK(ifp); goto done; } } - IF_ADDR_RUNLOCK(ifp); } ifa = NULL; done: - IFNET_RUNLOCK_NOSLEEP(); return (ifa); } @@ -1993,28 +1997,23 @@ ifa_ifwithdstaddr(const struct sockaddr *addr, int fibnum) struct ifnet *ifp; struct ifaddr *ifa; - IFNET_RLOCK_NOSLEEP(); - TAILQ_FOREACH(ifp, &V_ifnet, if_link) { + MPASS(in_epoch()); + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { if ((ifp->if_flags & IFF_POINTOPOINT) == 0) continue; if ((fibnum != RT_ALL_FIBS) && (ifp->if_fib != fibnum)) continue; - IF_ADDR_RLOCK(ifp); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != addr->sa_family) continue; if (ifa->ifa_dstaddr != NULL && sa_equal(addr, ifa->ifa_dstaddr)) { - ifa_ref(ifa); - IF_ADDR_RUNLOCK(ifp); goto done; } } - IF_ADDR_RUNLOCK(ifp); } ifa = NULL; done: - IFNET_RUNLOCK_NOSLEEP(); return (ifa); } @@ -2031,6 +2030,7 @@ ifa_ifwithnet(const struct sockaddr *addr, int ignore_ptp, int fibnum) u_int af = addr->sa_family; const char *addr_data = addr->sa_data, *cplim; + MPASS(in_epoch()); /* * AF_LINK addresses can be looked up directly by their index number, * so do that if we can. @@ -2047,11 +2047,9 @@ ifa_ifwithnet(const struct sockaddr *addr, int ignore_ptp, int fibnum) * on ifa_maybe once we find one, as we release the IF_ADDR_RLOCK() that * kept it stable when we move onto the next interface. */ - IFNET_RLOCK_NOSLEEP(); - TAILQ_FOREACH(ifp, &V_ifnet, if_link) { + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { if ((fibnum != RT_ALL_FIBS) && (ifp->if_fib != fibnum)) continue; - IF_ADDR_RLOCK(ifp); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { const char *cp, *cp2, *cp3; @@ -2069,7 +2067,6 @@ next: continue; */ if (ifa->ifa_dstaddr != NULL && sa_equal(addr, ifa->ifa_dstaddr)) { - ifa_ref(ifa); IF_ADDR_RUNLOCK(ifp); goto done; } @@ -2103,21 +2100,14 @@ next: continue; ifa_preferred(ifa_maybe, ifa) || rn_refines((caddr_t)ifa->ifa_netmask, (caddr_t)ifa_maybe->ifa_netmask)) { - if (ifa_maybe != NULL) - ifa_free(ifa_maybe); ifa_maybe = ifa; - ifa_ref(ifa_maybe); } } } - IF_ADDR_RUNLOCK(ifp); } ifa = ifa_maybe; ifa_maybe = NULL; done: - IFNET_RUNLOCK_NOSLEEP(); - if (ifa_maybe != NULL) - ifa_free(ifa_maybe); return (ifa); } @@ -2136,7 +2126,7 @@ ifaof_ifpforaddr(const struct sockaddr *addr, struct ifnet *ifp) if (af >= AF_MAX) return (NULL); - IF_ADDR_RLOCK(ifp); + MPASS(in_epoch()); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != af) continue; @@ -2166,9 +2156,6 @@ ifaof_ifpforaddr(const struct sockaddr *addr, struct ifnet *ifp) } ifa = ifa_maybe; done: - if (ifa != NULL) - ifa_ref(ifa); - IF_ADDR_RUNLOCK(ifp); return (ifa); } @@ -2204,14 +2191,15 @@ link_rtrequest(int cmd, struct rtentry *rt, struct rt_addrinfo *info) if (cmd != RTM_ADD || ((ifa = rt->rt_ifa) == NULL) || ((ifp = ifa->ifa_ifp) == NULL) || ((dst = rt_key(rt)) == NULL)) return; + NET_EPOCH_ENTER(); ifa = ifaof_ifpforaddr(dst, ifp); if (ifa) { oifa = rt->rt_ifa; rt->rt_ifa = ifa; - ifa_free(oifa); if (ifa->ifa_rtrequest && ifa->ifa_rtrequest != link_rtrequest) ifa->ifa_rtrequest(cmd, rt, info); } + NET_EPOCH_EXIT(); } struct sockaddr_dl * @@ -2414,7 +2402,7 @@ ifunit_ref(const char *name) struct ifnet *ifp; IFNET_RLOCK_NOSLEEP(); - TAILQ_FOREACH(ifp, &V_ifnet, if_link) { + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { if (strncmp(name, ifp->if_xname, IFNAMSIZ) == 0 && !(ifp->if_flags & IFF_DYING)) break; @@ -2431,7 +2419,7 @@ ifunit(const char *name) struct ifnet *ifp; IFNET_RLOCK_NOSLEEP(); - TAILQ_FOREACH(ifp, &V_ifnet, if_link) { + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { if (strncmp(name, ifp->if_xname, IFNAMSIZ) == 0) break; } @@ -3256,7 +3244,7 @@ ifconf(u_long cmd, caddr_t data) valid_len = 0; IFNET_RLOCK(); - TAILQ_FOREACH(ifp, &V_ifnet, if_link) { + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { int addrs; /* @@ -3609,7 +3597,7 @@ if_delmulti(struct ifnet *ifp, struct sockaddr *sa) struct ifnet *oifp; IFNET_RLOCK_NOSLEEP(); - TAILQ_FOREACH(oifp, &V_ifnet, if_link) + CK_STAILQ_FOREACH(oifp, &V_ifnet, if_link) if (ifp == oifp) break; if (ifp != oifp) @@ -3683,7 +3671,7 @@ if_delmulti_ifma_flags(struct ifmultiaddr *ifma, int flags) struct ifnet *oifp; IFNET_RLOCK_NOSLEEP(); - TAILQ_FOREACH(oifp, &V_ifnet, if_link) + CK_STAILQ_FOREACH(oifp, &V_ifnet, if_link) if (ifp == oifp) break; if (ifp != oifp) { @@ -3810,23 +3798,24 @@ if_setlladdr(struct ifnet *ifp, const u_char *lladdr, int len) struct sockaddr_dl *sdl; struct ifaddr *ifa; struct ifreq ifr; + int rc; - IF_ADDR_RLOCK(ifp); + rc = 0; + NET_EPOCH_ENTER(); ifa = ifp->if_addr; if (ifa == NULL) { - IF_ADDR_RUNLOCK(ifp); - return (EINVAL); + rc = EINVAL; + goto out; } - ifa_ref(ifa); - IF_ADDR_RUNLOCK(ifp); + sdl = (struct sockaddr_dl *)ifa->ifa_addr; if (sdl == NULL) { - ifa_free(ifa); - return (EINVAL); + rc = EINVAL; + goto out; } if (len != sdl->sdl_alen) { /* don't allow length to change */ - ifa_free(ifa); - return (EINVAL); + rc = EINVAL; + goto out; } switch (ifp->if_type) { case IFT_ETHER: @@ -3835,11 +3824,10 @@ if_setlladdr(struct ifnet *ifp, const u_char *lladdr, int len) case IFT_BRIDGE: case IFT_IEEE8023ADLAG: bcopy(lladdr, LLADDR(sdl), len); - ifa_free(ifa); break; default: - ifa_free(ifa); - return (ENODEV); + rc = ENODEV; + goto out; } /* @@ -3860,7 +3848,9 @@ if_setlladdr(struct ifnet *ifp, const u_char *lladdr, int len) } } EVENTHANDLER_INVOKE(iflladdr_event, ifp); - return (0); + out: + NET_EPOCH_EXIT(); + return (rc); } /* diff --git a/sys/net/if_llatbl.c b/sys/net/if_llatbl.c index 29471afc600d..2d32ec93ff5b 100644 --- a/sys/net/if_llatbl.c +++ b/sys/net/if_llatbl.c @@ -146,7 +146,7 @@ htable_foreach_lle(struct lltable *llt, llt_foreach_cb_t *f, void *farg) error = 0; for (i = 0; i < llt->llt_hsize; i++) { - LIST_FOREACH_SAFE(lle, &llt->lle_head[i], lle_next, next) { + CK_LIST_FOREACH_SAFE(lle, &llt->lle_head[i], lle_next, next) { error = f(llt, lle, farg); if (error != 0) break; @@ -173,7 +173,7 @@ htable_link_entry(struct lltable *llt, struct llentry *lle) lle->lle_tbl = llt; lle->lle_head = lleh; lle->la_flags |= LLE_LINKED; - LIST_INSERT_HEAD(lleh, lle, lle_next); + CK_LIST_INSERT_HEAD(lleh, lle, lle_next); } static void @@ -182,7 +182,7 @@ htable_unlink_entry(struct llentry *lle) if ((lle->la_flags & LLE_LINKED) != 0) { IF_AFDATA_WLOCK_ASSERT(lle->lle_tbl->llt_ifp); - LIST_REMOVE(lle, lle_next); + CK_LIST_REMOVE(lle, lle_next); lle->la_flags &= ~(LLE_VALID | LLE_LINKED); #if 0 lle->lle_tbl = NULL; @@ -224,7 +224,7 @@ htable_prefix_free(struct lltable *llt, const struct sockaddr *addr, pmd.addr = addr; pmd.mask = mask; pmd.flags = flags; - LIST_INIT(&pmd.dchain); + CK_LIST_INIT(&pmd.dchain); IF_AFDATA_WLOCK(llt->llt_ifp); /* Push matching lles to chain */ @@ -514,7 +514,7 @@ lltable_free(struct lltable *llt) lltable_unlink(llt); - LIST_INIT(&dchain); + CK_LIST_INIT(&dchain); IF_AFDATA_WLOCK(llt->llt_ifp); /* Push all lles to @dchain */ lltable_foreach_lle(llt, lltable_free_cb, &dchain); @@ -544,7 +544,7 @@ lltable_drain(int af) continue; for (i=0; i < llt->llt_hsize; i++) { - LIST_FOREACH(lle, &llt->lle_head[i], lle_next) { + CK_LIST_FOREACH(lle, &llt->lle_head[i], lle_next) { LLE_WLOCK(lle); if (lle->la_hold) { m_freem(lle->la_hold); @@ -620,7 +620,7 @@ lltable_allocate_htbl(uint32_t hsize) M_LLTABLE, M_WAITOK | M_ZERO); for (i = 0; i < llt->llt_hsize; i++) - LIST_INIT(&llt->lle_head[i]); + CK_LIST_INIT(&llt->lle_head[i]); /* Set some default callbacks */ llt->llt_link_entry = htable_link_entry; @@ -917,7 +917,7 @@ llatbl_llt_show(struct lltable *llt) llt, llt->llt_af, llt->llt_ifp); for (i = 0; i < llt->llt_hsize; i++) { - LIST_FOREACH(lle, &llt->lle_head[i], lle_next) { + CK_LIST_FOREACH(lle, &llt->lle_head[i], lle_next) { llatbl_lle_show((struct llentry_sa *)lle); if (db_pager_quit) diff --git a/sys/net/if_llatbl.h b/sys/net/if_llatbl.h index 9cec4ac75e50..665d1ed6630c 100644 --- a/sys/net/if_llatbl.h +++ b/sys/net/if_llatbl.h @@ -34,6 +34,7 @@ __FBSDID("$FreeBSD$"); #include #include +#include struct ifnet; struct sysctl_req; @@ -80,6 +81,7 @@ struct llentry { struct callout lle_timer; struct rwlock lle_lock; struct mtx req_mtx; + struct epoch_context lle_epoch_ctx; }; #define LLE_WLOCK(lle) rw_wlock(&(lle)->lle_lock) diff --git a/sys/net/if_var.h b/sys/net/if_var.h index c3618f5c5801..7228fd899d17 100644 --- a/sys/net/if_var.h +++ b/sys/net/if_var.h @@ -89,10 +89,10 @@ struct netdump_methods; #include -TAILQ_HEAD(ifnethead, ifnet); /* we use TAILQs so that the order of */ +CK_STAILQ_HEAD(ifnethead, ifnet); /* we use TAILQs so that the order of */ CK_STAILQ_HEAD(ifaddrhead, ifaddr); /* instantiation is preserved in the list */ CK_STAILQ_HEAD(ifmultihead, ifmultiaddr); -TAILQ_HEAD(ifgrouphead, ifg_group); +CK_STAILQ_HEAD(ifgrouphead, ifg_group); #ifdef _KERNEL VNET_DECLARE(struct pfil_head, link_pfil_hook); /* packet filter hooks */ @@ -238,9 +238,9 @@ typedef void (if_snd_tag_free_t)(struct m_snd_tag *); */ struct ifnet { /* General book keeping of interface lists. */ - TAILQ_ENTRY(ifnet) if_link; /* all struct ifnets are chained */ + STAILQ_ENTRY(ifnet) if_link; /* all struct ifnets are chained (CK_) */ LIST_ENTRY(ifnet) if_clones; /* interfaces of a cloner */ - TAILQ_HEAD(, ifg_list) if_groups; /* linked list of groups per if */ + STAILQ_HEAD(, ifg_list) if_groups; /* linked list of groups per if (CK_) */ /* protected by if_addr_lock */ u_char if_alloctype; /* if_type at time of allocation */ @@ -297,7 +297,7 @@ struct ifnet { struct ifaddr *if_addr; /* pointer to link-level address */ void *if_hw_addr; /* hardware link-level address */ const u_int8_t *if_broadcastaddr; /* linklevel broadcast bytestring */ - struct rwlock if_afdata_lock; + struct mtx if_afdata_lock; void *if_afdata[AF_MAX]; int if_afdata_initialized; @@ -380,6 +380,7 @@ struct ifnet { * Netdump hooks to be called while dumping. */ struct netdump_methods *if_netdump_methods; + struct epoch_context if_epoch_ctx; /* * Spare fields to be added before branching a stable branch, so @@ -404,6 +405,9 @@ struct ifnet { #define IF_ADDR_WUNLOCK(if) mtx_unlock(&(if)->if_addr_lock) #define IF_ADDR_LOCK_ASSERT(if) MPASS(in_epoch() || mtx_owned(&(if)->if_addr_lock)) #define IF_ADDR_WLOCK_ASSERT(if) mtx_assert(&(if)->if_addr_lock, MA_OWNED) +#define NET_EPOCH_ENTER() epoch_enter_preempt(net_epoch_preempt) +#define NET_EPOCH_EXIT() epoch_exit_preempt(net_epoch_preempt) + /* * Function variations on locking macros intended to be used by loadable @@ -448,18 +452,18 @@ struct ifg_group { char ifg_group[IFNAMSIZ]; u_int ifg_refcnt; void *ifg_pf_kif; - TAILQ_HEAD(, ifg_member) ifg_members; - TAILQ_ENTRY(ifg_group) ifg_next; + STAILQ_HEAD(, ifg_member) ifg_members; /* (CK_) */ + STAILQ_ENTRY(ifg_group) ifg_next; /* (CK_) */ }; struct ifg_member { - TAILQ_ENTRY(ifg_member) ifgm_next; + STAILQ_ENTRY(ifg_member) ifgm_next; /* (CK_) */ struct ifnet *ifgm_ifp; }; struct ifg_list { struct ifg_group *ifgl_group; - TAILQ_ENTRY(ifg_list) ifgl_next; + STAILQ_ENTRY(ifg_list) ifgl_next; /* (CK_) */ }; #ifdef _SYS_EVENTHANDLER_H_ @@ -475,21 +479,21 @@ EVENTHANDLER_DECLARE(group_change_event, group_change_event_handler_t); #endif /* _SYS_EVENTHANDLER_H_ */ #define IF_AFDATA_LOCK_INIT(ifp) \ - rw_init(&(ifp)->if_afdata_lock, "if_afdata") + mtx_init(&(ifp)->if_afdata_lock, "if_afdata", NULL, MTX_DEF) -#define IF_AFDATA_WLOCK(ifp) rw_wlock(&(ifp)->if_afdata_lock) -#define IF_AFDATA_RLOCK(ifp) rw_rlock(&(ifp)->if_afdata_lock) -#define IF_AFDATA_WUNLOCK(ifp) rw_wunlock(&(ifp)->if_afdata_lock) -#define IF_AFDATA_RUNLOCK(ifp) rw_runlock(&(ifp)->if_afdata_lock) +#define IF_AFDATA_WLOCK(ifp) mtx_lock(&(ifp)->if_afdata_lock) +#define IF_AFDATA_RLOCK(ifp) epoch_enter_preempt(net_epoch_preempt) +#define IF_AFDATA_WUNLOCK(ifp) mtx_unlock(&(ifp)->if_afdata_lock) +#define IF_AFDATA_RUNLOCK(ifp) epoch_exit_preempt(net_epoch_preempt) #define IF_AFDATA_LOCK(ifp) IF_AFDATA_WLOCK(ifp) #define IF_AFDATA_UNLOCK(ifp) IF_AFDATA_WUNLOCK(ifp) -#define IF_AFDATA_TRYLOCK(ifp) rw_try_wlock(&(ifp)->if_afdata_lock) -#define IF_AFDATA_DESTROY(ifp) rw_destroy(&(ifp)->if_afdata_lock) +#define IF_AFDATA_TRYLOCK(ifp) mtx_trylock(&(ifp)->if_afdata_lock) +#define IF_AFDATA_DESTROY(ifp) mtx_destroy(&(ifp)->if_afdata_lock) -#define IF_AFDATA_LOCK_ASSERT(ifp) rw_assert(&(ifp)->if_afdata_lock, RA_LOCKED) -#define IF_AFDATA_RLOCK_ASSERT(ifp) rw_assert(&(ifp)->if_afdata_lock, RA_RLOCKED) -#define IF_AFDATA_WLOCK_ASSERT(ifp) rw_assert(&(ifp)->if_afdata_lock, RA_WLOCKED) -#define IF_AFDATA_UNLOCK_ASSERT(ifp) rw_assert(&(ifp)->if_afdata_lock, RA_UNLOCKED) +#define IF_AFDATA_LOCK_ASSERT(ifp) MPASS(in_epoch() || mtx_owned(&(ifp)->if_afdata_lock)) +#define IF_AFDATA_RLOCK_ASSERT(ifp) MPASS(in_epoch()); +#define IF_AFDATA_WLOCK_ASSERT(ifp) mtx_assert(&(ifp)->if_afdata_lock, MA_OWNED) +#define IF_AFDATA_UNLOCK_ASSERT(ifp) mtx_assert(&(ifp)->if_afdata_lock, MA_NOTOWNED) /* * 72 was chosen below because it is the size of a TCP/IP @@ -569,16 +573,16 @@ extern struct sx ifnet_sxlock; * write, but also whether it was acquired with sleep support or not. */ #define IFNET_RLOCK_ASSERT() sx_assert(&ifnet_sxlock, SA_SLOCKED) -#define IFNET_RLOCK_NOSLEEP_ASSERT() rw_assert(&ifnet_rwlock, RA_RLOCKED) +#define IFNET_RLOCK_NOSLEEP_ASSERT() MPASS(in_epoch()) #define IFNET_WLOCK_ASSERT() do { \ sx_assert(&ifnet_sxlock, SA_XLOCKED); \ rw_assert(&ifnet_rwlock, RA_WLOCKED); \ } while (0) #define IFNET_RLOCK() sx_slock(&ifnet_sxlock) -#define IFNET_RLOCK_NOSLEEP() rw_rlock(&ifnet_rwlock) +#define IFNET_RLOCK_NOSLEEP() epoch_enter_preempt(net_epoch_preempt) #define IFNET_RUNLOCK() sx_sunlock(&ifnet_sxlock) -#define IFNET_RUNLOCK_NOSLEEP() rw_runlock(&ifnet_rwlock) +#define IFNET_RUNLOCK_NOSLEEP() epoch_exit_preempt(net_epoch_preempt) /* * Look up an ifnet given its index; the _ref variant also acquires a diff --git a/sys/net/route.c b/sys/net/route.c index 57b0c51927ad..e51d50e940aa 100644 --- a/sys/net/route.c +++ b/sys/net/route.c @@ -594,12 +594,12 @@ rtredirect_fib(struct sockaddr *dst, struct rib_head *rnh; ifa = NULL; + NET_EPOCH_ENTER(); rnh = rt_tables_get_rnh(fibnum, dst->sa_family); if (rnh == NULL) { error = EAFNOSUPPORT; goto out; } - /* verify the gateway is directly reachable */ if ((ifa = ifa_ifwithnet(gateway, 0, fibnum)) == NULL) { error = ENETUNREACH; @@ -687,7 +687,8 @@ rtredirect_fib(struct sockaddr *dst, done: if (rt) RTFREE_LOCKED(rt); -out: + out: + NET_EPOCH_EXIT(); if (error) V_rtstat.rts_badredirect++; else if (stat != NULL) @@ -698,8 +699,6 @@ rtredirect_fib(struct sockaddr *dst, info.rti_info[RTAX_NETMASK] = netmask; info.rti_info[RTAX_AUTHOR] = src; rt_missmsg_fib(RTM_REDIRECT, &info, flags, error, fibnum); - if (ifa != NULL) - ifa_free(ifa); } /* @@ -730,6 +729,7 @@ ifa_ifwithroute(int flags, const struct sockaddr *dst, struct sockaddr *gateway, struct ifaddr *ifa; int not_found = 0; + MPASS(in_epoch()); if ((flags & RTF_GATEWAY) == 0) { /* * If we are adding a route to an interface, @@ -758,7 +758,7 @@ ifa_ifwithroute(int flags, const struct sockaddr *dst, struct sockaddr *gateway, rt = rtalloc1_fib(gateway, 0, flags, fibnum); if (rt == NULL) - return (NULL); + goto out; /* * dismiss a gateway that is reachable only * through the default router @@ -777,21 +777,19 @@ ifa_ifwithroute(int flags, const struct sockaddr *dst, struct sockaddr *gateway, } if (!not_found && rt->rt_ifa != NULL) { ifa = rt->rt_ifa; - ifa_ref(ifa); } RT_REMREF(rt); RT_UNLOCK(rt); if (not_found || ifa == NULL) - return (NULL); + goto out; } if (ifa->ifa_addr->sa_family != dst->sa_family) { struct ifaddr *oifa = ifa; ifa = ifaof_ifpforaddr(dst, ifa->ifa_ifp); if (ifa == NULL) ifa = oifa; - else - ifa_free(oifa); } + out: return (ifa); } @@ -1283,11 +1281,11 @@ rt_getifa_fib(struct rt_addrinfo *info, u_int fibnum) * ifp may be specified by sockaddr_dl * when protocol address is ambiguous. */ + NET_EPOCH_ENTER(); if (info->rti_ifp == NULL && ifpaddr != NULL && ifpaddr->sa_family == AF_LINK && (ifa = ifa_ifwithnet(ifpaddr, 0, fibnum)) != NULL) { info->rti_ifp = ifa->ifa_ifp; - ifa_free(ifa); } if (info->rti_ifa == NULL && ifaaddr != NULL) info->rti_ifa = ifa_ifwithaddr(ifaaddr); @@ -1308,8 +1306,10 @@ rt_getifa_fib(struct rt_addrinfo *info, u_int fibnum) if ((ifa = info->rti_ifa) != NULL) { if (info->rti_ifp == NULL) info->rti_ifp = ifa->ifa_ifp; + ifa_ref(info->rti_ifa); } else error = ENETUNREACH; + NET_EPOCH_EXIT(); return (error); } diff --git a/sys/net/rtsock.c b/sys/net/rtsock.c index 4042c5f6a24b..6fb2c7013857 100644 --- a/sys/net/rtsock.c +++ b/sys/net/rtsock.c @@ -784,12 +784,14 @@ route_output(struct mbuf *m, struct socket *so, ...) rt->rt_ifp->if_type == IFT_PROPVIRTUAL) { struct ifaddr *ifa; + NET_EPOCH_ENTER(); ifa = ifa_ifwithnet(info.rti_info[RTAX_DST], 1, RT_ALL_FIBS); if (ifa != NULL) rt_maskedcopy(ifa->ifa_addr, &laddr, ifa->ifa_netmask); + NET_EPOCH_EXIT(); } else rt_maskedcopy(rt->rt_ifa->ifa_addr, &laddr, @@ -1736,7 +1738,7 @@ sysctl_iflist(int af, struct walkarg *w) bzero((caddr_t)&info, sizeof(info)); bzero(&ifd, sizeof(ifd)); IFNET_RLOCK_NOSLEEP(); - TAILQ_FOREACH(ifp, &V_ifnet, if_link) { + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { if (w->w_arg && w->w_arg != ifp->if_index) continue; if_data_copy(ifp, &ifd); @@ -1806,7 +1808,7 @@ sysctl_ifmalist(int af, struct walkarg *w) bzero((caddr_t)&info, sizeof(info)); IFNET_RLOCK_NOSLEEP(); - TAILQ_FOREACH(ifp, &V_ifnet, if_link) { + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { if (w->w_arg && w->w_arg != ifp->if_index) continue; ifa = ifp->if_addr; diff --git a/sys/netinet/igmp.c b/sys/netinet/igmp.c index e3c8ece53ad3..55545aacf906 100644 --- a/sys/netinet/igmp.c +++ b/sys/netinet/igmp.c @@ -1218,11 +1218,11 @@ igmp_input_v1_report(struct ifnet *ifp, /*const*/ struct ip *ip, * Replace 0.0.0.0 with the subnet address if told to do so. */ if (V_igmp_recvifkludge && in_nullhost(ip->ip_src)) { + NET_EPOCH_ENTER(); IFP_TO_IA(ifp, ia, &in_ifa_tracker); - if (ia != NULL) { + if (ia != NULL) ip->ip_src.s_addr = htonl(ia->ia_subnet); - ifa_free(&ia->ia_ifa); - } + NET_EPOCH_EXIT(); } CTR3(KTR_IGMPV3, "process v1 report 0x%08x on ifp %p(%s)", @@ -1315,24 +1315,23 @@ igmp_input_v2_report(struct ifnet *ifp, /*const*/ struct ip *ip, * leave requires knowing that we are the only member of a * group. */ + NET_EPOCH_ENTER(); IFP_TO_IA(ifp, ia, &in_ifa_tracker); if (ia != NULL && in_hosteq(ip->ip_src, IA_SIN(ia)->sin_addr)) { - ifa_free(&ia->ia_ifa); + NET_EPOCH_EXIT(); return (0); } IGMPSTAT_INC(igps_rcv_reports); if (ifp->if_flags & IFF_LOOPBACK) { - if (ia != NULL) - ifa_free(&ia->ia_ifa); + NET_EPOCH_EXIT(); return (0); } if (!IN_MULTICAST(ntohl(igmp->igmp_group.s_addr)) || !in_hosteq(igmp->igmp_group, ip->ip_dst)) { - if (ia != NULL) - ifa_free(&ia->ia_ifa); + NET_EPOCH_EXIT(); IGMPSTAT_INC(igps_rcv_badreports); return (EINVAL); } @@ -1348,8 +1347,7 @@ igmp_input_v2_report(struct ifnet *ifp, /*const*/ struct ip *ip, if (ia != NULL) ip->ip_src.s_addr = htonl(ia->ia_subnet); } - if (ia != NULL) - ifa_free(&ia->ia_ifa); + NET_EPOCH_EXIT(); CTR3(KTR_IGMPV3, "process v2 report 0x%08x on ifp %p(%s)", ntohl(igmp->igmp_group.s_addr), ifp, ifp->if_xname); @@ -3526,11 +3524,11 @@ igmp_v3_encap_report(struct ifnet *ifp, struct mbuf *m) if (m->m_flags & M_IGMP_LOOP) { struct in_ifaddr *ia; + NET_EPOCH_ENTER(); IFP_TO_IA(ifp, ia, &in_ifa_tracker); - if (ia != NULL) { + if (ia != NULL) ip->ip_src = ia->ia_addr.sin_addr; - ifa_free(&ia->ia_ifa); - } + NET_EPOCH_EXIT(); } ip->ip_dst.s_addr = htonl(INADDR_ALLRPTS_GROUP); diff --git a/sys/netinet/in.c b/sys/netinet/in.c index e9b8f5248fe5..995b7517ed63 100644 --- a/sys/netinet/in.c +++ b/sys/netinet/in.c @@ -912,7 +912,7 @@ in_ifscrub_all(void) struct ifaliasreq ifr; IFNET_RLOCK(); - TAILQ_FOREACH(ifp, &V_ifnet, if_link) { + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { /* Cannot lock here - lock recursion. */ /* IF_ADDR_RLOCK(ifp); */ CK_STAILQ_FOREACH_SAFE(ifa, &ifp->if_addrhead, ifa_link, nifa) { @@ -1055,9 +1055,11 @@ struct in_llentry { * Do actual deallocation of @lle. */ static void -in_lltable_destroy_lle_unlocked(struct llentry *lle) +in_lltable_destroy_lle_unlocked(epoch_context_t ctx) { + struct llentry *lle; + lle = __containerof(ctx, struct llentry, lle_epoch_ctx); LLE_LOCK_DESTROY(lle); LLE_REQ_DESTROY(lle); free(lle, M_LLTABLE); @@ -1085,7 +1087,7 @@ in_lltable_destroy_lle(struct llentry *lle) { LLE_WUNLOCK(lle); - in_lltable_destroy_lle_unlocked(lle); + epoch_call(net_epoch_preempt, &lle->lle_epoch_ctx, in_lltable_destroy_lle_unlocked); } static struct llentry * @@ -1294,7 +1296,7 @@ in_lltable_find_dst(struct lltable *llt, struct in_addr dst) hashidx = in_lltable_hash_dst(dst, llt->llt_hsize); lleh = &llt->lle_head[hashidx]; - LIST_FOREACH(lle, lleh, lle_next) { + CK_LIST_FOREACH(lle, lleh, lle_next) { if (lle->la_flags & LLE_DELETED) continue; if (lle->r_l3addr.addr4.s_addr == dst.s_addr) @@ -1350,7 +1352,7 @@ in_lltable_alloc(struct lltable *llt, u_int flags, const struct sockaddr *l3addr linkhdrsize = LLE_MAX_LINKHDR; if (lltable_calc_llheader(ifp, AF_INET, IF_LLADDR(ifp), linkhdr, &linkhdrsize, &lladdr_off) != 0) { - in_lltable_destroy_lle_unlocked(lle); + epoch_call(net_epoch_preempt, &lle->lle_epoch_ctx, in_lltable_destroy_lle_unlocked); return (NULL); } lltable_set_entry_addr(ifp, lle, linkhdr, linkhdrsize, diff --git a/sys/netinet/in_mcast.c b/sys/netinet/in_mcast.c index b6f0f367abcd..0219f722ff26 100644 --- a/sys/netinet/in_mcast.c +++ b/sys/netinet/in_mcast.c @@ -1876,12 +1876,12 @@ inp_getmoptions(struct inpcb *inp, struct sockopt *sopt) mreqn.imr_address = imo->imo_multicast_addr; } else if (ifp != NULL) { mreqn.imr_ifindex = ifp->if_index; + NET_EPOCH_ENTER(); IFP_TO_IA(ifp, ia, &in_ifa_tracker); - if (ia != NULL) { + if (ia != NULL) mreqn.imr_address = IA_SIN(ia)->sin_addr; - ifa_free(&ia->ia_ifa); - } + NET_EPOCH_EXIT(); } } INP_WUNLOCK(inp); diff --git a/sys/netinet/in_pcb.c b/sys/netinet/in_pcb.c index e197b4ac4b36..d8b203441388 100644 --- a/sys/netinet/in_pcb.c +++ b/sys/netinet/in_pcb.c @@ -794,7 +794,6 @@ in_pcbladdr(struct inpcb *inp, struct in_addr *faddr, struct in_addr *laddr, int error; KASSERT(laddr != NULL, ("%s: laddr NULL", __func__)); - /* * Bypass source address selection and use the primary jail IP * if requested. @@ -827,28 +826,30 @@ in_pcbladdr(struct inpcb *inp, struct in_addr *faddr, struct in_addr *laddr, * network and try to find a corresponding interface to take * the source address from. */ + NET_EPOCH_ENTER(); if (sro.ro_rt == NULL || sro.ro_rt->rt_ifp == NULL) { struct in_ifaddr *ia; struct ifnet *ifp; ia = ifatoia(ifa_ifwithdstaddr((struct sockaddr *)sin, inp->inp_socket->so_fibnum)); - if (ia == NULL) + if (ia == NULL) { ia = ifatoia(ifa_ifwithnet((struct sockaddr *)sin, 0, inp->inp_socket->so_fibnum)); + + } if (ia == NULL) { + printf("ifa_ifwithnet failed\n"); error = ENETUNREACH; goto done; } if (cred == NULL || !prison_flag(cred, PR_IP4)) { laddr->s_addr = ia->ia_addr.sin_addr.s_addr; - ifa_free(&ia->ia_ifa); goto done; } ifp = ia->ia_ifp; - ifa_free(&ia->ia_ifa); ia = NULL; IF_ADDR_RLOCK(ifp); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { @@ -964,7 +965,6 @@ in_pcbladdr(struct inpcb *inp, struct in_addr *faddr, struct in_addr *laddr, goto done; } laddr->s_addr = ia->ia_addr.sin_addr.s_addr; - ifa_free(&ia->ia_ifa); goto done; } @@ -973,7 +973,6 @@ in_pcbladdr(struct inpcb *inp, struct in_addr *faddr, struct in_addr *laddr, struct ifnet *ifp; ifp = ia->ia_ifp; - ifa_free(&ia->ia_ifa); ia = NULL; IF_ADDR_RLOCK(ifp); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { @@ -1002,6 +1001,7 @@ in_pcbladdr(struct inpcb *inp, struct in_addr *faddr, struct in_addr *laddr, } done: + NET_EPOCH_EXIT(); if (sro.ro_rt != NULL) RTFREE(sro.ro_rt); return (error); diff --git a/sys/netinet/in_var.h b/sys/netinet/in_var.h index b674505fe932..5b7a464bd3e8 100644 --- a/sys/netinet/in_var.h +++ b/sys/netinet/in_var.h @@ -176,8 +176,6 @@ do { \ (ia) != NULL && (ia)->ia_ifp != (ifp); \ (ia) = CK_STAILQ_NEXT((ia), ia_link)) \ continue; \ - if ((ia) != NULL) \ - ifa_ref(&(ia)->ia_ifa); \ IN_IFADDR_RUNLOCK((t)); \ } while (0) diff --git a/sys/netinet/ip_divert.c b/sys/netinet/ip_divert.c index 72f0d94e209a..faf9aa0b558f 100644 --- a/sys/netinet/ip_divert.c +++ b/sys/netinet/ip_divert.c @@ -468,13 +468,15 @@ div_output(struct socket *so, struct mbuf *m, struct sockaddr_in *sin, bzero(sin->sin_zero, sizeof(sin->sin_zero)); sin->sin_port = 0; + NET_EPOCH_ENTER(); ifa = ifa_ifwithaddr((struct sockaddr *) sin); if (ifa == NULL) { error = EADDRNOTAVAIL; + NET_EPOCH_EXIT(); goto cantsend; } m->m_pkthdr.rcvif = ifa->ifa_ifp; - ifa_free(ifa); + NET_EPOCH_EXIT(); } #ifdef MAC mac_socket_create_mbuf(so, m); diff --git a/sys/netinet/ip_icmp.c b/sys/netinet/ip_icmp.c index 4bc6a354b2e1..a657c59dda13 100644 --- a/sys/netinet/ip_icmp.c +++ b/sys/netinet/ip_icmp.c @@ -406,6 +406,7 @@ icmp_input(struct mbuf **mp, int *offp, int proto) inet_ntoa_r(ip->ip_dst, dstbuf), icmplen); } #endif + NET_EPOCH_ENTER(); if (icmplen < ICMP_MINLEN) { ICMPSTAT_INC(icps_tooshort); goto freeit; @@ -413,6 +414,7 @@ icmp_input(struct mbuf **mp, int *offp, int proto) i = hlen + min(icmplen, ICMP_ADVLENMIN); if (m->m_len < i && (m = m_pullup(m, i)) == NULL) { ICMPSTAT_INC(icps_tooshort); + NET_EPOCH_EXIT(); return (IPPROTO_DONE); } ip = mtod(m, struct ip *); @@ -530,6 +532,7 @@ icmp_input(struct mbuf **mp, int *offp, int proto) if (m->m_len < i && (m = m_pullup(m, i)) == NULL) { /* This should actually not happen */ ICMPSTAT_INC(icps_tooshort); + NET_EPOCH_EXIT(); return (IPPROTO_DONE); } ip = mtod(m, struct ip *); @@ -605,10 +608,8 @@ icmp_input(struct mbuf **mp, int *offp, int proto) (struct sockaddr *)&icmpdst, m->m_pkthdr.rcvif); if (ia == NULL) break; - if (ia->ia_ifp == NULL) { - ifa_free(&ia->ia_ifa); + if (ia->ia_ifp == NULL) break; - } icp->icmp_type = ICMP_MASKREPLY; if (V_icmpmaskfake == 0) icp->icmp_mask = ia->ia_sockmask.sin_addr.s_addr; @@ -620,11 +621,11 @@ icmp_input(struct mbuf **mp, int *offp, int proto) else if (ia->ia_ifp->if_flags & IFF_POINTOPOINT) ip->ip_src = satosin(&ia->ia_dstaddr)->sin_addr; } - ifa_free(&ia->ia_ifa); reflect: ICMPSTAT_INC(icps_reflect); ICMPSTAT_INC(icps_outhist[icp->icmp_type]); icmp_reflect(m); + NET_EPOCH_EXIT(); return (IPPROTO_DONE); case ICMP_REDIRECT: @@ -701,11 +702,13 @@ icmp_input(struct mbuf **mp, int *offp, int proto) } raw: + NET_EPOCH_EXIT(); *mp = m; rip_input(mp, offp, proto); return (IPPROTO_DONE); freeit: + NET_EPOCH_EXIT(); m_freem(m); return (IPPROTO_DONE); } diff --git a/sys/netinet/ip_input.c b/sys/netinet/ip_input.c index fb13acfcc64c..94ebf22cd6b0 100644 --- a/sys/netinet/ip_input.c +++ b/sys/netinet/ip_input.c @@ -399,7 +399,7 @@ ip_destroy(void *unused __unused) /* Make sure the IPv4 routes are gone as well. */ IFNET_RLOCK(); - TAILQ_FOREACH(ifp, &V_ifnet, if_link) + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) rt_flushifroutes_af(ifp, AF_INET); IFNET_RUNLOCK(); @@ -977,9 +977,9 @@ ip_forward(struct mbuf *m, int srcrt) #else in_rtalloc_ign(&ro, 0, M_GETFIB(m)); #endif + NET_EPOCH_ENTER(); if (ro.ro_rt != NULL) { ia = ifatoia(ro.ro_rt->rt_ifa); - ifa_ref(&ia->ia_ifa); } else ia = NULL; /* @@ -1025,7 +1025,7 @@ ip_forward(struct mbuf *m, int srcrt) m_freem(mcopy); if (error != EINPROGRESS) IPSTAT_INC(ips_cantforward); - return; + goto out; } /* No IPsec processing required */ } @@ -1078,16 +1078,12 @@ ip_forward(struct mbuf *m, int srcrt) else { if (mcopy) m_freem(mcopy); - if (ia != NULL) - ifa_free(&ia->ia_ifa); - return; + goto out; } } - if (mcopy == NULL) { - if (ia != NULL) - ifa_free(&ia->ia_ifa); - return; - } + if (mcopy == NULL) + goto out; + switch (error) { @@ -1129,13 +1125,11 @@ ip_forward(struct mbuf *m, int srcrt) case ENOBUFS: case EACCES: /* ipfw denied packet */ m_freem(mcopy); - if (ia != NULL) - ifa_free(&ia->ia_ifa); - return; + goto out; } - if (ia != NULL) - ifa_free(&ia->ia_ifa); icmp_error(mcopy, type, code, dest.s_addr, mtu); + out: + NET_EPOCH_EXIT(); } #define CHECK_SO_CT(sp, ct) \ diff --git a/sys/netinet/ip_mroute.c b/sys/netinet/ip_mroute.c index 20fc88b20063..fbf298fa54a0 100644 --- a/sys/netinet/ip_mroute.c +++ b/sys/netinet/ip_mroute.c @@ -878,13 +878,15 @@ add_vif(struct vifctl *vifcp) ifp = NULL; } else { sin.sin_addr = vifcp->vifc_lcl_addr; + NET_EPOCH_ENTER(); ifa = ifa_ifwithaddr((struct sockaddr *)&sin); if (ifa == NULL) { + NET_EPOCH_EXIT(); VIF_UNLOCK(); return EADDRNOTAVAIL; } ifp = ifa->ifa_ifp; - ifa_free(ifa); + NET_EPOCH_EXIT(); } if ((vifcp->vifc_flags & VIFF_TUNNEL) != 0) { diff --git a/sys/netinet/ip_options.c b/sys/netinet/ip_options.c index 9a8da231bc7d..dd0914acb0b9 100644 --- a/sys/netinet/ip_options.c +++ b/sys/netinet/ip_options.c @@ -110,6 +110,7 @@ ip_dooptions(struct mbuf *m, int pass) struct nhop4_extended nh_ext; struct sockaddr_in ipaddr = { sizeof(ipaddr), AF_INET }; + NET_EPOCH_ENTER(); /* Ignore or reject packets with IP options. */ if (V_ip_doopts == 0) return 0; @@ -224,6 +225,7 @@ ip_dooptions(struct mbuf *m, int pass) #endif IPSTAT_INC(ips_cantforward); m_freem(m); + NET_EPOCH_EXIT(); return (1); } } @@ -250,7 +252,6 @@ ip_dooptions(struct mbuf *m, int pass) memcpy(cp + off, &(IA_SIN(ia)->sin_addr), sizeof(struct in_addr)); - ifa_free(&ia->ia_ifa); } else { /* XXX MRT 0 for routing */ if (fib4_lookup_nh_ext(M_GETFIB(m), @@ -298,7 +299,6 @@ ip_dooptions(struct mbuf *m, int pass) if ((ia = (INA)ifa_ifwithaddr((SA)&ipaddr)) != NULL) { memcpy(cp + off, &(IA_SIN(ia)->sin_addr), sizeof(struct in_addr)); - ifa_free(&ia->ia_ifa); } else if (fib4_lookup_nh_ext(M_GETFIB(m), ipaddr.sin_addr, 0, 0, &nh_ext) == 0) { memcpy(cp + off, &nh_ext.nh_src, @@ -353,7 +353,6 @@ ip_dooptions(struct mbuf *m, int pass) continue; (void)memcpy(sin, &IA_SIN(ia)->sin_addr, sizeof(struct in_addr)); - ifa_free(&ia->ia_ifa); cp[IPOPT_OFFSET] += sizeof(struct in_addr); off += sizeof(struct in_addr); break; @@ -381,12 +380,14 @@ ip_dooptions(struct mbuf *m, int pass) cp[IPOPT_OFFSET] += sizeof(uint32_t); } } + NET_EPOCH_EXIT(); if (forward && V_ipforwarding) { ip_forward(m, 1); return (1); } return (0); bad: + NET_EPOCH_EXIT(); icmp_error(m, type, code, 0, 0); IPSTAT_INC(ips_badoptions); return (1); diff --git a/sys/netinet/ip_output.c b/sys/netinet/ip_output.c index 1a594cd1b680..b2580cc08638 100644 --- a/sys/netinet/ip_output.c +++ b/sys/netinet/ip_output.c @@ -225,7 +225,6 @@ ip_output(struct mbuf *m, struct mbuf *opt, struct route *ro, int flags, struct route iproute; struct rtentry *rte; /* cache for ro->ro_rt */ uint32_t fibnum; - int have_ia_ref; #if defined(IPSEC) || defined(IPSEC_SUPPORT) int no_route_but_check_spd = 0; #endif @@ -281,6 +280,7 @@ ip_output(struct mbuf *m, struct mbuf *opt, struct route *ro, int flags, dst->sin_len = sizeof(*dst); dst->sin_addr = ip->ip_dst; } + NET_EPOCH_ENTER(); again: /* * Validate route against routing table additions; @@ -306,7 +306,6 @@ ip_output(struct mbuf *m, struct mbuf *opt, struct route *ro, int flags, rte = NULL; } ia = NULL; - have_ia_ref = 0; /* * If routing to interface only, short circuit routing lookup. * The use of an all-ones broadcast address implies this; an @@ -322,7 +321,6 @@ ip_output(struct mbuf *m, struct mbuf *opt, struct route *ro, int flags, error = ENETUNREACH; goto bad; } - have_ia_ref = 1; ip->ip_dst.s_addr = INADDR_BROADCAST; dst->sin_addr = ip->ip_dst; ifp = ia->ia_ifp; @@ -337,7 +335,6 @@ ip_output(struct mbuf *m, struct mbuf *opt, struct route *ro, int flags, error = ENETUNREACH; goto bad; } - have_ia_ref = 1; ifp = ia->ia_ifp; ip->ip_ttl = 1; isbroadcast = ifp->if_flags & IFF_BROADCAST ? @@ -350,8 +347,6 @@ ip_output(struct mbuf *m, struct mbuf *opt, struct route *ro, int flags, */ ifp = imo->imo_multicast_ifp; IFP_TO_IA(ifp, ia, &in_ifa_tracker); - if (ia) - have_ia_ref = 1; isbroadcast = 0; /* fool gcc */ } else { /* @@ -579,8 +574,6 @@ ip_output(struct mbuf *m, struct mbuf *opt, struct route *ro, int flags, case -1: /* Need to try again */ /* Reset everything for a new round */ RO_RTFREE(ro); - if (have_ia_ref) - ifa_free(&ia->ia_ifa); ro->ro_prepend = NULL; rte = NULL; gw = dst; @@ -735,10 +728,9 @@ ip_output(struct mbuf *m, struct mbuf *opt, struct route *ro, int flags, * calling RTFREE on it again. */ ro->ro_rt = NULL; - if (have_ia_ref) - ifa_free(&ia->ia_ifa); + NET_EPOCH_EXIT(); return (error); -bad: + bad: m_freem(m); goto done; } diff --git a/sys/netinet/netdump/netdump_client.c b/sys/netinet/netdump/netdump_client.c index c6174b9c1fec..a662fa86e9c7 100644 --- a/sys/netinet/netdump/netdump_client.c +++ b/sys/netinet/netdump/netdump_client.c @@ -1069,7 +1069,7 @@ netdump_configure(struct netdump_conf *conf, struct thread *td) return (EINVAL); } IFNET_RLOCK_NOSLEEP(); - TAILQ_FOREACH(ifp, &V_ifnet, if_link) { + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { if (strcmp(ifp->if_xname, conf->ndc_iface) == 0) break; } diff --git a/sys/netinet/raw_ip.c b/sys/netinet/raw_ip.c index 6c90418e7573..d56003bd0170 100644 --- a/sys/netinet/raw_ip.c +++ b/sys/netinet/raw_ip.c @@ -929,7 +929,7 @@ rip_bind(struct socket *so, struct sockaddr *nam, struct thread *td) inp = sotoinpcb(so); KASSERT(inp != NULL, ("rip_bind: inp == NULL")); - if (TAILQ_EMPTY(&V_ifnet) || + if (CK_STAILQ_EMPTY(&V_ifnet) || (addr->sin_family != AF_INET && addr->sin_family != AF_IMPLINK) || (addr->sin_addr.s_addr && (inp->inp_flags & INP_BINDANY) == 0 && @@ -954,7 +954,7 @@ rip_connect(struct socket *so, struct sockaddr *nam, struct thread *td) if (nam->sa_len != sizeof(*addr)) return (EINVAL); - if (TAILQ_EMPTY(&V_ifnet)) + if (CK_STAILQ_EMPTY(&V_ifnet)) return (EADDRNOTAVAIL); if (addr->sin_family != AF_INET && addr->sin_family != AF_IMPLINK) return (EAFNOSUPPORT); diff --git a/sys/netinet/sctp_bsd_addr.c b/sys/netinet/sctp_bsd_addr.c index 2a25a5d2ec45..4b17bb50ba3f 100644 --- a/sys/netinet/sctp_bsd_addr.c +++ b/sys/netinet/sctp_bsd_addr.c @@ -207,7 +207,7 @@ sctp_init_ifns_for_vrf(int vrfid) #endif IFNET_RLOCK(); - TAILQ_FOREACH(ifn, &MODULE_GLOBAL(ifnet), if_link) { + CK_STAILQ_FOREACH(ifn, &MODULE_GLOBAL(ifnet), if_link) { if (sctp_is_desired_interface_type(ifn) == 0) { /* non desired type */ continue; @@ -360,7 +360,7 @@ void struct ifaddr *ifa; IFNET_RLOCK(); - TAILQ_FOREACH(ifn, &MODULE_GLOBAL(ifnet), if_link) { + CK_STAILQ_FOREACH(ifn, &MODULE_GLOBAL(ifnet), if_link) { if (!(*pred) (ifn)) { continue; } diff --git a/sys/netinet6/icmp6.c b/sys/netinet6/icmp6.c index 8c5026b41bf5..02f92f845293 100644 --- a/sys/netinet6/icmp6.c +++ b/sys/netinet6/icmp6.c @@ -1677,7 +1677,7 @@ ni6_addrs(struct icmp6_nodeinfo *ni6, struct mbuf *m, struct ifnet **ifpp, } IFNET_RLOCK_NOSLEEP(); - TAILQ_FOREACH(ifp, &V_ifnet, if_link) { + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { addrsofif = 0; IF_ADDR_RLOCK(ifp); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { @@ -1761,10 +1761,10 @@ ni6_store_addrs(struct icmp6_nodeinfo *ni6, struct icmp6_nodeinfo *nni6, return (0); /* needless to copy */ IFNET_RLOCK_NOSLEEP(); - ifp = ifp0 ? ifp0 : TAILQ_FIRST(&V_ifnet); + ifp = ifp0 ? ifp0 : CK_STAILQ_FIRST(&V_ifnet); again: - for (; ifp; ifp = TAILQ_NEXT(ifp, if_link)) { + for (; ifp; ifp = CK_STAILQ_NEXT(ifp, if_link)) { IF_ADDR_RLOCK(ifp); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET6) diff --git a/sys/netinet6/in6.c b/sys/netinet6/in6.c index 848fb8ad8cdd..cff28e063619 100644 --- a/sys/netinet6/in6.c +++ b/sys/netinet6/in6.c @@ -1973,7 +1973,7 @@ in6_setmaxmtu(void) struct ifnet *ifp; IFNET_RLOCK_NOSLEEP(); - TAILQ_FOREACH(ifp, &V_ifnet, if_link) { + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { /* this function can be called during ifnet initialization */ if (!ifp->if_afdata[AF_INET6]) continue; @@ -2045,9 +2045,11 @@ struct in6_llentry { * Do actual deallocation of @lle. */ static void -in6_lltable_destroy_lle_unlocked(struct llentry *lle) +in6_lltable_destroy_lle_unlocked(epoch_context_t ctx) { + struct llentry *lle; + lle = __containerof(ctx, struct llentry, lle_epoch_ctx); LLE_LOCK_DESTROY(lle); LLE_REQ_DESTROY(lle); free(lle, M_LLTABLE); @@ -2062,7 +2064,7 @@ in6_lltable_destroy_lle(struct llentry *lle) { LLE_WUNLOCK(lle); - in6_lltable_destroy_lle_unlocked(lle); + epoch_call(net_epoch_preempt, &lle->lle_epoch_ctx, in6_lltable_destroy_lle_unlocked); } static struct llentry * @@ -2165,11 +2167,13 @@ in6_lltable_rtcheck(struct ifnet *ifp, * Create an ND6 cache for an IPv6 neighbor * that is not covered by our own prefix. */ + NET_EPOCH_ENTER(); ifa = ifaof_ifpforaddr(l3addr, ifp); if (ifa != NULL) { - ifa_free(ifa); + NET_EPOCH_EXIT(); return 0; } + NET_EPOCH_EXIT(); log(LOG_INFO, "IPv6 address: \"%s\" is not on the network\n", ip6_sprintf(ip6buf, &sin6->sin6_addr)); return EINVAL; @@ -2231,7 +2235,7 @@ in6_lltable_find_dst(struct lltable *llt, const struct in6_addr *dst) hashidx = in6_lltable_hash_dst(dst, llt->llt_hsize); lleh = &llt->lle_head[hashidx]; - LIST_FOREACH(lle, lleh, lle_next) { + CK_LIST_FOREACH(lle, lleh, lle_next) { if (lle->la_flags & LLE_DELETED) continue; if (IN6_ARE_ADDR_EQUAL(&lle->r_l3addr.addr6, dst)) @@ -2286,7 +2290,7 @@ in6_lltable_alloc(struct lltable *llt, u_int flags, linkhdrsize = LLE_MAX_LINKHDR; if (lltable_calc_llheader(ifp, AF_INET6, IF_LLADDR(ifp), linkhdr, &linkhdrsize, &lladdr_off) != 0) { - in6_lltable_destroy_lle_unlocked(lle); + epoch_call(net_epoch_preempt, &lle->lle_epoch_ctx, in6_lltable_destroy_lle_unlocked); return (NULL); } lltable_set_entry_addr(ifp, lle, linkhdr, linkhdrsize, diff --git a/sys/netinet6/in6_ifattach.c b/sys/netinet6/in6_ifattach.c index b427aea84cc0..e20228dce485 100644 --- a/sys/netinet6/in6_ifattach.c +++ b/sys/netinet6/in6_ifattach.c @@ -384,7 +384,7 @@ get_ifid(struct ifnet *ifp0, struct ifnet *altifp, /* next, try to get it from some other hardware interface */ IFNET_RLOCK_NOSLEEP(); - TAILQ_FOREACH(ifp, &V_ifnet, if_link) { + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { if (ifp == ifp0) continue; if (in6_get_hw_ifid(ifp, in6) != 0) @@ -826,7 +826,7 @@ in6_tmpaddrtimer(void *arg) V_ip6_temp_regen_advance) * hz, in6_tmpaddrtimer, curvnet); bzero(nullbuf, sizeof(nullbuf)); - TAILQ_FOREACH(ifp, &V_ifnet, if_link) { + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { if (ifp->if_afdata[AF_INET6] == NULL) continue; ndi = ND_IFINFO(ifp); diff --git a/sys/netinet6/in6_pcb.c b/sys/netinet6/in6_pcb.c index da5201f07e3d..7567a7bb39a2 100644 --- a/sys/netinet6/in6_pcb.c +++ b/sys/netinet6/in6_pcb.c @@ -170,9 +170,11 @@ in6_pcbbind(struct inpcb *inp, struct sockaddr *nam, struct ifaddr *ifa; sin6->sin6_port = 0; /* yech... */ + NET_EPOCH_ENTER(); if ((ifa = ifa_ifwithaddr((struct sockaddr *)sin6)) == NULL && (inp->inp_flags & INP_BINDANY) == 0) { + NET_EPOCH_EXIT(); return (EADDRNOTAVAIL); } @@ -185,11 +187,10 @@ in6_pcbbind(struct inpcb *inp, struct sockaddr *nam, if (ifa != NULL && ((struct in6_ifaddr *)ifa)->ia6_flags & (IN6_IFF_ANYCAST|IN6_IFF_NOTREADY|IN6_IFF_DETACHED)) { - ifa_free(ifa); + NET_EPOCH_EXIT(); return (EADDRNOTAVAIL); } - if (ifa != NULL) - ifa_free(ifa); + NET_EPOCH_EXIT(); } if (lport) { struct inpcb *t; diff --git a/sys/netinet6/ip6_input.c b/sys/netinet6/ip6_input.c index 49746179b16f..87788cb3b6d0 100644 --- a/sys/netinet6/ip6_input.c +++ b/sys/netinet6/ip6_input.c @@ -377,7 +377,7 @@ ip6_destroy(void *unused __unused) /* Cleanup addresses. */ IFNET_RLOCK(); - TAILQ_FOREACH(ifp, &V_ifnet, if_link) { + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { /* Cannot lock here - lock recursion. */ /* IF_ADDR_LOCK(ifp); */ CK_STAILQ_FOREACH_SAFE(ifa, &ifp->if_addrhead, ifa_link, nifa) { diff --git a/sys/netinet6/nd6.c b/sys/netinet6/nd6.c index 391c96c4c205..f713759b76cc 100644 --- a/sys/netinet6/nd6.c +++ b/sys/netinet6/nd6.c @@ -2133,7 +2133,7 @@ nd6_slowtimo(void *arg) callout_reset(&V_nd6_slowtimo_ch, ND6_SLOWTIMER_INTERVAL * hz, nd6_slowtimo, curvnet); IFNET_RLOCK_NOSLEEP(); - TAILQ_FOREACH(ifp, &V_ifnet, if_link) { + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { if (ifp->if_afdata[AF_INET6] == NULL) continue; nd6if = ND_IFINFO(ifp); diff --git a/sys/netinet6/raw_ip6.c b/sys/netinet6/raw_ip6.c index b8de413593d6..a6811c68c2f4 100644 --- a/sys/netinet6/raw_ip6.c +++ b/sys/netinet6/raw_ip6.c @@ -734,23 +734,25 @@ rip6_bind(struct socket *so, struct sockaddr *nam, struct thread *td) return (EINVAL); if ((error = prison_check_ip6(td->td_ucred, &addr->sin6_addr)) != 0) return (error); - if (TAILQ_EMPTY(&V_ifnet) || addr->sin6_family != AF_INET6) + if (CK_STAILQ_EMPTY(&V_ifnet) || addr->sin6_family != AF_INET6) return (EADDRNOTAVAIL); if ((error = sa6_embedscope(addr, V_ip6_use_defzone)) != 0) return (error); + NET_EPOCH_ENTER(); if (!IN6_IS_ADDR_UNSPECIFIED(&addr->sin6_addr) && - (ifa = ifa_ifwithaddr((struct sockaddr *)addr)) == NULL) + (ifa = ifa_ifwithaddr((struct sockaddr *)addr)) == NULL) { + NET_EPOCH_EXIT(); return (EADDRNOTAVAIL); + } if (ifa != NULL && ((struct in6_ifaddr *)ifa)->ia6_flags & (IN6_IFF_ANYCAST|IN6_IFF_NOTREADY| IN6_IFF_DETACHED|IN6_IFF_DEPRECATED)) { - ifa_free(ifa); + NET_EPOCH_EXIT(); return (EADDRNOTAVAIL); } - if (ifa != NULL) - ifa_free(ifa); + NET_EPOCH_EXIT(); INP_INFO_WLOCK(&V_ripcbinfo); INP_WLOCK(inp); inp->in6p_laddr = addr->sin6_addr; @@ -772,7 +774,7 @@ rip6_connect(struct socket *so, struct sockaddr *nam, struct thread *td) if (nam->sa_len != sizeof(*addr)) return (EINVAL); - if (TAILQ_EMPTY(&V_ifnet)) + if (CK_STAILQ_EMPTY(&V_ifnet)) return (EADDRNOTAVAIL); if (addr->sin6_family != AF_INET6) return (EAFNOSUPPORT); diff --git a/sys/netpfil/pf/pf_if.c b/sys/netpfil/pf/pf_if.c index 156dc73705ea..3365c9c84a57 100644 --- a/sys/netpfil/pf/pf_if.c +++ b/sys/netpfil/pf/pf_if.c @@ -129,9 +129,9 @@ pfi_initialize_vnet(void) PF_RULES_WUNLOCK(); IFNET_RLOCK(); - TAILQ_FOREACH(ifg, &V_ifg_head, ifg_next) + CK_STAILQ_FOREACH(ifg, &V_ifg_head, ifg_next) pfi_attach_ifgroup(ifg); - TAILQ_FOREACH(ifp, &V_ifnet, if_link) + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) pfi_attach_ifnet(ifp); IFNET_RUNLOCK(); } @@ -300,7 +300,7 @@ pfi_kif_match(struct pfi_kif *rule_kif, struct pfi_kif *packet_kif) if (rule_kif->pfik_group != NULL) /* XXXGL: locking? */ - TAILQ_FOREACH(p, &packet_kif->pfik_ifp->if_groups, ifgl_next) + CK_STAILQ_FOREACH(p, &packet_kif->pfik_ifp->if_groups, ifgl_next) if (p->ifgl_group == rule_kif->pfik_group) return (1); @@ -466,7 +466,7 @@ pfi_kif_update(struct pfi_kif *kif) /* again for all groups kif is member of */ if (kif->pfik_ifp != NULL) { IF_ADDR_RLOCK(kif->pfik_ifp); - TAILQ_FOREACH(ifgl, &kif->pfik_ifp->if_groups, ifgl_next) + CK_STAILQ_FOREACH(ifgl, &kif->pfik_ifp->if_groups, ifgl_next) pfi_kif_update((struct pfi_kif *) ifgl->ifgl_group->ifg_pf_kif); IF_ADDR_RUNLOCK(kif->pfik_ifp); @@ -506,7 +506,7 @@ pfi_table_update(struct pfr_ktable *kt, struct pfi_kif *kif, int net, int flags) pfi_instance_add(kif->pfik_ifp, net, flags); else if (kif->pfik_group != NULL) { IFNET_RLOCK_NOSLEEP(); - TAILQ_FOREACH(ifgm, &kif->pfik_group->ifg_members, ifgm_next) + CK_STAILQ_FOREACH(ifgm, &kif->pfik_group->ifg_members, ifgm_next) pfi_instance_add(ifgm->ifgm_ifp, net, flags); IFNET_RUNLOCK_NOSLEEP(); } @@ -668,7 +668,7 @@ pfi_update_status(const char *name, struct pf_status *pfs) struct pfi_kif *p; struct pfi_kif_cmp key; struct ifg_member p_member, *ifgm; - TAILQ_HEAD(, ifg_member) ifg_members; + CK_STAILQ_HEAD(, ifg_member) ifg_members; int i, j, k; strlcpy(key.pfik_name, name, sizeof(key.pfik_name)); @@ -683,14 +683,14 @@ pfi_update_status(const char *name, struct pf_status *pfs) /* build a temporary list for p only */ bzero(&p_member, sizeof(p_member)); p_member.ifgm_ifp = p->pfik_ifp; - TAILQ_INIT(&ifg_members); - TAILQ_INSERT_TAIL(&ifg_members, &p_member, ifgm_next); + CK_STAILQ_INIT(&ifg_members); + CK_STAILQ_INSERT_TAIL(&ifg_members, &p_member, ifgm_next); } if (pfs) { bzero(pfs->pcounters, sizeof(pfs->pcounters)); bzero(pfs->bcounters, sizeof(pfs->bcounters)); } - TAILQ_FOREACH(ifgm, &ifg_members, ifgm_next) { + CK_STAILQ_FOREACH(ifgm, &ifg_members, ifgm_next) { if (ifgm->ifgm_ifp == NULL || ifgm->ifgm_ifp->if_pf_kif == NULL) continue; p = (struct pfi_kif *)ifgm->ifgm_ifp->if_pf_kif; diff --git a/sys/nfs/bootp_subr.c b/sys/nfs/bootp_subr.c index 8bf89850fa5c..d7657155f55d 100644 --- a/sys/nfs/bootp_subr.c +++ b/sys/nfs/bootp_subr.c @@ -409,12 +409,12 @@ bootpboot_p_iflist(void) printf("Interface list:\n"); IFNET_RLOCK(); - for (ifp = TAILQ_FIRST(&V_ifnet); + for (ifp = CK_STAILQ_FIRST(&V_ifnet); ifp != NULL; - ifp = TAILQ_NEXT(ifp, if_link)) { + ifp = CK_STAILQ_NEXT(ifp, if_link)) { for (ifa = CK_STAILQ_FIRST(&ifp->if_addrhead); ifa != NULL; - ifa = TAILQ_NEXT(ifa, ifa_link)) + ifa = CK_STAILQ_NEXT(ifa, ifa_link)) if (ifa->ifa_addr->sa_family == AF_INET) bootpboot_p_if(ifp, ifa); } @@ -1637,7 +1637,7 @@ bootpc_init(void) */ ifcnt = 0; IFNET_RLOCK(); - TAILQ_FOREACH(ifp, &V_ifnet, if_link) { + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { if ((ifp->if_flags & (IFF_LOOPBACK | IFF_POINTOPOINT | IFF_BROADCAST)) != IFF_BROADCAST) @@ -1660,7 +1660,7 @@ bootpc_init(void) retry: ifctx = STAILQ_FIRST(&gctx->interfaces); IFNET_RLOCK(); - TAILQ_FOREACH(ifp, &V_ifnet, if_link) { + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { if (ifctx == NULL) break; #ifdef BOOTP_WIRED_TO diff --git a/sys/nfs/nfs_diskless.c b/sys/nfs/nfs_diskless.c index a118e73ef75d..b621ed452e0e 100644 --- a/sys/nfs/nfs_diskless.c +++ b/sys/nfs/nfs_diskless.c @@ -224,7 +224,7 @@ nfs_setup_diskless(void) retry: CURVNET_SET(TD_TO_VNET(curthread)); IFNET_RLOCK(); - TAILQ_FOREACH(ifp, &V_ifnet, if_link) { + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family == AF_LINK) { sdl = (struct sockaddr_dl *)ifa->ifa_addr; diff --git a/sys/ofed/drivers/infiniband/core/ib_roce_gid_mgmt.c b/sys/ofed/drivers/infiniband/core/ib_roce_gid_mgmt.c index b7717200cb96..2befd51176e3 100644 --- a/sys/ofed/drivers/infiniband/core/ib_roce_gid_mgmt.c +++ b/sys/ofed/drivers/infiniband/core/ib_roce_gid_mgmt.c @@ -191,7 +191,7 @@ roce_gid_update_addr_callback(struct ib_device *device, u8 port, CURVNET_SET(ndev->if_vnet); IFNET_RLOCK(); - TAILQ_FOREACH(idev, &V_ifnet, if_link) { + CK_STAILQ_FOREACH(idev, &V_ifnet, if_link) { if (idev != ndev) { if (idev->if_type != IFT_L2VLAN) continue;