lib,zebra: use nhg_hash_entry pointer in route_entry

Replace the existing list of nexthops (via a nexthop_group
struct) in the route_entry with a direct pointer to zebra's
new shared group (from zebra_nhg.h). This allows more
direct access to that shared group and the info it carries.

Signed-off-by: Mark Stapp <mjs@voltanet.io>

18 files changed, 192 insertions, 299 deletions

diff --git a/zebra/redistribute.c b/zebra/redistribute.c
index 98603c969..6aa52bcb6 100644
--- a/zebra/redistribute.c
+++ b/zebra/redistribute.c
@@ -637,13 +637,15 @@ int zebra_add_import_table_entry(struct zebra_vrf *zvrf, struct route_node *rn,
 	struct route_entry *newre;
 	struct route_entry *same;
 	struct prefix p;
+	struct nexthop_group *ng;
 	route_map_result_t ret = RMAP_PERMITMATCH;
 	afi_t afi;
 
 	afi = family2afi(rn->p.family);
 	if (rmap_name)
 		ret = zebra_import_table_route_map_check(
-			afi, re->type, re->instance, &rn->p, re->ng->nexthop,
+			afi, re->type, re->instance, &rn->p,
+			re->nhe->nhg->nexthop,
 			zvrf->vrf->vrf_id, re->tag, rmap_name);
 
 	if (ret != RMAP_PERMITMATCH) {
@@ -678,10 +680,11 @@ int zebra_add_import_table_entry(struct zebra_vrf *zvrf, struct route_node *rn,
 	newre->table = zvrf->table_id;
 	newre->uptime = monotime(NULL);
 	newre->instance = re->table;
-	newre->ng = nexthop_group_new();
-	route_entry_copy_nexthops(newre, re->ng->nexthop);
 
-	rib_add_multipath(afi, SAFI_UNICAST, &p, NULL, newre);
+	ng = nexthop_group_new();
+	copy_nexthops(&ng->nexthop, re->nhe->nhg->nexthop, NULL);
+
+	rib_add_multipath(afi, SAFI_UNICAST, &p, NULL, newre, ng);
 
 	return 0;
 }
@@ -696,8 +699,9 @@ int zebra_del_import_table_entry(struct zebra_vrf *zvrf, struct route_node *rn,
 	prefix_copy(&p, &rn->p);
 
 	rib_delete(afi, SAFI_UNICAST, zvrf->vrf->vrf_id, ZEBRA_ROUTE_TABLE,
-		   re->table, re->flags, &p, NULL, re->ng->nexthop, re->nhe_id,
-		   zvrf->table_id, re->metric, re->distance, false);
+		   re->table, re->flags, &p, NULL, re->nhe->nhg->nexthop,
+		   re->nhe_id, zvrf->table_id, re->metric, re->distance,
+		   false);
 
 	return 0;
 }
diff --git a/zebra/rib.h b/zebra/rib.h
index 6a7b65c55..927675e3d 100644
--- a/zebra/rib.h
+++ b/zebra/rib.h
@@ -88,10 +88,14 @@ struct route_entry {
 	/* Link list. */
 	struct re_list_item next;
 
-	/* Nexthop structure (from RIB) */
-	struct nexthop_group *ng;
+	/* Nexthop group, shared/refcounted, based on the nexthop(s)
+	 * provided by the owner of the route
+	 */
+	struct nhg_hash_entry *nhe;
 
-	/* Nexthop group from FIB (optional) */
+	/* Nexthop group from FIB (optional), reflecting what is actually
+	 * installed in the FIB if that differs.
+	 */
 	struct nexthop_group fib_ng;
 
 	/* Nexthop group hash entry ID */
@@ -308,31 +312,6 @@ typedef enum {
 	RIB_UPDATE_MAX
 } rib_update_event_t;
 
-extern struct nexthop *route_entry_nexthop_ifindex_add(struct route_entry *re,
-						       ifindex_t ifindex,
-						       vrf_id_t nh_vrf_id);
-extern struct nexthop *
-route_entry_nexthop_blackhole_add(struct route_entry *re,
-				  enum blackhole_type bh_type);
-extern struct nexthop *route_entry_nexthop_ipv4_add(struct route_entry *re,
-						    struct in_addr *ipv4,
-						    struct in_addr *src,
-						    vrf_id_t nh_vrf_id);
-extern struct nexthop *
-route_entry_nexthop_ipv4_ifindex_add(struct route_entry *re,
-				     struct in_addr *ipv4, struct in_addr *src,
-				     ifindex_t ifindex, vrf_id_t nh_vrf_id);
-extern void route_entry_nexthop_delete(struct route_entry *re,
-				       struct nexthop *nexthop);
-extern struct nexthop *route_entry_nexthop_ipv6_add(struct route_entry *re,
-						    struct in6_addr *ipv6,
-						    vrf_id_t nh_vrf_id);
-extern struct nexthop *
-route_entry_nexthop_ipv6_ifindex_add(struct route_entry *re,
-				     struct in6_addr *ipv6, ifindex_t ifindex,
-				     vrf_id_t nh_vrf_id);
-extern void route_entry_nexthop_add(struct route_entry *re,
-				    struct nexthop *nexthop);
 extern void route_entry_copy_nexthops(struct route_entry *re,
 				      struct nexthop *nh);
 int route_entry_update_nhe(struct route_entry *re, struct nhg_hash_entry *new);
@@ -370,7 +349,8 @@ extern int rib_add(afi_t afi, safi_t safi, vrf_id_t vrf_id, int type,
 		   uint32_t mtu, uint8_t distance, route_tag_t tag);
 
 extern int rib_add_multipath(afi_t afi, safi_t safi, struct prefix *p,
-			     struct prefix_ipv6 *src_p, struct route_entry *re);
+			     struct prefix_ipv6 *src_p, struct route_entry *re,
+			     struct nexthop_group *ng);
 
 extern void rib_delete(afi_t afi, safi_t safi, vrf_id_t vrf_id, int type,
 		       unsigned short instance, int flags, struct prefix *p,
@@ -535,7 +515,7 @@ static inline struct nexthop_group *rib_active_nhg(struct route_entry *re)
 	if (re->fib_ng.nexthop)
 		return &(re->fib_ng);
 	else
-		return re->ng;
+		return re->nhe->nhg;
 }
 
 extern void zebra_vty_init(void);
diff --git a/zebra/rt_netlink.c b/zebra/rt_netlink.c
index c9a9a81b1..42a62c174 100644
--- a/zebra/rt_netlink.c
+++ b/zebra/rt_netlink.c
@@ -390,7 +390,7 @@ parse_nexthop_unicast(ns_id_t ns_id, struct rtmsg *rtm, struct rtattr **tb,
 }
 
 static uint8_t parse_multipath_nexthops_unicast(ns_id_t ns_id,
-						struct route_entry *re,
+						struct nexthop_group *ng,
 						struct rtmsg *rtm,
 						struct rtnexthop *rtnh,
 						struct rtattr **tb,
@@ -407,8 +407,6 @@ static uint8_t parse_multipath_nexthops_unicast(ns_id_t ns_id,
 	int len = RTA_PAYLOAD(tb[RTA_MULTIPATH]);
 	vrf_id_t nh_vrf_id = vrf_id;
 
-	re->ng = nexthop_group_new();
-
 	for (;;) {
 		struct nexthop *nh = NULL;
 
@@ -454,21 +452,19 @@ static uint8_t parse_multipath_nexthops_unicast(ns_id_t ns_id,
 
 		if (gate && rtm->rtm_family == AF_INET) {
 			if (index)
-				nh = route_entry_nexthop_ipv4_ifindex_add(
-					re, gate, prefsrc, index, nh_vrf_id);
+				nh = nexthop_from_ipv4_ifindex(
+					gate, prefsrc, index, nh_vrf_id);
 			else
-				nh = route_entry_nexthop_ipv4_add(
-					re, gate, prefsrc, nh_vrf_id);
+				nh = nexthop_from_ipv4(gate, prefsrc,
+						       nh_vrf_id);
 		} else if (gate && rtm->rtm_family == AF_INET6) {
 			if (index)
-				nh = route_entry_nexthop_ipv6_ifindex_add(
-					re, gate, index, nh_vrf_id);
+				nh = nexthop_from_ipv6_ifindex(
+					gate, index, nh_vrf_id);
 			else
-				nh = route_entry_nexthop_ipv6_add(re, gate,
-								  nh_vrf_id);
+				nh = nexthop_from_ipv6(gate, nh_vrf_id);
 		} else
-			nh = route_entry_nexthop_ifindex_add(re, index,
-							     nh_vrf_id);
+			nh = nexthop_from_ifindex(index, nh_vrf_id);
 
 		if (nh) {
 			if (num_labels)
@@ -477,6 +473,9 @@ static uint8_t parse_multipath_nexthops_unicast(ns_id_t ns_id,
 
 			if (rtnh->rtnh_flags & RTNH_F_ONLINK)
 				SET_FLAG(nh->flags, NEXTHOP_FLAG_ONLINK);
+
+			/* Add to temporary list */
+			nexthop_group_add_sorted(ng, nh);
 		}
 
 		if (rtnh->rtnh_len == 0)
@@ -486,10 +485,7 @@ static uint8_t parse_multipath_nexthops_unicast(ns_id_t ns_id,
 		rtnh = RTNH_NEXT(rtnh);
 	}
 
-	uint8_t nhop_num = nexthop_group_nexthop_num(re->ng);
-
-	if (!nhop_num)
-		nexthop_group_delete(&re->ng);
+	uint8_t nhop_num = nexthop_group_nexthop_num(ng);
 
 	return nhop_num;
 }
@@ -737,6 +733,7 @@ static int netlink_route_change_read_unicast(struct nlmsghdr *h, ns_id_t ns_id,
 		} else {
 			/* This is a multipath route */
 			struct route_entry *re;
+			struct nexthop_group *ng = NULL;
 			struct rtnexthop *rtnh =
 				(struct rtnexthop *)RTA_DATA(tb[RTA_MULTIPATH]);
 
@@ -753,19 +750,30 @@ static int netlink_route_change_read_unicast(struct nlmsghdr *h, ns_id_t ns_id,
 			re->nhe_id = nhe_id;
 
 			if (!nhe_id) {
-				uint8_t nhop_num =
+				uint8_t nhop_num;
+
+				/* Use temporary list of nexthops; parse
+				 * message payload's nexthops.
+				 */
+				ng = nexthop_group_new();
+				nhop_num =
 					parse_multipath_nexthops_unicast(
-						ns_id, re, rtm, rtnh, tb,
+						ns_id, ng, rtm, rtnh, tb,
 						prefsrc, vrf_id);
 
 				zserv_nexthop_num_warn(
 					__func__, (const struct prefix *)&p,
 					nhop_num);
+
+				if (nhop_num == 0) {
+					nexthop_group_delete(&ng);
+					ng = NULL;
+				}
 			}
 
-			if (nhe_id || re->ng)
+			if (nhe_id || ng)
 				rib_add_multipath(afi, SAFI_UNICAST, &p,
-						  &src_p, re);
+						  &src_p, re, ng);
 			else
 				XFREE(MTYPE_RE, re);
 		}
diff --git a/zebra/zapi_msg.c b/zebra/zapi_msg.c
index d6ade783c..93ee2fc1f 100644
--- a/zebra/zapi_msg.c
+++ b/zebra/zapi_msg.c
@@ -559,7 +559,8 @@ int zsend_redistribute_route(int cmd, struct zserv *client,
 		memcpy(&api.src_prefix, src_p, sizeof(api.src_prefix));
 	}
 
-	for (nexthop = re->ng->nexthop; nexthop; nexthop = nexthop->next) {
+	for (nexthop = re->nhe->nhg->nexthop;
+	     nexthop; nexthop = nexthop->next) {
 		if (!CHECK_FLAG(nexthop->flags, NEXTHOP_FLAG_ACTIVE))
 			continue;
 
@@ -666,7 +667,7 @@ static int zsend_ipv4_nexthop_lookup_mrib(struct zserv *client,
 		 * nexthop we are looking up. Therefore, we will just iterate
 		 * over the top chain of nexthops.
 		 */
-		for (nexthop = re->ng->nexthop; nexthop;
+		for (nexthop = re->nhe->nhg->nexthop; nexthop;
 		     nexthop = nexthop->next)
 			if (CHECK_FLAG(nexthop->flags, NEXTHOP_FLAG_ACTIVE))
 				num += zserv_encode_nexthop(s, nexthop);
@@ -1393,6 +1394,7 @@ static void zread_route_add(ZAPI_HANDLER_ARGS)
 	struct prefix_ipv6 *src_p = NULL;
 	struct route_entry *re;
 	struct nexthop *nexthop = NULL;
+	struct nexthop_group *ng = NULL;
 	int i, ret;
 	vrf_id_t vrf_id = 0;
 	struct ipaddr vtep_ip;
@@ -1424,7 +1426,6 @@ static void zread_route_add(ZAPI_HANDLER_ARGS)
 	re->flags = api.flags;
 	re->uptime = monotime(NULL);
 	re->vrf_id = vrf_id;
-	re->ng = nexthop_group_new();
 
 	if (api.tableid)
 		re->table = api.tableid;
@@ -1438,11 +1439,13 @@ static void zread_route_add(ZAPI_HANDLER_ARGS)
 			  __func__, &api.prefix,
 			  zebra_route_string(client->proto));
 
-		nexthop_group_delete(&re->ng);
 		XFREE(MTYPE_RE, re);
 		return;
 	}
 
+	/* Use temporary list of nexthops */
+	ng = nexthop_group_new();
+
 	/*
 	 * TBD should _all_ of the nexthop add operations use
 	 * api_nh->vrf_id instead of re->vrf_id ? I only changed
@@ -1457,8 +1460,8 @@ static void zread_route_add(ZAPI_HANDLER_ARGS)
 
 		switch (api_nh->type) {
 		case NEXTHOP_TYPE_IFINDEX:
-			nexthop = route_entry_nexthop_ifindex_add(
-				re, api_nh->ifindex, api_nh->vrf_id);
+			nexthop = nexthop_from_ifindex(api_nh->ifindex,
+						       api_nh->vrf_id);
 			break;
 		case NEXTHOP_TYPE_IPV4:
 			if (IS_ZEBRA_DEBUG_RECV) {
@@ -1469,8 +1472,8 @@ static void zread_route_add(ZAPI_HANDLER_ARGS)
 				zlog_debug("%s: nh=%s, vrf_id=%d", __func__,
 					   nhbuf, api_nh->vrf_id);
 			}
-			nexthop = route_entry_nexthop_ipv4_add(
-				re, &api_nh->gate.ipv4, NULL, api_nh->vrf_id);
+			nexthop = nexthop_from_ipv4(&api_nh->gate.ipv4,
+						    NULL, api_nh->vrf_id);
 			break;
 		case NEXTHOP_TYPE_IPV4_IFINDEX:
 
@@ -1486,8 +1489,8 @@ static void zread_route_add(ZAPI_HANDLER_ARGS)
 					__func__, nhbuf, api_nh->vrf_id,
 					re->vrf_id, ifindex);
 			}
-			nexthop = route_entry_nexthop_ipv4_ifindex_add(
-				re, &api_nh->gate.ipv4, NULL, ifindex,
+			nexthop = nexthop_from_ipv4_ifindex(
+				&api_nh->gate.ipv4, NULL, ifindex,
 				api_nh->vrf_id);
 
 			/* Special handling for IPv4 routes sourced from EVPN:
@@ -1504,15 +1507,15 @@ static void zread_route_add(ZAPI_HANDLER_ARGS)
 			}
 			break;
 		case NEXTHOP_TYPE_IPV6:
-			nexthop = route_entry_nexthop_ipv6_add(
-				re, &api_nh->gate.ipv6, api_nh->vrf_id);
+			nexthop = nexthop_from_ipv6(&api_nh->gate.ipv6,
+						    api_nh->vrf_id);
 			break;
 		case NEXTHOP_TYPE_IPV6_IFINDEX:
 			memset(&vtep_ip, 0, sizeof(struct ipaddr));
 			ifindex = api_nh->ifindex;
-			nexthop = route_entry_nexthop_ipv6_ifindex_add(
-				re, &api_nh->gate.ipv6, ifindex,
-				api_nh->vrf_id);
+			nexthop = nexthop_from_ipv6_ifindex(&api_nh->gate.ipv6,
+							    ifindex,
+							    api_nh->vrf_id);
 
 			/* Special handling for IPv6 routes sourced from EVPN:
 			 * the nexthop and associated MAC need to be installed.
@@ -1527,8 +1530,7 @@ static void zread_route_add(ZAPI_HANDLER_ARGS)
 			}
 			break;
 		case NEXTHOP_TYPE_BLACKHOLE:
-			nexthop = route_entry_nexthop_blackhole_add(
-				re, api_nh->bh_type);
+			nexthop = nexthop_from_blackhole(api_nh->bh_type);
 			break;
 		}
 
@@ -1537,10 +1539,11 @@ static void zread_route_add(ZAPI_HANDLER_ARGS)
 				EC_ZEBRA_NEXTHOP_CREATION_FAILED,
 				"%s: Nexthops Specified: %d but we failed to properly create one",
 				__PRETTY_FUNCTION__, api.nexthop_num);
-			nexthop_group_delete(&re->ng);
+			nexthop_group_delete(&ng);
 			XFREE(MTYPE_RE, re);
 			return;
 		}
+
 		if (api_nh->onlink)
 			SET_FLAG(nexthop->flags, NEXTHOP_FLAG_ONLINK);
 
@@ -1563,6 +1566,9 @@ static void zread_route_add(ZAPI_HANDLER_ARGS)
 					   api_nh->label_num,
 					   &api_nh->labels[0]);
 		}
+
+		/* Add new nexthop to temporary list */
+		nexthop_group_add_sorted(ng, nexthop);
 	}
 
 	if (CHECK_FLAG(api.message, ZAPI_MESSAGE_DISTANCE))
@@ -1579,14 +1585,14 @@ static void zread_route_add(ZAPI_HANDLER_ARGS)
 		flog_warn(EC_ZEBRA_RX_SRCDEST_WRONG_AFI,
 			  "%s: Received SRC Prefix but afi is not v6",
 			  __PRETTY_FUNCTION__);
-		nexthop_group_delete(&re->ng);
+		nexthop_group_delete(&ng);
 		XFREE(MTYPE_RE, re);
 		return;
 	}
 	if (CHECK_FLAG(api.message, ZAPI_MESSAGE_SRCPFX))
 		src_p = &api.src_prefix;
 
-	ret = rib_add_multipath(afi, api.safi, &api.prefix, src_p, re);
+	ret = rib_add_multipath(afi, api.safi, &api.prefix, src_p, re, ng);
 
 	/* Stats */
 	switch (api.prefix.family) {
diff --git a/zebra/zebra_dplane.c b/zebra/zebra_dplane.c
index 7f993442a..70bb2b675 100644
--- a/zebra/zebra_dplane.c
+++ b/zebra/zebra_dplane.c
@@ -1507,7 +1507,8 @@ static int dplane_ctx_route_init(struct zebra_dplane_ctx *ctx,
 	ctx->u.rinfo.zd_safi = info->safi;
 
 	/* Copy nexthops; recursive info is included too */
-	copy_nexthops(&(ctx->u.rinfo.zd_ng.nexthop), re->ng->nexthop, NULL);
+	copy_nexthops(&(ctx->u.rinfo.zd_ng.nexthop),
+		      re->nhe->nhg->nexthop, NULL);
 
 	/* Ensure that the dplane's nexthops flags are clear. */
 	for (ALL_NEXTHOPS(ctx->u.rinfo.zd_ng, nexthop))
@@ -1746,7 +1747,7 @@ static int dplane_ctx_pw_init(struct zebra_dplane_ctx *ctx,
 
 			if (re)
 				copy_nexthops(&(ctx->u.pw.nhg.nexthop),
-					      re->ng->nexthop, NULL);
+					      re->nhe->nhg->nexthop, NULL);
 
 			route_unlock_node(rn);
 		}
@@ -1842,7 +1843,7 @@ dplane_route_update_internal(struct route_node *rn,
 			 * We'll need these to do per-nexthop deletes.
 			 */
 			copy_nexthops(&(ctx->u.rinfo.zd_old_ng.nexthop),
-				      old_re->ng->nexthop, NULL);
+				      old_re->nhe->nhg->nexthop, NULL);
 #endif	/* !HAVE_NETLINK */
 		}
 
diff --git a/zebra/zebra_fpm_dt.c b/zebra/zebra_fpm_dt.c
index debcf60ee..389781d4f 100644
--- a/zebra/zebra_fpm_dt.c
+++ b/zebra/zebra_fpm_dt.c
@@ -90,7 +90,7 @@ static int zfpm_dt_find_route(rib_dest_t **dest_p, struct route_entry **re_p)
 		if (!re)
 			continue;
 
-		if (nexthop_group_active_nexthop_num(re->ng) == 0)
+		if (nexthop_group_active_nexthop_num(re->nhe->nhg) == 0)
 			continue;
 
 		*dest_p = dest;
diff --git a/zebra/zebra_fpm_netlink.c b/zebra/zebra_fpm_netlink.c
index b54d8fbc1..7786dc246 100644
--- a/zebra/zebra_fpm_netlink.c
+++ b/zebra/zebra_fpm_netlink.c
@@ -314,7 +314,7 @@ static int netlink_route_info_fill(netlink_route_info_t *ri, int cmd,
 	ri->rtm_type = RTN_UNICAST;
 	ri->metric = &re->metric;
 
-	for (ALL_NEXTHOPS_PTR(re->ng, nexthop)) {
+	for (ALL_NEXTHOPS_PTR(re->nhe->nhg, nexthop)) {
 		if (ri->num_nhs >= zrouter.multipath_num)
 			break;
 
diff --git a/zebra/zebra_fpm_protobuf.c b/zebra/zebra_fpm_protobuf.c
index a11517ab8..c09fa1c65 100644
--- a/zebra/zebra_fpm_protobuf.c
+++ b/zebra/zebra_fpm_protobuf.c
@@ -173,7 +173,7 @@ static Fpm__AddRoute *create_add_route_message(qpb_allocator_t *allocator,
 	 * Figure out the set of nexthops to be added to the message.
 	 */
 	num_nhs = 0;
-	for (ALL_NEXTHOPS_PTR(re->ng, nexthop)) {
+	for (ALL_NEXTHOPS_PTR(re->nhe->nhg, nexthop)) {
 		if (num_nhs >= zrouter.multipath_num)
 			break;
 
diff --git a/zebra/zebra_mpls.c b/zebra/zebra_mpls.c
index 3a44db9dc..87ea0cf0f 100644
--- a/zebra/zebra_mpls.c
+++ b/zebra/zebra_mpls.c
@@ -185,7 +185,8 @@ static int lsp_install(struct zebra_vrf *zvrf, mpls_label_t label,
 	 * the label advertised by the recursive nexthop (plus we don't have the
 	 * logic yet to push multiple labels).
 	 */
-	for (nexthop = re->ng->nexthop; nexthop; nexthop = nexthop->next) {
+	for (nexthop = re->nhe->nhg->nexthop;
+	     nexthop; nexthop = nexthop->next) {
 		/* Skip inactive and recursive entries. */
 		if (!CHECK_FLAG(nexthop->flags, NEXTHOP_FLAG_ACTIVE))
 			continue;
@@ -635,7 +636,7 @@ static int nhlfe_nexthop_active_ipv4(zebra_nhlfe_t *nhlfe,
 		    || !CHECK_FLAG(match->flags, ZEBRA_FLAG_SELECTED))
 			continue;
 
-		for (match_nh = match->ng->nexthop; match_nh;
+		for (match_nh = match->nhe->nhg->nexthop; match_nh;
 		     match_nh = match_nh->next) {
 			if (match->type == ZEBRA_ROUTE_CONNECT
 			    || nexthop->ifindex == match_nh->ifindex) {
@@ -686,10 +687,10 @@ static int nhlfe_nexthop_active_ipv6(zebra_nhlfe_t *nhlfe,
 			break;
 	}
 
-	if (!match || !match->ng->nexthop)
+	if (!match || !match->nhe->nhg->nexthop)
 		return 0;
 
-	nexthop->ifindex = match->ng->nexthop->ifindex;
+	nexthop->ifindex = match->nhe->nhg->nexthop->ifindex;
 	return 1;
 }
 
@@ -2626,7 +2627,7 @@ int mpls_ftn_update(int add, struct zebra_vrf *zvrf, enum lsp_types_t type,
 	 * We can't just change the values here since we are hashing
 	 * on labels. We need to create a whole new group
 	 */
-	nexthop_group_copy(&new_grp, re->ng);
+	nexthop_group_copy(&new_grp, re->nhe->nhg);
 
 	found = false;
 	for (nexthop = new_grp.nexthop; nexthop; nexthop = nexthop->next) {
@@ -2707,7 +2708,7 @@ int mpls_ftn_uninstall(struct zebra_vrf *zvrf, enum lsp_types_t type,
 	if (re == NULL)
 		return -1;
 
-	nexthop_group_copy(&new_grp, re->ng);
+	nexthop_group_copy(&new_grp, re->nhe->nhg);
 
 	for (nexthop = new_grp.nexthop; nexthop; nexthop = nexthop->next)
 		nexthop_del_labels(nexthop);
@@ -2921,7 +2922,7 @@ static void mpls_ftn_uninstall_all(struct zebra_vrf *zvrf,
 		RNODE_FOREACH_RE (rn, re) {
 			struct nexthop_group new_grp = {};
 
-			nexthop_group_copy(&new_grp, re->ng);
+			nexthop_group_copy(&new_grp, re->nhe->nhg);
 
 			for (nexthop = new_grp.nexthop; nexthop;
 			     nexthop = nexthop->next) {
diff --git a/zebra/zebra_nhg.c b/zebra/zebra_nhg.c
index 1108cc450..61ca200a5 100644
--- a/zebra/zebra_nhg.c
+++ b/zebra/zebra_nhg.c
@@ -299,13 +299,22 @@ zebra_nhg_connect_depends(struct nhg_hash_entry *nhe,
 	}
 }
 
-static struct nhg_hash_entry *zebra_nhg_copy(struct nhg_hash_entry *copy,
-					     uint32_t id)
+struct nhg_hash_entry *zebra_nhg_alloc(void)
 {
 	struct nhg_hash_entry *nhe;
 
 	nhe = XCALLOC(MTYPE_NHG, sizeof(struct nhg_hash_entry));
 
+	return nhe;
+}
+
+static struct nhg_hash_entry *zebra_nhg_copy(const struct nhg_hash_entry *copy,
+					     uint32_t id)
+{
+	struct nhg_hash_entry *nhe;
+
+	nhe = zebra_nhg_alloc();
+
 	nhe->id = id;
 
 	nhe->nhg = nexthop_group_new();
@@ -468,7 +477,7 @@ static void handle_recursive_depend(struct nhg_connected_tree_head *nhg_depends,
 	struct nhg_hash_entry *depend = NULL;
 	struct nexthop_group resolved_ng = {};
 
-	_nexthop_group_add_sorted(&resolved_ng, nh);
+	nexthop_group_add_sorted(&resolved_ng, nh);
 
 	depend = zebra_nhg_rib_find(0, &resolved_ng, afi);
 	depends_add(nhg_depends, depend);
@@ -582,7 +591,7 @@ zebra_nhg_find_nexthop(uint32_t id, struct nexthop *nh, afi_t afi, int type)
 	struct nhg_hash_entry *nhe = NULL;
 	struct nexthop_group nhg = {};
 
-	_nexthop_group_add_sorted(&nhg, nh);
+	nexthop_group_add_sorted(&nhg, nh);
 
 	zebra_nhg_find(&nhe, id, &nhg, NULL, nh->vrf_id, afi, 0);
 
@@ -1128,12 +1137,8 @@ static void zebra_nhg_free_members(struct nhg_hash_entry *nhe)
 	nhg_connected_tree_free(&nhe->nhg_dependents);
 }
 
-void zebra_nhg_free(void *arg)
+void zebra_nhg_free(struct nhg_hash_entry *nhe)
 {
-	struct nhg_hash_entry *nhe = NULL;
-
-	nhe = (struct nhg_hash_entry *)arg;
-
 	if (nhe->refcnt)
 		zlog_debug("nhe_id=%u hash refcnt=%d", nhe->id, nhe->refcnt);
 
@@ -1142,6 +1147,11 @@ void zebra_nhg_free(void *arg)
 	XFREE(MTYPE_NHG, nhe);
 }
 
+void zebra_nhg_hash_free(void *p)
+{
+	zebra_nhg_free((struct nhg_hash_entry *)p);
+}
+
 void zebra_nhg_decrement_ref(struct nhg_hash_entry *nhe)
 {
 	nhe->refcnt--;
@@ -1436,7 +1446,7 @@ static int nexthop_active(afi_t afi, struct route_entry *re,
 
 		if (match->type == ZEBRA_ROUTE_CONNECT) {
 			/* Directly point connected route. */
-			newhop = match->ng->nexthop;
+			newhop = match->nhe->nhg->nexthop;
 			if (newhop) {
 				if (nexthop->type == NEXTHOP_TYPE_IPV4
 				    || nexthop->type == NEXTHOP_TYPE_IPV6)
@@ -1445,7 +1455,7 @@ static int nexthop_active(afi_t afi, struct route_entry *re,
 			return 1;
 		} else if (CHECK_FLAG(re->flags, ZEBRA_FLAG_ALLOW_RECURSION)) {
 			resolved = 0;
-			for (ALL_NEXTHOPS_PTR(match->ng, newhop)) {
+			for (ALL_NEXTHOPS_PTR(match->nhe->nhg, newhop)) {
 				if (!CHECK_FLAG(match->status,
 						ROUTE_ENTRY_INSTALLED))
 					continue;
@@ -1466,7 +1476,7 @@ static int nexthop_active(afi_t afi, struct route_entry *re,
 			return resolved;
 		} else if (re->type == ZEBRA_ROUTE_STATIC) {
 			resolved = 0;
-			for (ALL_NEXTHOPS_PTR(match->ng, newhop)) {
+			for (ALL_NEXTHOPS_PTR(match->nhe->nhg, newhop)) {
 				if (!CHECK_FLAG(match->status,
 						ROUTE_ENTRY_INSTALLED))
 					continue;
@@ -1657,7 +1667,7 @@ int nexthop_active_update(struct route_node *rn, struct route_entry *re)
 	UNSET_FLAG(re->status, ROUTE_ENTRY_CHANGED);
 
 	/* Copy over the nexthops in current state */
-	nexthop_group_copy(&new_grp, re->ng);
+	nexthop_group_copy(&new_grp, re->nhe->nhg);
 
 	for (nexthop = new_grp.nexthop; nexthop; nexthop = nexthop->next) {
 
diff --git a/zebra/zebra_nhg.h b/zebra/zebra_nhg.h
index 4aa433a9a..522ec1e9d 100644
--- a/zebra/zebra_nhg.h
+++ b/zebra/zebra_nhg.h
@@ -158,6 +158,11 @@ struct nhg_ctx {
  * NHE abstracted tree functions.
  * Use these where possible instead of the direct ones access ones.
  */
+struct nhg_hash_entry *zebra_nhg_alloc(void);
+void zebra_nhg_free(struct nhg_hash_entry *nhe);
+/* In order to clear a generic hash, we need a generic api, sigh. */
+void zebra_nhg_hash_free(void *p);
+
 extern struct nhg_hash_entry *zebra_nhg_resolve(struct nhg_hash_entry *nhe);
 
 extern unsigned int zebra_nhg_depends_count(const struct nhg_hash_entry *nhe);
diff --git a/zebra/zebra_nhg_private.h b/zebra/zebra_nhg_private.h
index 170e2357e..79107b047 100644
--- a/zebra/zebra_nhg_private.h
+++ b/zebra/zebra_nhg_private.h
@@ -57,6 +57,4 @@ extern void nhg_connected_tree_del_nhe(struct nhg_connected_tree_head *head,
 extern void nhg_connected_tree_add_nhe(struct nhg_connected_tree_head *head,
 				       struct nhg_hash_entry *nhe);
 
-extern void zebra_nhg_free(void *arg);
-
 #endif /* __ZEBRA_NHG_PRIVATE_H__ */
diff --git a/zebra/zebra_pw.c b/zebra/zebra_pw.c
index 3f1567a95..618a23240 100644
--- a/zebra/zebra_pw.c
+++ b/zebra/zebra_pw.c
@@ -259,7 +259,7 @@ static int zebra_pw_check_reachability(struct zebra_pw *pw)
 	 * Need to ensure that there's a label binding for all nexthops.
 	 * Otherwise, ECMP for this route could render the pseudowire unusable.
 	 */
-	for (ALL_NEXTHOPS_PTR(re->ng, nexthop)) {
+	for (ALL_NEXTHOPS_PTR(re->nhe->nhg, nexthop)) {
 		if (!nexthop->nh_label) {
 			if (IS_ZEBRA_DEBUG_PW)
 				zlog_debug("%s: unlabeled route for %s",
diff --git a/zebra/zebra_rib.c b/zebra/zebra_rib.c
index 31d46e5a2..5eeeebd6e 100644
--- a/zebra/zebra_rib.c
+++ b/zebra/zebra_rib.c
@@ -193,150 +193,19 @@ int zebra_check_addr(const struct prefix *p)
 	return 1;
 }
 
-/* Add nexthop to the end of a rib node's nexthop list */
-void route_entry_nexthop_add(struct route_entry *re, struct nexthop *nexthop)
-{
-	_nexthop_group_add_sorted(re->ng, nexthop);
-}
-
-
 /**
  * copy_nexthop - copy a nexthop to the rib structure.
  */
 void route_entry_copy_nexthops(struct route_entry *re, struct nexthop *nh)
 {
-	assert(!re->ng->nexthop);
-	copy_nexthops(&re->ng->nexthop, nh, NULL);
-}
-
-/* Delete specified nexthop from the list. */
-void route_entry_nexthop_delete(struct route_entry *re, struct nexthop *nexthop)
-{
-	if (nexthop->next)
-		nexthop->next->prev = nexthop->prev;
-	if (nexthop->prev)
-		nexthop->prev->next = nexthop->next;
-	else
-		re->ng->nexthop = nexthop->next;
-}
-
-
-struct nexthop *route_entry_nexthop_ifindex_add(struct route_entry *re,
-						ifindex_t ifindex,
-						vrf_id_t nh_vrf_id)
-{
-	struct nexthop *nexthop;
-
-	nexthop = nexthop_new();
-	nexthop->type = NEXTHOP_TYPE_IFINDEX;
-	nexthop->ifindex = ifindex;
-	nexthop->vrf_id = nh_vrf_id;
-
-	route_entry_nexthop_add(re, nexthop);
-
-	return nexthop;
-}
-
-struct nexthop *route_entry_nexthop_ipv4_add(struct route_entry *re,
-					     struct in_addr *ipv4,
-					     struct in_addr *src,
-					     vrf_id_t nh_vrf_id)
-{
-	struct nexthop *nexthop;
-
-	nexthop = nexthop_new();
-	nexthop->type = NEXTHOP_TYPE_IPV4;
-	nexthop->vrf_id = nh_vrf_id;
-	nexthop->gate.ipv4 = *ipv4;
-	if (src)
-		nexthop->src.ipv4 = *src;
-
-	route_entry_nexthop_add(re, nexthop);
-
-	return nexthop;
-}
-
-struct nexthop *route_entry_nexthop_ipv4_ifindex_add(struct route_entry *re,
-						     struct in_addr *ipv4,
-						     struct in_addr *src,
-						     ifindex_t ifindex,
-						     vrf_id_t nh_vrf_id)
-{
-	struct nexthop *nexthop;
-	struct interface *ifp;
-
-	nexthop = nexthop_new();
-	nexthop->vrf_id = nh_vrf_id;
-	nexthop->type = NEXTHOP_TYPE_IPV4_IFINDEX;
-	nexthop->gate.ipv4 = *ipv4;
-	if (src)
-		nexthop->src.ipv4 = *src;
-	nexthop->ifindex = ifindex;
-	ifp = if_lookup_by_index(nexthop->ifindex, nh_vrf_id);
-	/*Pending: need to think if null ifp here is ok during bootup?
-	  There was a crash because ifp here was coming to be NULL */
-	if (ifp)
-		if (connected_is_unnumbered(ifp))
-			SET_FLAG(nexthop->flags, NEXTHOP_FLAG_ONLINK);
-
-	route_entry_nexthop_add(re, nexthop);
-
-	return nexthop;
-}
-
-struct nexthop *route_entry_nexthop_ipv6_add(struct route_entry *re,
-					     struct in6_addr *ipv6,
-					     vrf_id_t nh_vrf_id)
-{
-	struct nexthop *nexthop;
-
-	nexthop = nexthop_new();
-	nexthop->vrf_id = nh_vrf_id;
-	nexthop->type = NEXTHOP_TYPE_IPV6;
-	nexthop->gate.ipv6 = *ipv6;
-
-	route_entry_nexthop_add(re, nexthop);
-
-	return nexthop;
-}
-
-struct nexthop *route_entry_nexthop_ipv6_ifindex_add(struct route_entry *re,
-						     struct in6_addr *ipv6,
-						     ifindex_t ifindex,
-						     vrf_id_t nh_vrf_id)
-{
-	struct nexthop *nexthop;
-
-	nexthop = nexthop_new();
-	nexthop->vrf_id = nh_vrf_id;
-	nexthop->type = NEXTHOP_TYPE_IPV6_IFINDEX;
-	nexthop->gate.ipv6 = *ipv6;
-	nexthop->ifindex = ifindex;
-
-	route_entry_nexthop_add(re, nexthop);
-
-	return nexthop;
-}
-
-struct nexthop *route_entry_nexthop_blackhole_add(struct route_entry *re,
-						  enum blackhole_type bh_type)
-{
-	struct nexthop *nexthop;
-
-	nexthop = nexthop_new();
-	nexthop->vrf_id = VRF_DEFAULT;
-	nexthop->type = NEXTHOP_TYPE_BLACKHOLE;
-	nexthop->bh_type = bh_type;
-
-	route_entry_nexthop_add(re, nexthop);
-
-	return nexthop;
+	assert(!re->nhe->nhg->nexthop);
+	copy_nexthops(&re->nhe->nhg->nexthop, nh, NULL);
 }
 
 static void route_entry_attach_ref(struct route_entry *re,
 				   struct nhg_hash_entry *new)
 {
-	re->ng = new->nhg;
+	re->nhe = new;
 	re->nhe_id = new->id;
 
 	zebra_nhg_increment_ref(new);
@@ -348,7 +217,7 @@ int route_entry_update_nhe(struct route_entry *re, struct nhg_hash_entry *new)
 	int ret = 0;
 
 	if (new == NULL) {
-		re->ng = NULL;
+		re->nhe->nhg = NULL;
 		goto done;
 	}
 
@@ -359,7 +228,7 @@ int route_entry_update_nhe(struct route_entry *re, struct nhg_hash_entry *new)
 
 		if (old)
 			zebra_nhg_decrement_ref(old);
-	} else if (!re->ng)
+	} else if (!re->nhe->nhg)
 		/* This is the first time it's being attached */
 		route_entry_attach_ref(re, new);
 
@@ -535,7 +404,7 @@ int zebra_rib_labeled_unicast(struct route_entry *re)
 	if (re->type != ZEBRA_ROUTE_BGP)
 		return 0;
 
-	for (ALL_NEXTHOPS_PTR(re->ng, nexthop))
+	for (ALL_NEXTHOPS_PTR(re->nhe->nhg, nexthop))
 		if (!nexthop->nh_label || !nexthop->nh_label->num_labels)
 			return 0;
 
@@ -559,7 +428,7 @@ void rib_install_kernel(struct route_node *rn, struct route_entry *re,
 	srcdest_rnode_prefixes(rn, &p, &src_p);
 
 	if (info->safi != SAFI_UNICAST) {
-		for (ALL_NEXTHOPS_PTR(re->ng, nexthop))
+		for (ALL_NEXTHOPS_PTR(re->nhe->nhg, nexthop))
 			SET_FLAG(nexthop->flags, NEXTHOP_FLAG_FIB);
 		return;
 	}
@@ -637,7 +506,7 @@ void rib_uninstall_kernel(struct route_node *rn, struct route_entry *re)
 
 	if (info->safi != SAFI_UNICAST) {
 		UNSET_FLAG(re->status, ROUTE_ENTRY_INSTALLED);
-		for (ALL_NEXTHOPS_PTR(re->ng, nexthop))
+		for (ALL_NEXTHOPS_PTR(re->nhe->nhg, nexthop))
 			UNSET_FLAG(nexthop->flags, NEXTHOP_FLAG_FIB);
 		return;
 	}
@@ -697,7 +566,7 @@ static void rib_uninstall(struct route_node *rn, struct route_entry *re)
 			re->fib_ng.nexthop = NULL;
 		}
 
-		for (ALL_NEXTHOPS_PTR(re->ng, nexthop))
+		for (ALL_NEXTHOPS_PTR(re->nhe->nhg, nexthop))
 			UNSET_FLAG(nexthop->flags, NEXTHOP_FLAG_FIB);
 	}
 
@@ -873,7 +742,7 @@ static void rib_process_add_fib(struct zebra_vrf *zvrf, struct route_node *rn,
 
 	/* Update real nexthop. This may actually determine if nexthop is active
 	 * or not. */
-	if (!nexthop_group_active_nexthop_num(new->ng)) {
+	if (!nexthop_group_active_nexthop_num(new->nhe->nhg)) {
 		UNSET_FLAG(new->status, ROUTE_ENTRY_CHANGED);
 		return;
 	}
@@ -942,7 +811,7 @@ static void rib_process_update_fib(struct zebra_vrf *zvrf,
 
 		/* Update the nexthop; we could determine here that nexthop is
 		 * inactive. */
-		if (nexthop_group_active_nexthop_num(new->ng))
+		if (nexthop_group_active_nexthop_num(new->nhe->nhg))
 			nh_active = 1;
 
 		/* If nexthop is active, install the selected route, if
@@ -1060,7 +929,7 @@ static struct route_entry *rib_choose_best(struct route_entry *current,
 		/* both are connected.  are either loop or vrf? */
 		struct nexthop *nexthop = NULL;
 
-		for (ALL_NEXTHOPS_PTR(alternate->ng, nexthop)) {
+		for (ALL_NEXTHOPS_PTR(alternate->nhe->nhg, nexthop)) {
 			struct interface *ifp = if_lookup_by_index(
 				nexthop->ifindex, alternate->vrf_id);
 
@@ -1068,7 +937,7 @@ static struct route_entry *rib_choose_best(struct route_entry *current,
 				return alternate;
 		}
 
-		for (ALL_NEXTHOPS_PTR(current->ng, nexthop)) {
+		for (ALL_NEXTHOPS_PTR(current->nhe->nhg, nexthop)) {
 			struct interface *ifp = if_lookup_by_index(
 				nexthop->ifindex, current->vrf_id);
 
@@ -1399,7 +1268,7 @@ static void zebra_rib_fixup_system(struct route_node *rn)
 		SET_FLAG(re->status, ROUTE_ENTRY_INSTALLED);
 		UNSET_FLAG(re->status, ROUTE_ENTRY_QUEUED);
 
-		for (ALL_NEXTHOPS_PTR(re->ng, nhop)) {
+		for (ALL_NEXTHOPS_PTR(re->nhe->nhg, nhop)) {
 			if (CHECK_FLAG(nhop->flags, NEXTHOP_FLAG_RECURSIVE))
 				continue;
 
@@ -1502,13 +1371,13 @@ static bool rib_update_re_from_ctx(struct route_entry *re,
 
 	/* Get the first `installed` one to check against.
 	 * If the dataplane doesn't set these to be what was actually installed,
-	 * it will just be whatever was in re->ng?
+	 * it will just be whatever was in re->nhe->nhg?
 	 */
 	if (CHECK_FLAG(ctx_nexthop->flags, NEXTHOP_FLAG_RECURSIVE)
 	    || !CHECK_FLAG(ctx_nexthop->flags, NEXTHOP_FLAG_ACTIVE))
 		ctx_nexthop = nexthop_next_active_resolved(ctx_nexthop);
 
-	for (ALL_NEXTHOPS_PTR(re->ng, nexthop)) {
+	for (ALL_NEXTHOPS_PTR(re->nhe->nhg, nexthop)) {
 
 		if (CHECK_FLAG(nexthop->flags, NEXTHOP_FLAG_RECURSIVE))
 			continue;
@@ -2455,8 +2324,8 @@ void rib_unlink(struct route_node *rn, struct route_entry *re)
 		nhe = zebra_nhg_lookup_id(re->nhe_id);
 		if (nhe)
 			zebra_nhg_decrement_ref(nhe);
-	} else if (re->ng)
-		nexthop_group_delete(&re->ng);
+	} else if (re->nhe->nhg)
+		nexthop_group_delete(&re->nhe->nhg);
 
 	nexthops_free(re->fib_ng.nexthop);
 
@@ -2523,10 +2392,10 @@ void _route_entry_dump(const char *func, union prefixconstptr pp,
 		"%s: metric == %u, mtu == %u, distance == %u, flags == %u, status == %u",
 		straddr, re->metric, re->mtu, re->distance, re->flags, re->status);
 	zlog_debug("%s: nexthop_num == %u, nexthop_active_num == %u", straddr,
-		   nexthop_group_nexthop_num(re->ng),
-		   nexthop_group_active_nexthop_num(re->ng));
+		   nexthop_group_nexthop_num(re->nhe->nhg),
+		   nexthop_group_active_nexthop_num(re->nhe->nhg));
 
-	for (ALL_NEXTHOPS_PTR(re->ng, nexthop)) {
+	for (ALL_NEXTHOPS_PTR(re->nhe->nhg, nexthop)) {
 		struct interface *ifp;
 		struct vrf *vrf = vrf_lookup_by_id(nexthop->vrf_id);
 
@@ -2675,7 +2544,8 @@ void rib_lookup_and_pushup(struct prefix_ipv4 *p, vrf_id_t vrf_id)
 }
 
 int rib_add_multipath(afi_t afi, safi_t safi, struct prefix *p,
-		      struct prefix_ipv6 *src_p, struct route_entry *re)
+		      struct prefix_ipv6 *src_p, struct route_entry *re,
+		      struct nexthop_group *ng)
 {
 	struct nhg_hash_entry *nhe = NULL;
 	struct route_table *table;
@@ -2692,8 +2562,8 @@ int rib_add_multipath(afi_t afi, safi_t safi, struct prefix *p,
 	table = zebra_vrf_get_table_with_table_id(afi, safi, re->vrf_id,
 						  re->table);
 	if (!table) {
-		if (re->ng)
-			nexthop_group_delete(&re->ng);
+		if (ng)
+			nexthop_group_delete(&ng);
 		XFREE(MTYPE_RE, re);
 		return 0;
 	}
@@ -2710,13 +2580,13 @@ int rib_add_multipath(afi_t afi, safi_t safi, struct prefix *p,
 			return -1;
 		}
 	} else {
-		nhe = zebra_nhg_rib_find(0, re->ng, afi);
+		nhe = zebra_nhg_rib_find(0, ng, afi);
 
 		/*
 		 * The nexthops got copied over into an nhe,
 		 * so free them now.
 		 */
-		nexthop_group_delete(&re->ng);
+		nexthop_group_delete(&ng);
 
 		if (!nhe) {
 			char buf[PREFIX_STRLEN] = "";
@@ -2885,7 +2755,8 @@ void rib_delete(afi_t afi, safi_t safi, vrf_id_t vrf_id, int type,
 
 		if (re->type == ZEBRA_ROUTE_KERNEL && re->metric != metric)
 			continue;
-		if (re->type == ZEBRA_ROUTE_CONNECT && (rtnh = re->ng->nexthop)
+		if (re->type == ZEBRA_ROUTE_CONNECT &&
+		    (rtnh = re->nhe->nhg->nexthop)
 		    && rtnh->type == NEXTHOP_TYPE_IFINDEX && nh) {
 			if (rtnh->ifindex != nh->ifindex)
 				continue;
@@ -2903,7 +2774,7 @@ void rib_delete(afi_t afi, safi_t safi, vrf_id_t vrf_id, int type,
 			same = re;
 			break;
 		}
-		for (ALL_NEXTHOPS_PTR(re->ng, rtnh)) {
+		for (ALL_NEXTHOPS_PTR(re->nhe->nhg, rtnh)) {
 			/*
 			 * No guarantee all kernel send nh with labels
 			 * on delete.
@@ -2945,7 +2816,7 @@ void rib_delete(afi_t afi, safi_t safi, vrf_id_t vrf_id, int type,
 			if (allow_delete) {
 				UNSET_FLAG(fib->status, ROUTE_ENTRY_INSTALLED);
 				/* Unset flags. */
-				for (rtnh = fib->ng->nexthop; rtnh;
+				for (rtnh = fib->nhe->nhg->nexthop; rtnh;
 				     rtnh = rtnh->next)
 					UNSET_FLAG(rtnh->flags,
 						   NEXTHOP_FLAG_FIB);
@@ -3001,7 +2872,7 @@ void rib_delete(afi_t afi, safi_t safi, vrf_id_t vrf_id, int type,
 		if (CHECK_FLAG(flags, ZEBRA_FLAG_EVPN_ROUTE)) {
 			struct nexthop *tmp_nh;
 
-			for (ALL_NEXTHOPS_PTR(re->ng, tmp_nh)) {
+			for (ALL_NEXTHOPS_PTR(re->nhe->nhg, tmp_nh)) {
 				struct ipaddr vtep_ip;
 
 				memset(&vtep_ip, 0, sizeof(struct ipaddr));
@@ -3041,6 +2912,7 @@ int rib_add(afi_t afi, safi_t safi, vrf_id_t vrf_id, int type,
 {
 	struct route_entry *re = NULL;
 	struct nexthop *nexthop = NULL;
+	struct nexthop_group *ng = NULL;
 
 	/* Allocate new route_entry structure. */
 	re = XCALLOC(MTYPE_RE, sizeof(struct route_entry));
@@ -3056,16 +2928,19 @@ int rib_add(afi_t afi, safi_t safi, vrf_id_t vrf_id, int type,
 	re->tag = tag;
 	re->nhe_id = nhe_id;
 
+	/* If the owner of the route supplies a shared nexthop-group id,
+	 * we'll use that. Otherwise, pass the nexthop along directly.
+	 */
 	if (!nhe_id) {
-		re->ng = nexthop_group_new();
+		ng = nexthop_group_new();
 
 		/* Add nexthop. */
 		nexthop = nexthop_new();
 		*nexthop = *nh;
-		route_entry_nexthop_add(re, nexthop);
+		nexthop_group_add_sorted(ng, nexthop);
 	}
 
-	return rib_add_multipath(afi, safi, p, src_p, re);
+	return rib_add_multipath(afi, safi, p, src_p, re, ng);
 }
 
 static const char *rib_update_event2str(rib_update_event_t event)
@@ -3323,7 +3198,7 @@ void rib_sweep_table(struct route_table *table)
 			 * this decision needs to be revisited
 			 */
 			SET_FLAG(re->status, ROUTE_ENTRY_INSTALLED);
-			for (ALL_NEXTHOPS_PTR(re->ng, nexthop))
+			for (ALL_NEXTHOPS_PTR(re->nhe->nhg, nexthop))
 				SET_FLAG(nexthop->flags, NEXTHOP_FLAG_FIB);
 
 			rib_uninstall_kernel(rn, re);
diff --git a/zebra/zebra_rnh.c b/zebra/zebra_rnh.c
index 60e23cc4d..9a6631a59 100644
--- a/zebra/zebra_rnh.c
+++ b/zebra/zebra_rnh.c
@@ -384,7 +384,7 @@ static void zebra_rnh_clear_nexthop_rnh_filters(struct route_entry *re)
 	struct nexthop *nexthop;
 
 	if (re) {
-		for (nexthop = re->ng->nexthop; nexthop;
+		for (nexthop = re->nhe->nhg->nexthop; nexthop;
 		     nexthop = nexthop->next) {
 			UNSET_FLAG(nexthop->flags, NEXTHOP_FLAG_RNH_FILTERED);
 		}
@@ -403,7 +403,7 @@ static int zebra_rnh_apply_nht_rmap(afi_t afi, struct zebra_vrf *zvrf,
 	route_map_result_t ret;
 
 	if (prn && re) {
-		for (nexthop = re->ng->nexthop; nexthop;
+		for (nexthop = re->nhe->nhg->nexthop; nexthop;
 		     nexthop = nexthop->next) {
 			ret = zebra_nht_route_map_check(
 				afi, proto, &prn->p, zvrf, re, nexthop);
@@ -688,7 +688,7 @@ zebra_rnh_resolve_nexthop_entry(struct zebra_vrf *zvrf, afi_t afi,
 			/* Just being SELECTED isn't quite enough - must
 			 * have an installed nexthop to be useful.
 			 */
-			for (ALL_NEXTHOPS_PTR(re->ng, nexthop)) {
+			for (ALL_NEXTHOPS_PTR(re->nhe->nhg, nexthop)) {
 				if (rnh_nexthop_valid(re, nexthop))
 					break;
 			}
@@ -707,7 +707,8 @@ zebra_rnh_resolve_nexthop_entry(struct zebra_vrf *zvrf, afi_t afi,
 					break;
 				if (re->type == ZEBRA_ROUTE_NHRP) {
 
-					for (nexthop = re->ng->nexthop; nexthop;
+					for (nexthop = re->nhe->nhg->nexthop;
+					     nexthop;
 					     nexthop = nexthop->next)
 						if (nexthop->type
 						    == NEXTHOP_TYPE_IFINDEX)
@@ -940,7 +941,7 @@ static void free_state(vrf_id_t vrf_id, struct route_entry *re,
 		return;
 
 	/* free RE and nexthops */
-	nexthop_group_delete(&re->ng);
+	zebra_nhg_free(re->nhe);
 	XFREE(MTYPE_RE, re);
 }
 
@@ -963,9 +964,11 @@ static void copy_state(struct rnh *rnh, struct route_entry *re,
 	state->metric = re->metric;
 	state->vrf_id = re->vrf_id;
 	state->status = re->status;
-	state->ng = nexthop_group_new();
 
-	route_entry_copy_nexthops(state, re->ng->nexthop);
+	state->nhe = zebra_nhg_alloc();
+	state->nhe->nhg = nexthop_group_new();
+
+	nexthop_group_copy(state->nhe->nhg, re->nhe->nhg);
 	rnh->state = state;
 }
 
@@ -983,11 +986,12 @@ static int compare_state(struct route_entry *r1, struct route_entry *r2)
 	if (r1->metric != r2->metric)
 		return 1;
 
-	if (nexthop_group_nexthop_num(r1->ng)
-	    != nexthop_group_nexthop_num(r2->ng))
+	if (nexthop_group_nexthop_num(r1->nhe->nhg)
+	    != nexthop_group_nexthop_num(r2->nhe->nhg))
 		return 1;
 
-	if (nexthop_group_hash(r1->ng) != nexthop_group_hash(r2->ng))
+	if (nexthop_group_hash(r1->nhe->nhg) !=
+	    nexthop_group_hash(r2->nhe->nhg))
 		return 1;
 
 	return 0;
@@ -1037,7 +1041,7 @@ static int send_client(struct rnh *rnh, struct zserv *client, rnh_type_t type,
 		num = 0;
 		nump = stream_get_endp(s);
 		stream_putc(s, 0);
-		for (ALL_NEXTHOPS_PTR(re->ng, nh))
+		for (ALL_NEXTHOPS_PTR(re->nhe->nhg, nh))
 			if (rnh_nexthop_valid(re, nh)) {
 				stream_putl(s, nh->vrf_id);
 				stream_putc(s, nh->type);
@@ -1137,7 +1141,7 @@ static void print_rnh(struct route_node *rn, struct vty *vty)
 	if (rnh->state) {
 		vty_out(vty, " resolved via %s\n",
 			zebra_route_string(rnh->state->type));
-		for (nexthop = rnh->state->ng->nexthop; nexthop;
+		for (nexthop = rnh->state->nhe->nhg->nexthop; nexthop;
 		     nexthop = nexthop->next)
 			print_nh(nexthop, vty);
 	} else
diff --git a/zebra/zebra_router.c b/zebra/zebra_router.c
index e573093b7..a891ffb76 100644
--- a/zebra/zebra_router.c
+++ b/zebra/zebra_router.c
@@ -29,7 +29,7 @@
 #include "zebra_pbr.h"
 #include "zebra_vxlan.h"
 #include "zebra_mlag.h"
-#include "zebra_nhg_private.h"
+#include "zebra_nhg.h"
 #include "debug.h"
 
 DEFINE_MTYPE_STATIC(ZEBRA, RIB_TABLE_INFO, "RIB table info")
@@ -223,7 +223,7 @@ void zebra_router_terminate(void)
 	zebra_vxlan_disable();
 	zebra_mlag_terminate();
 
-	hash_clean(zrouter.nhgs, zebra_nhg_free);
+	hash_clean(zrouter.nhgs, zebra_nhg_hash_free);
 	hash_free(zrouter.nhgs);
 	hash_clean(zrouter.nhgs_id, NULL);
 	hash_free(zrouter.nhgs_id);
diff --git a/zebra/zebra_snmp.c b/zebra/zebra_snmp.c
index 56c766432..4c5265198 100644
--- a/zebra/zebra_snmp.c
+++ b/zebra/zebra_snmp.c
@@ -285,8 +285,8 @@ static void check_replace(struct route_node *np2, struct route_entry *re2,
 		return;
 	}
 
-	if (in_addr_cmp((uint8_t *)&(*re)->ng->nexthop->gate.ipv4,
-			(uint8_t *)&re2->ng->nexthop->gate.ipv4)
+	if (in_addr_cmp((uint8_t *)&(*re)->nhe->nhg->nexthop->gate.ipv4,
+			(uint8_t *)&re2->nhe->nhg->nexthop->gate.ipv4)
 	    <= 0)
 		return;
 
@@ -371,9 +371,9 @@ static void get_fwtable_route_node(struct variable *v, oid objid[],
 			if (!in_addr_cmp(&(*np)->p.u.prefix,
 					 (uint8_t *)&dest)) {
 				RNODE_FOREACH_RE (*np, *re) {
-					if (!in_addr_cmp((uint8_t *)&(*re)
-								 ->ng->nexthop
-								 ->gate.ipv4,
+					if (!in_addr_cmp((uint8_t *)&(*re)->nhe
+							 ->nhg->nexthop
+							 ->gate.ipv4,
 							 (uint8_t *)&nexthop))
 						if (proto
 						    == proto_trans((*re)->type))
@@ -406,8 +406,8 @@ static void get_fwtable_route_node(struct variable *v, oid objid[],
 				    || ((policy == policy2) && (proto < proto2))
 				    || ((policy == policy2) && (proto == proto2)
 					&& (in_addr_cmp(
-						    (uint8_t *)&re2->ng->nexthop
-							    ->gate.ipv4,
+						    (uint8_t *)&re2->nhe
+						    ->nhg->nexthop->gate.ipv4,
 						    (uint8_t *)&nexthop)
 					    >= 0)))
 					check_replace(np2, re2, np, re);
@@ -432,7 +432,7 @@ static void get_fwtable_route_node(struct variable *v, oid objid[],
 	{
 		struct nexthop *nexthop;
 
-		nexthop = (*re)->ng->nexthop;
+		nexthop = (*re)->nhe->nhg->nexthop;
 		if (nexthop) {
 			pnt = (uint8_t *)&nexthop->gate.ipv4;
 			for (i = 0; i < 4; i++)
@@ -462,7 +462,7 @@ static uint8_t *ipFwTable(struct variable *v, oid objid[], size_t *objid_len,
 	if (!np)
 		return NULL;
 
-	nexthop = re->ng->nexthop;
+	nexthop = re->nhe->nhg->nexthop;
 	if (!nexthop)
 		return NULL;
 
diff --git a/zebra/zebra_vty.c b/zebra/zebra_vty.c
index 12517f313..b0436dd40 100644
--- a/zebra/zebra_vty.c
+++ b/zebra/zebra_vty.c
@@ -263,7 +263,7 @@ static void vty_show_ip_route_detail(struct vty *vty, struct route_node *rn,
 		if (show_ng)
 			vty_out(vty, "  Nexthop Group ID: %u\n", re->nhe_id);
 
-		for (ALL_NEXTHOPS_PTR(re->ng, nexthop)) {
+		for (ALL_NEXTHOPS_PTR(re->nhe->nhg, nexthop)) {
 			char addrstr[32];
 
 			vty_out(vty, "  %c%s",
@@ -413,7 +413,7 @@ static void vty_show_ip_route(struct vty *vty, struct route_node *rn,
 	if (is_fib)
 		nhg = rib_active_nhg(re);
 	else
-		nhg = re->ng;
+		nhg = re->nhe->nhg;
 
 	if (json) {
 		json_route = json_object_new_object();
@@ -466,9 +466,10 @@ static void vty_show_ip_route(struct vty *vty, struct route_node *rn,
 		json_object_int_add(json_route, "internalFlags",
 				    re->flags);
 		json_object_int_add(json_route, "internalNextHopNum",
-				    nexthop_group_nexthop_num(re->ng));
+				    nexthop_group_nexthop_num(re->nhe->nhg));
 		json_object_int_add(json_route, "internalNextHopActiveNum",
-				    nexthop_group_active_nexthop_num(re->ng));
+				    nexthop_group_active_nexthop_num(
+					    re->nhe->nhg));
 		if (uptime < ONE_DAY_SECOND)
 			sprintf(buf, "%02d:%02d:%02d", tm->tm_hour, tm->tm_min,
 				tm->tm_sec);
@@ -1833,7 +1834,7 @@ static void vty_show_ip_route_summary_prefix(struct vty *vty,
 				fib_cnt[ZEBRA_ROUTE_TOTAL]++;
 				fib_cnt[re->type]++;
 			}
-			for (nexthop = re->ng->nexthop; (!cnt && nexthop);
+			for (nexthop = re->nhe->nhg->nexthop; (!cnt && nexthop);
 			     nexthop = nexthop->next) {
 				cnt++;
 				rib_cnt[ZEBRA_ROUTE_TOTAL]++;