pbrd: Add PBR to FRR

This is an implementation of PBR for FRR.

This implemenation uses a combination of rules and
tables to determine how packets will flow.

PBR introduces a new concept of 'nexthop-groups' to
specify a group of nexthops that will be used for
ecmp.  Nexthop-groups are specified on the cli via:

nexthop-group DONNA
  nexthop 192.168.208.1
  nexthop 192.168.209.1
  nexthop 192.168.210.1
!

PBR sees the nexthop-group and installs these as a default
route with these nexthops starting at table 10000
robot# show pbr nexthop-groups
Nexthop-Group: DONNA Table: 10001 Valid: 1 Installed: 1
	Valid: 1  nexthop 192.168.209.1
	Valid: 1  nexthop 192.168.210.1
	Valid: 1  nexthop 192.168.208.1

I have also introduced the ability to specify a table
in a 'show ip route table XXX' to see the specified tables.

robot# show ip route table 10001
Codes: K - kernel route, C - connected, S - static, R - RIP,
       O - OSPF, I - IS-IS, B - BGP, P - PIM, E - EIGRP, N - NHRP,
       T - Table, v - VNC, V - VNC-Direct, A - Babel, D - SHARP,
       F - PBR,
       > - selected route, * - FIB route

F>* 0.0.0.0/0 [0/0] via 192.168.208.1, enp0s8, 00:14:25
  *                 via 192.168.209.1, enp0s9, 00:14:25
  *                 via 192.168.210.1, enp0s10, 00:14:25

PBR tracks PBR-MAPS via the pbr-map command:

!
pbr-map EVA seq 10
  match src-ip 4.3.4.0/24
  set nexthop-group DONNA
!
pbr-map EVA seq 20
  match dst-ip 4.3.5.0/24
  set nexthop-group DONNA
!

pbr-maps can have 'match src-ip <prefix>' and 'match dst-ip <prefix>'
to affect decisions about incoming packets.  Additionally if you
only have one nexthop to use for a pbr-map you do not need
to setup a nexthop-group and can specify 'set nexthop XXXX'.

To apply the pbr-map to an incoming interface you do this:

interface enp0s10
 pbr-policy EVA
!

When a pbr-map is applied to interfaces it can be installed
into the kernel as a rule:

[sharpd@robot frr1]$ ip rule show
0:	from all lookup local
309:	from 4.3.4.0/24 iif enp0s10 lookup 10001
319:	from all to 4.3.5.0/24 iif enp0s10 lookup 10001
1000:	from all lookup [l3mdev-table]
32766:	from all lookup main
32767:	from all lookup default

[sharpd@robot frr1]$ ip route show table 10001
default proto pbr metric 20
	nexthop via 192.168.208.1 dev enp0s8 weight 1
	nexthop via 192.168.209.1 dev enp0s9 weight 1
	nexthop via 192.168.210.1 dev enp0s10 weight 1

The linux kernel now will use the rules and tables to properly
apply these policies.

Signed-off-by: Donald Sharp <sharpd@cumulusnetworks.com>
Signed-off-by: Don Slice <dslice@cumulusnetworks.com>
Signed-off-by: Quentin Young <qlyoung@cumulusnetworks.com>

19 files changed, 3733 insertions, 0 deletions

diff --git a/pbrd/.gitignore b/pbrd/.gitignore
new file mode 100644
index 000000000..ff95d8852
--- /dev/null
+++ b/pbrd/.gitignore
@@ -0,0 +1,15 @@
+!Makefile
+Makefile.in
+libpbr.a
+pbrd
+tags
+TAGS
+.deps
+*.o
+*.lo
+*.la
+*.libs
+.arch-inventory
+.arch-ids
+*~
+*.loT
diff --git a/pbrd/Makefile b/pbrd/Makefile
new file mode 100644
index 000000000..e8999c340
--- /dev/null
+++ b/pbrd/Makefile
@@ -0,0 +1,10 @@
+all: ALWAYS
+	@$(MAKE) -s -C .. pbrd/pbrd
+%: ALWAYS
+	@$(MAKE) -s -C .. pbrd/$@
+
+Makefile:
+	#nothing
+ALWAYS:
+.PHONY: ALWAYS makefiles
+.SUFFIXES:
diff --git a/pbrd/pbr_debug.c b/pbrd/pbr_debug.c
new file mode 100644
index 000000000..163459512
--- /dev/null
+++ b/pbrd/pbr_debug.c
@@ -0,0 +1,171 @@
+/*
+ * PBR - debugging
+ * Copyright (C) 2018 Cumulus Networks, Inc.
+ *                    Quentin Young
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; see the file COPYING; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+#include <zebra.h>
+
+#include "debug.h"
+#include "command.h"
+#include "vector.h"
+
+#ifndef VTYSH_EXTRACT_PL
+#include "pbrd/pbr_debug_clippy.c"
+#endif
+#include "pbrd/pbr_debug.h"
+
+struct debug pbr_dbg_map = {0, "PBR map"};
+struct debug pbr_dbg_zebra = {0, "PBR Zebra communications"};
+struct debug pbr_dbg_nht = {0, "PBR nexthop tracking"};
+struct debug pbr_dbg_event = {0, "PBR events"};
+
+struct debug *pbr_debugs[] = {&pbr_dbg_map, &pbr_dbg_zebra, &pbr_dbg_nht,
+			      &pbr_dbg_event};
+
+const char *pbr_debugs_conflines[] = {
+	"debug pbr map",
+	"debug pbr zebra",
+	"debug pbr nht",
+	"debug pbr events",
+};
+
+/*
+ * Set or unset flags on all debugs for pbrd.
+ *
+ * flags
+ *    The flags to set
+ *
+ * set
+ *    Whether to set or unset the specified flags
+ */
+static void pbr_debug_set_all(uint32_t flags, bool set)
+{
+	for (unsigned int i = 0; i < array_size(pbr_debugs); i++) {
+		DEBUG_FLAGS_SET(pbr_debugs[i], flags, set);
+
+		/* if all modes have been turned off, don't preserve options */
+		if (!DEBUG_MODE_CHECK(pbr_debugs[i], DEBUG_MODE_ALL))
+			DEBUG_CLEAR(pbr_debugs[i]);
+	}
+}
+
+/*
+ * Check flags on all debugs for pbrd.
+ *
+ * flags
+ *    The flags to set
+ *
+ * Returns:
+ *    The subset of the given flags that were set in all pbrd debugs
+ */
+static uint32_t pbr_debug_check_all(uint32_t flags)
+{
+	uint32_t mode = DEBUG_MODE_ALL;
+	for (unsigned int i = 0; i < array_size(pbr_debugs); i++)
+		mode &= DEBUG_MODE_CHECK(pbr_debugs[i], flags);
+	return mode;
+}
+
+static int pbr_debug_config_write_helper(struct vty *vty, bool config)
+{
+	uint32_t mode = DEBUG_MODE_ALL;
+
+	if (config)
+		mode = DEBUG_MODE_CONF;
+
+	if (pbr_debug_check_all(DEBUG_MODE_CONF) == mode) {
+		vty_out(vty, "debug pbr\n");
+		return 0;
+	}
+
+	for (unsigned int i = 0; i < array_size(pbr_debugs); i++)
+		if (DEBUG_MODE_CHECK(pbr_debugs[i], mode))
+			vty_out(vty, "%s\n", pbr_debugs_conflines[i]);
+	return 0;
+}
+
+int pbr_debug_config_write(struct vty *vty)
+{
+	return pbr_debug_config_write_helper(vty, true);
+}
+
+/* PBR debugging CLI ------------------------------------------------------- */
+/* clang-format off */
+
+DEFPY(debug_pbr,
+      debug_pbr_cmd,
+      "[no] debug pbr [{map$map|zebra$zebra|nht$nht|events$events}]",
+      NO_STR
+      DEBUG_STR
+      "Policy Based Routing\n"
+      "Policy maps\n"
+      "PBRD <-> Zebra communications\n"
+      "Nexthop tracking\n"
+      "Events\n")
+{
+	uint32_t mode = DEBUG_NODE2MODE(vty->node);
+	if (map)
+		DEBUG_MODE_SET(&pbr_dbg_map, mode, !no);
+	if (zebra)
+		DEBUG_MODE_SET(&pbr_dbg_zebra, mode, !no);
+	if (nht)
+		DEBUG_MODE_SET(&pbr_dbg_nht, mode, !no);
+	if (events)
+		DEBUG_MODE_SET(&pbr_dbg_event, mode, !no);
+
+	/* no specific debug --> act on all of them */
+	if (strmatch(argv[argc - 1]->text, "pbr"))
+		pbr_debug_set_all(mode, !no);
+
+	return CMD_SUCCESS;
+}
+
+DEFUN_NOSH(show_debugging_pbr,
+	   show_debugging_pbr_cmd,
+	   "show debugging [pbr]",
+	   SHOW_STR
+	   DEBUG_STR
+	   "Policy Based Routing\n")
+{
+	vty_out(vty, "PBR debugging status:\n");
+
+	pbr_debug_config_write_helper(vty, false);
+
+	return CMD_SUCCESS;
+}
+
+/* clang-format on */
+/* ------------------------------------------------------------------------- */
+
+static struct cmd_node debug_node = {DEBUG_NODE, "", 1};
+
+struct debug_callbacks pbr_dbg_cbs = {.debug_set_all = pbr_debug_set_all};
+
+void pbr_debug_init(void)
+{
+	debug_init(&pbr_dbg_cbs);
+}
+
+void pbr_debug_init_vty(void)
+{
+	install_node(&debug_node, pbr_debug_config_write);
+
+	install_element(VIEW_NODE, &debug_pbr_cmd);
+	install_element(CONFIG_NODE, &debug_pbr_cmd);
+
+	install_element(VIEW_NODE, &show_debugging_pbr_cmd);
+}
diff --git a/pbrd/pbr_debug.h b/pbrd/pbr_debug.h
new file mode 100644
index 000000000..274472462
--- /dev/null
+++ b/pbrd/pbr_debug.h
@@ -0,0 +1,53 @@
+/*
+ * PBR - debugging
+ * Copyright (C) 2018 Cumulus Networks, Inc.
+ *                    Quentin Young
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; see the file COPYING; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+#ifndef __PBR_DEBUG_H__
+#define __PBR_DEBUG_H__
+
+#include <zebra.h>
+
+#include "debug.h"
+
+/* PBR debugging records */
+extern struct debug pbr_dbg_map;
+extern struct debug pbr_dbg_zebra;
+extern struct debug pbr_dbg_nht;
+extern struct debug pbr_dbg_event;
+
+/*
+ * Initialize PBR debugging.
+ *
+ * Installs VTY commands and registers callbacks.
+ */
+void pbr_debug_init(void);
+
+/*
+ * Install PBR debugging VTY commands.
+ */
+void pbr_debug_init_vty(void);
+
+/*
+ * Print PBR debugging configuration.
+ *
+ * vty
+ *    VTY to print debugging configuration to.
+ */
+int pbr_debug_config_write(struct vty *vty);
+
+#endif /* __PBR_DEBUG_H__ */
diff --git a/pbrd/pbr_event.c b/pbrd/pbr_event.c
new file mode 100644
index 000000000..72a59801b
--- /dev/null
+++ b/pbrd/pbr_event.c
@@ -0,0 +1,219 @@
+/*
+ * PBR-event Code
+ * Copyright (C) 2018 Cumulus Networks, Inc.
+ *               Donald Sharp
+ *
+ * This file is part of FRR.
+ *
+ * FRR is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2, or (at your option) any
+ * later version.
+ *
+ * FRR is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; see the file COPYING; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+#include <zebra.h>
+
+#include <thread.h>
+#include <workqueue.h>
+#include <nexthop.h>
+#include <log.h>
+#include <vty.h>
+
+#include "pbrd/pbr_event.h"
+#include "pbrd/pbr_map.h"
+#include "pbrd/pbr_nht.h"
+#include "pbrd/pbr_memory.h"
+#include "pbrd/pbr_debug.h"
+
+DEFINE_MTYPE_STATIC(PBRD, PBR_EVENT, "Event WorkQueue")
+
+struct work_queue *pbr_event_wq;
+
+static const char *pbr_event_wqentry2str(struct pbr_event *pbre,
+					 char *buffer, size_t buflen)
+{
+	switch(pbre->event) {
+	case PBR_NHG_NEW:
+		snprintf(buffer, buflen, "Nexthop Group Added %s",
+			 pbre->name);
+		break;
+	case PBR_NHG_ADD_NEXTHOP:
+		snprintf(buffer, buflen, "Nexthop Group Nexthop Added %s",
+			 pbre->name);
+		break;
+	case PBR_NHG_DEL_NEXTHOP:
+		snprintf(buffer, buflen, "Nexthop Group Nexthop Deleted %s",
+			 pbre->name);
+		break;
+	case PBR_NHG_DELETE:
+		snprintf(buffer, buflen, "Nexthop Group Deleted %s",
+			 pbre->name);
+		break;
+	case PBR_MAP_NEXTHOP_ADD:
+		snprintf(buffer, buflen, "Nexthop Added to %s(%d)", pbre->name,
+			 pbre->seqno);
+		break;
+	case PBR_MAP_NEXTHOP_DELETE:
+		snprintf(buffer, buflen, "Nexthop Deleted from %s(%d)",
+			 pbre->name, pbre->seqno);
+		break;
+	case PBR_MAP_NHG_ADD:
+		snprintf(buffer, buflen, "Nexthop Group Added to %s(%d)",
+			 pbre->name, pbre->seqno);
+		break;
+	case PBR_MAP_NHG_DELETE:
+		snprintf(buffer, buflen, "Nexthop Group Deleted from %s(%d)",
+			 pbre->name, pbre->seqno);
+		break;
+	case PBR_MAP_ADD:
+		snprintf(buffer, buflen, "PBR-MAP %s Added",
+			 pbre->name);
+		break;
+	case PBR_MAP_MODIFY:
+		snprintf(buffer, buflen, "PBR_MAP %s Modified",
+			 pbre->name);
+		break;
+	case PBR_MAP_DELETE:
+		snprintf(buffer, buflen, "PBR_MAP %s Deleted",
+			 pbre->name);
+		break;
+	case PBR_NH_CHANGED:
+		snprintf(buffer, buflen, "Nexthop Call back from Zebra");
+		break;
+	case PBR_MAP_INSTALL:
+		snprintf(buffer, buflen, "PBR_MAP %s Installing into zapi",
+			 pbre->name);
+		break;
+	case PBR_POLICY_CHANGED:
+		snprintf(buffer, buflen,
+			 "PBR-Policy %s applied to an interface", pbre->name);
+		break;
+	case PBR_MAP_POLICY_INSTALL:
+		snprintf(buffer, buflen, "PBR-POLICY installation time for %s",
+			 pbre->name);
+		break;
+	case PBR_POLICY_DELETED:
+		snprintf(buffer, buflen, "PBR-POLICY deleted from %s",
+			 pbre->name);
+		break;
+	}
+
+	return buffer;
+}
+
+void pbr_event_free(struct pbr_event **pbre)
+{
+	XFREE(MTYPE_PBR_EVENT, *pbre);
+}
+
+static void pbr_event_delete_wq(struct work_queue *wq, void *data)
+{
+	struct pbr_event *pbre = (struct pbr_event *)data;
+
+	XFREE(MTYPE_PBR_EVENT, pbre);
+}
+
+static wq_item_status pbr_event_process_wq(struct work_queue *wq, void *data)
+{
+	struct pbr_event *pbre = (struct pbr_event *)data;
+	char buffer[256];
+
+	DEBUGD(&pbr_dbg_event, "%s: Handling event %s", __PRETTY_FUNCTION__,
+	       pbr_event_wqentry2str(pbre, buffer, sizeof(buffer)));
+
+	switch (pbre->event) {
+	case PBR_NHG_NEW:
+		pbr_nht_add_group(pbre->name);
+		pbr_map_check_nh_group_change(pbre->name);
+		break;
+	case PBR_NHG_ADD_NEXTHOP:
+		pbr_nht_change_group(pbre->name);
+		pbr_map_check_nh_group_change(pbre->name);
+		break;
+	case PBR_NHG_DEL_NEXTHOP:
+		pbr_nht_change_group(pbre->name);
+		pbr_map_check_nh_group_change(pbre->name);
+		break;
+	case PBR_NHG_DELETE:
+		pbr_nht_delete_group(pbre->name);
+		pbr_map_check_nh_group_change(pbre->name);
+		break;
+	case PBR_MAP_NEXTHOP_ADD:
+		pbr_nht_add_individual_nexthop(pbre->name, pbre->seqno);
+		pbr_map_check(pbre->name, pbre->seqno);
+		break;
+	case PBR_MAP_NEXTHOP_DELETE:
+		pbr_nht_delete_individual_nexthop(pbre->name, pbre->seqno);
+		pbr_map_check(pbre->name, pbre->seqno);
+		break;
+	case PBR_MAP_NHG_ADD:
+		pbr_map_check(pbre->name, pbre->seqno);
+		break;
+	case PBR_MAP_NHG_DELETE:
+		pbr_map_check(pbre->name, pbre->seqno);
+		break;
+	case PBR_MAP_ADD:
+		pbr_map_add_interfaces(pbre->name);
+		break;
+	case PBR_MAP_MODIFY:
+		pbr_map_check(pbre->name, pbre->seqno);
+		break;
+	case PBR_MAP_DELETE:
+		pbr_map_delete(pbre->name, pbre->seqno);
+		break;
+	case PBR_NH_CHANGED:
+		pbr_map_check_nh_group_change(pbre->name);
+		break;
+	case PBR_MAP_INSTALL:
+		pbr_map_install(pbre->name);
+		break;
+	case PBR_POLICY_CHANGED:
+		pbr_map_check_policy_change(pbre->name);
+		break;
+	case PBR_MAP_POLICY_INSTALL:
+		pbr_map_policy_install(pbre->name);
+		break;
+	case PBR_POLICY_DELETED:
+		pbr_map_policy_delete(pbre->name);
+		break;
+	}
+
+	return WQ_SUCCESS;
+}
+
+void pbr_event_enqueue(struct pbr_event *pbre)
+{
+	work_queue_add(pbr_event_wq, pbre);
+}
+
+struct pbr_event *pbr_event_new(enum pbr_events ev, const char *name)
+{
+	struct pbr_event *event;
+	event = XCALLOC(MTYPE_PBR_EVENT, sizeof(struct pbr_event));
+	event->event = ev;
+	if (name)
+		strlcpy(event->name, name, sizeof(event->name));
+	return event;
+}
+
+extern struct thread_master *master;
+
+void pbr_event_init(void)
+{
+	pbr_event_wq = work_queue_new(master, "PBR Main Work Queue");
+	pbr_event_wq->spec.workfunc = &pbr_event_process_wq;
+	pbr_event_wq->spec.del_item_data = &pbr_event_delete_wq;
+}
+
+void pbr_event_stop(void)
+{
+	work_queue_free_and_null(&pbr_event_wq);
+}
diff --git a/pbrd/pbr_event.h b/pbrd/pbr_event.h
new file mode 100644
index 000000000..2d895d969
--- /dev/null
+++ b/pbrd/pbr_event.h
@@ -0,0 +1,137 @@
+/*
+ * PBR-event Header
+ * Copyright (C) 2018 Cumulus Networks, Inc.
+ *               Donald Sharp
+ *
+ * This file is part of FRR.
+ *
+ * FRR is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2, or (at your option) any
+ * later version.
+ *
+ * FRR is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; see the file COPYING; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+#ifndef __PBR_EVENT_H__
+#define __PBR_EVENT_H__
+
+enum pbr_events {
+	/*
+	 * A NHG has been added to the system, handle it
+	 */
+	PBR_NHG_NEW,
+
+	/*
+	 * A NHG has been modified( added a new nexthop )
+	 */
+	PBR_NHG_ADD_NEXTHOP,
+
+	/*
+	 * A NHG has been modified( deleted a nexthop )
+	 */
+	PBR_NHG_DEL_NEXTHOP,
+
+	/*
+	 * A NHG has been deleted from the system
+	 */
+	PBR_NHG_DELETE,
+
+	/*
+	 * A individual nexthop has been added
+	 */
+	PBR_MAP_NEXTHOP_ADD,
+
+	/*
+	 * A individual nexthop has been deleted
+	 */
+	PBR_MAP_NEXTHOP_DELETE,
+
+	/*
+	 * A nexthop group has been added to a pbr-map
+	 */
+	PBR_MAP_NHG_ADD,
+
+	/*
+	 * A nexthop group has been deleted from a pbr-map
+	 */
+	PBR_MAP_NHG_DELETE,
+
+	/*
+	 * A new pbr-map has been created
+	 */
+	PBR_MAP_ADD,
+
+	/*
+	 * The pbr-map has been modified in some fashion
+	 */
+	PBR_MAP_MODIFY,
+
+	/*
+	 * The pbr-map has been deleted from the system
+	 */
+	PBR_MAP_DELETE,
+
+	/*
+	 * Start the sequence of events to install/remove the policy
+	 * from being installed
+	 */
+	PBR_MAP_INSTALL,
+
+	/*
+	 * We believe we have gotten enough information to actually
+	 * install the rule portion, since the nexthops are installed
+	 */
+	PBR_MAP_POLICY_INSTALL,
+
+	/*
+	 * Callbacks for a Nexthop in a nexthop group has been
+	 * changed in some fashion
+	 */
+	PBR_NH_CHANGED,
+
+	/*
+	 * Callback for when a policy has been applied to an interface
+	 */
+	PBR_POLICY_CHANGED,
+
+	/*
+	 * Callback for when a interface has been issued a no
+	 * policy command
+	 */
+	PBR_POLICY_DELETED,
+};
+
+struct pbr_event {
+	enum pbr_events event;
+
+	char name[100];
+	union g_addr addr;
+	uint32_t seqno;
+};
+
+/*
+ * Return a event structure that can be filled in and enqueued.
+ * Assume this memory is owned by the event subsystem.
+ */
+extern struct pbr_event *pbr_event_new(enum pbr_events ev, const char *name);
+
+/*
+ * Free the associated pbr_event item
+ */
+extern void pbr_event_free(struct pbr_event **pbre);
+
+/*
+ * Enqueue an event for later processing
+ */
+void pbr_event_enqueue(struct pbr_event *pbre);
+
+extern void pbr_event_init(void);
+extern void pbr_event_stop(void);
+#endif
diff --git a/pbrd/pbr_main.c b/pbrd/pbr_main.c
new file mode 100644
index 000000000..c8ee2f599
--- /dev/null
+++ b/pbrd/pbr_main.c
@@ -0,0 +1,168 @@
+/*
+ * PBR - main code
+ * Copyright (C) 2018 Cumulus Networks, Inc.
+ *               Donald Sharp
+ *
+ * FRR is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2, or (at your option) any
+ * later version.
+ *
+ * FRR is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; see the file COPYING; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+#include <zebra.h>
+
+#include <lib/version.h>
+#include "getopt.h"
+#include "thread.h"
+#include "prefix.h"
+#include "linklist.h"
+#include "if.h"
+#include "vector.h"
+#include "vty.h"
+#include "command.h"
+#include "filter.h"
+#include "plist.h"
+#include "stream.h"
+#include "log.h"
+#include "memory.h"
+#include "privs.h"
+#include "sigevent.h"
+#include "zclient.h"
+#include "keychain.h"
+#include "distribute.h"
+#include "libfrr.h"
+#include "routemap.h"
+#include "nexthop.h"
+#include "nexthop_group.h"
+
+#include "pbr_nht.h"
+#include "pbr_map.h"
+#include "pbr_zebra.h"
+#include "pbr_event.h"
+#include "pbr_vty.h"
+#include "pbr_debug.h"
+
+zebra_capabilities_t _caps_p[] = {
+	ZCAP_NET_RAW, ZCAP_BIND, ZCAP_NET_ADMIN,
+};
+
+struct zebra_privs_t pbr_privs = {
+#if defined(FRR_USER) && defined(FRR_GROUP)
+	.user = FRR_USER,
+	.group = FRR_GROUP,
+#endif
+#if defined(VTY_GROUP)
+	.vty_group = VTY_GROUP,
+#endif
+	.caps_p = _caps_p,
+	.cap_num_p = array_size(_caps_p),
+	.cap_num_i = 0};
+
+struct option longopts[] = {{0}};
+
+/* Master of threads. */
+struct thread_master *master;
+
+/* SIGHUP handler. */
+static void sighup(void)
+{
+	zlog_info("SIGHUP received");
+}
+
+/* SIGINT / SIGTERM handler. */
+static void sigint(void)
+{
+	zlog_notice("Terminating on signal");
+
+	pbr_event_stop();
+	exit(0);
+}
+
+/* SIGUSR1 handler. */
+static void sigusr1(void)
+{
+	zlog_rotate();
+}
+
+struct quagga_signal_t pbr_signals[] = {
+	{
+		.signal = SIGHUP,
+		.handler = &sighup,
+	},
+	{
+		.signal = SIGUSR1,
+		.handler = &sigusr1,
+	},
+	{
+		.signal = SIGINT,
+		.handler = &sigint,
+	},
+	{
+		.signal = SIGTERM,
+		.handler = &sigint,
+	},
+};
+
+#define PBR_VTY_PORT 2615
+
+FRR_DAEMON_INFO(pbrd, PBR, .vty_port = PBR_VTY_PORT,
+
+		.proghelp = "Implementation of PBR.",
+
+		.signals = pbr_signals,
+		.n_signals = array_size(pbr_signals),
+
+		.privs = &pbr_privs, )
+
+int main(int argc, char **argv, char **envp)
+{
+	frr_preinit(&pbrd_di, argc, argv);
+	frr_opt_add("", longopts, "");
+
+	while (1) {
+		int opt;
+
+		opt = frr_getopt(argc, argv, NULL);
+
+		if (opt == EOF)
+			break;
+
+		switch (opt) {
+		case 0:
+			break;
+		default:
+			frr_help_exit(1);
+			break;
+		}
+	}
+
+	master = frr_init();
+
+	pbr_debug_init();
+
+	vrf_init(NULL, NULL, NULL, NULL);
+	nexthop_group_init(pbr_nhgroup_add_cb,
+			   pbr_nhgroup_add_nexthop_cb,
+			   pbr_nhgroup_del_nexthop_cb,
+			   pbr_nhgroup_delete_cb);
+
+	pbr_event_init();
+	pbr_nht_init();
+	pbr_map_init();
+	pbr_zebra_init();
+	pbr_vty_init();
+
+	frr_config_fork();
+	frr_run(master);
+
+	/* Not reached. */
+	return 0;
+}
diff --git a/pbrd/pbr_map.c b/pbrd/pbr_map.c
new file mode 100644
index 000000000..07607af50
--- /dev/null
+++ b/pbrd/pbr_map.c
@@ -0,0 +1,630 @@
+/*
+ * PBR-map Code
+ * Copyright (C) 2018 Cumulus Networks, Inc.
+ *               Donald Sharp
+ *
+ * FRR is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2, or (at your option) any
+ * later version.
+ *
+ * FRR is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; see the file COPYING; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+#include <zebra.h>
+
+#include "thread.h"
+#include "linklist.h"
+#include "prefix.h"
+#include "table.h"
+#include "vrf.h"
+#include "nexthop.h"
+#include "nexthop_group.h"
+#include "memory.h"
+#include "log.h"
+#include "vty.h"
+
+#include "pbr_nht.h"
+#include "pbr_map.h"
+#include "pbr_event.h"
+#include "pbr_zebra.h"
+#include "pbr_memory.h"
+#include "pbr_debug.h"
+
+DEFINE_MTYPE_STATIC(PBRD, PBR_MAP, "PBR Map")
+DEFINE_MTYPE_STATIC(PBRD, PBR_MAP_SEQNO, "PBR Map Sequence")
+DEFINE_MTYPE_STATIC(PBRD, PBR_MAP_INTERFACE, "PBR Map Interface")
+
+static uint32_t pbr_map_sequence_unique;
+
+static __inline int pbr_map_compare(const struct pbr_map *pbrmap1,
+				    const struct pbr_map *pbrmap2);
+
+RB_GENERATE(pbr_map_entry_head, pbr_map, pbr_map_entry, pbr_map_compare)
+
+struct pbr_map_entry_head pbr_maps = RB_INITIALIZER(&pbr_maps);
+
+DEFINE_QOBJ_TYPE(pbr_map_sequence)
+
+static __inline int pbr_map_compare(const struct pbr_map *pbrmap1,
+				    const struct pbr_map *pbrmap2)
+{
+	return strcmp(pbrmap1->name, pbrmap2->name);
+}
+
+static int pbr_map_sequence_compare(const struct pbr_map_sequence *pbrms1,
+				    const struct pbr_map_sequence *pbrms2)
+{
+	if (pbrms1->seqno == pbrms2->seqno)
+		return 0;
+
+	if (pbrms1->seqno < pbrms2->seqno)
+		return -1;
+
+	return 1;
+}
+
+static void pbr_map_sequence_delete(struct pbr_map_sequence *pbrms)
+{
+	if (pbrms->internal_nhg_name)
+		XFREE(MTYPE_TMP, pbrms->internal_nhg_name);
+
+	XFREE(MTYPE_PBR_MAP_SEQNO, pbrms);
+}
+
+static int pbr_map_interface_compare(const struct pbr_map_interface *pmi1,
+				     const struct pbr_map_interface *pmi2)
+{
+	return strcmp(pmi1->ifp->name, pmi2->ifp->name);
+}
+
+static void pbr_map_interface_list_delete(const struct pbr_map_interface *pmi)
+{
+	pbr_map_policy_delete(pmi->ifp->name);
+}
+
+static const char *pbr_map_reason_str[] = {
+	"Invalid NH-group",     "Invalid NH",	 "No Nexthops",
+	"Both NH and NH-Group", "Invalid Src or Dst", "Deleting Sequence",
+};
+
+void pbr_map_reason_string(unsigned int reason, char *buf, int size)
+{
+	unsigned int bit;
+	int len = 0;
+
+	if (!buf)
+		return;
+
+	for (bit = 0; bit < array_size(pbr_map_reason_str); bit++) {
+		if ((reason & (1 << bit)) && (len < size)) {
+			len += snprintf((buf + len), (size - len), "%s%s",
+					(len > 0) ? ", " : "",
+					pbr_map_reason_str[bit]);
+		}
+	}
+}
+
+
+void pbr_map_interface_delete(struct pbr_map *pbrm, struct interface *ifp_del)
+{
+
+	struct listnode *node;
+	struct pbr_map_interface *pmi;
+	struct pbr_event *pbre;
+
+	for (ALL_LIST_ELEMENTS_RO(pbrm->incoming, node, pmi)) {
+		if (ifp_del == pmi->ifp)
+			break;
+	}
+
+	if (pmi) {
+		pmi->delete = true;
+
+		pbre = pbr_event_new(PBR_POLICY_DELETED, pmi->ifp->name);
+		pbr_event_enqueue(pbre);
+	}
+}
+
+void pbr_map_add_interface(struct pbr_map *pbrm, struct interface *ifp_add)
+{
+	struct listnode *node;
+	struct pbr_map_interface *pmi;
+	struct pbr_event *pbre;
+
+	for (ALL_LIST_ELEMENTS_RO(pbrm->incoming, node, pmi)) {
+		if (ifp_add == pmi->ifp)
+			return;
+	}
+
+	pmi = XCALLOC(MTYPE_PBR_MAP_INTERFACE, sizeof(*pmi));
+	pmi->ifp = ifp_add;
+	pmi->pbrm = pbrm;
+	listnode_add_sort(pbrm->incoming, pmi);
+
+	pbre = pbr_event_new(PBR_POLICY_CHANGED, pbrm->name);
+	pbr_event_enqueue(pbre);
+}
+
+void pbr_map_write_interfaces(struct vty *vty, struct interface *ifp)
+{
+	struct pbr_interface *pbr_ifp = ifp->info;
+
+	if (!(strcmp(pbr_ifp->mapname, "") == 0))
+		vty_out(vty, " pbr-policy %s\n", pbr_ifp->mapname);
+}
+
+struct pbr_map *pbrm_find(const char *name)
+{
+	struct pbr_map pbrm;
+
+	strlcpy(pbrm.name, name, sizeof(pbrm.name));
+
+	return RB_FIND(pbr_map_entry_head, &pbr_maps, &pbrm);
+}
+
+extern void pbr_map_delete(const char *name, uint32_t seqno)
+{
+	struct pbr_map *pbrm;
+	struct pbr_map_sequence *pbrms;
+	struct listnode *node, *nnode;
+	bool uninstall = false;
+
+	pbrm = pbrm_find(name);
+
+	for (ALL_LIST_ELEMENTS(pbrm->seqnumbers, node, nnode, pbrms)) {
+		if (pbrms->reason & PBR_MAP_DEL_SEQUENCE_NUMBER) {
+			uninstall = true;
+			break;
+		}
+	}
+
+	if (uninstall)
+		pbr_send_pbr_map(pbrm, 0);
+
+	for (ALL_LIST_ELEMENTS(pbrm->seqnumbers, node, nnode, pbrms)) {
+		if (!(pbrms->reason & PBR_MAP_DEL_SEQUENCE_NUMBER))
+			continue;
+
+		if (pbrms->nhg)
+			pbr_nht_delete_individual_nexthop(pbrms->parent->name,
+							  pbrms->seqno);
+
+		listnode_delete(pbrm->seqnumbers, pbrms);
+	}
+
+	if (pbrm->seqnumbers->count == 0) {
+		RB_REMOVE(pbr_map_entry_head, &pbr_maps, pbrm);
+		XFREE(MTYPE_PBR_MAP, pbrm);
+	}
+}
+
+extern struct pbr_map_sequence *pbrms_lookup_unique(uint32_t unique,
+						    ifindex_t ifindex)
+{
+	struct pbr_map_sequence *pbrms;
+	struct listnode *snode, *inode;
+	struct pbr_map_interface *pmi;
+	struct pbr_map *pbrm;
+
+	RB_FOREACH (pbrm, pbr_map_entry_head, &pbr_maps) {
+		for (ALL_LIST_ELEMENTS_RO(pbrm->incoming, inode, pmi)) {
+			if (pmi->ifp->ifindex != ifindex)
+				continue;
+
+			for (ALL_LIST_ELEMENTS_RO(pbrm->seqnumbers, snode,
+						  pbrms)) {
+				DEBUGD(&pbr_dbg_map, "%s: Comparing %u to %u",
+				       __PRETTY_FUNCTION__, pbrms->unique,
+				       unique);
+				if (pbrms->unique == unique)
+					return pbrms;
+			}
+		}
+	}
+
+	return NULL;
+}
+
+extern struct pbr_map_sequence *pbrms_get(const char *name, uint32_t seqno)
+{
+	struct pbr_map *pbrm;
+	struct pbr_map_sequence *pbrms;
+	struct listnode *node;
+	struct pbr_event *pbre;
+
+	pbrm = pbrm_find(name);
+	if (!pbrm) {
+		pbrm = XCALLOC(MTYPE_PBR_MAP, sizeof(*pbrm));
+		strcpy(pbrm->name, name);
+
+		pbrm->seqnumbers = list_new();
+		pbrm->seqnumbers->cmp =
+			(int (*)(void *, void *))pbr_map_sequence_compare;
+		pbrm->seqnumbers->del =
+			 (void (*)(void *))pbr_map_sequence_delete;
+
+		pbrm->incoming = list_new();
+		pbrm->incoming->cmp =
+			(int (*)(void *, void *))pbr_map_interface_compare;
+		pbrm->incoming->del =
+			(void (*)(void *))pbr_map_interface_list_delete;
+
+		RB_INSERT(pbr_map_entry_head, &pbr_maps, pbrm);
+
+		pbre = pbr_event_new(PBR_MAP_ADD, name);
+	} else
+		pbre = NULL;
+
+	for (ALL_LIST_ELEMENTS_RO(pbrm->seqnumbers, node, pbrms)) {
+		if (pbrms->seqno == seqno)
+			break;
+
+	}
+
+	if (!pbrms) {
+		pbrms = XCALLOC(MTYPE_PBR_MAP_SEQNO, sizeof(*pbrms));
+		pbrms->unique = pbr_map_sequence_unique++;
+		pbrms->seqno = seqno;
+		pbrms->ruleno = pbr_nht_get_next_rule(seqno);
+		pbrms->parent = pbrm;
+		pbrms->reason =
+			PBR_MAP_INVALID_SRCDST |
+			PBR_MAP_INVALID_NO_NEXTHOPS;
+
+		QOBJ_REG(pbrms, pbr_map_sequence);
+		listnode_add_sort(pbrm->seqnumbers, pbrms);
+
+		pbrm->installed = false;
+	}
+
+	if (pbre)
+		pbr_event_enqueue(pbre);
+
+	return pbrms;
+}
+
+static void
+pbr_map_sequence_check_nexthops_valid(struct pbr_map_sequence *pbrms)
+{
+	/*
+	 * Check validness of the nexthop or nexthop-group
+	 */
+	if (!pbrms->nhg && !pbrms->nhgrp_name)
+		pbrms->reason |= PBR_MAP_INVALID_NO_NEXTHOPS;
+
+	if (pbrms->nhg && pbrms->nhgrp_name)
+		pbrms->reason |= PBR_MAP_INVALID_BOTH_NHANDGRP;
+
+	if (pbrms->nhg &&
+	    !pbr_nht_nexthop_group_valid(pbrms->internal_nhg_name))
+		pbrms->reason |= PBR_MAP_INVALID_NEXTHOP;
+
+	if (pbrms->nhgrp_name) {
+		if (!pbr_nht_nexthop_group_valid(pbrms->nhgrp_name))
+			pbrms->reason |= PBR_MAP_INVALID_NEXTHOP_GROUP;
+		else
+			pbrms->nhs_installed = true;
+	}
+}
+
+static void pbr_map_sequence_check_src_dst_valid(struct pbr_map_sequence *pbrms)
+{
+	if (!pbrms->src && !pbrms->dst)
+		pbrms->reason |= PBR_MAP_INVALID_SRCDST;
+}
+
+/*
+ * Checks to see if we think that the pbmrs is valid.  If we think
+ * the config is valid return true.
+ */
+static void pbr_map_sequence_check_valid(struct pbr_map_sequence *pbrms)
+{
+	pbr_map_sequence_check_nexthops_valid(pbrms);
+
+	pbr_map_sequence_check_src_dst_valid(pbrms);
+}
+
+static bool pbr_map_check_valid_internal(struct pbr_map *pbrm)
+{
+	struct pbr_map_sequence *pbrms;
+	struct listnode *node;
+
+	pbrm->valid = true;
+	for (ALL_LIST_ELEMENTS_RO(pbrm->seqnumbers, node, pbrms)) {
+		pbrms->reason = 0;
+		pbr_map_sequence_check_valid(pbrms);
+		/*
+		 * A pbr_map_sequence that is invalid causes
+		 * the whole shebang to be invalid
+		 */
+		if (pbrms->reason != 0)
+			pbrm->valid = false;
+	}
+
+	return pbrm->valid;
+}
+
+/*
+ * For a given PBR-MAP check to see if we think it is a
+ * valid config or not.  If so note that it is and return
+ * that we are valid.
+ */
+extern bool pbr_map_check_valid(const char *name)
+{
+	struct pbr_map *pbrm;
+
+	pbrm = pbrm_find(name);
+	if (!pbrm) {
+		DEBUGD(&pbr_dbg_map,
+		       "%s: Specified PBR-MAP(%s) does not exist?",
+		       __PRETTY_FUNCTION__, name);
+		return false;
+	}
+
+	pbr_map_check_valid_internal(pbrm);
+	return pbrm->valid;
+}
+
+extern void pbr_map_schedule_policy_from_nhg(const char *nh_group)
+{
+	struct pbr_map_sequence *pbrms;
+	struct pbr_event *pbre;
+	struct pbr_map *pbrm;
+	struct listnode *node;
+
+	RB_FOREACH (pbrm, pbr_map_entry_head, &pbr_maps) {
+		DEBUGD(&pbr_dbg_map, "%s: Looking at %s", __PRETTY_FUNCTION__,
+		       pbrm->name);
+		for (ALL_LIST_ELEMENTS_RO(pbrm->seqnumbers, node, pbrms)) {
+			DEBUGD(&pbr_dbg_map, "\tNH Grp name: %s",
+			       pbrms->nhgrp_name ? pbrms->nhgrp_name : "NULL");
+
+			if (pbrms->nhgrp_name
+			    && (strcmp(nh_group, pbrms->nhgrp_name) == 0)) {
+				pbrms->nhs_installed = true;
+
+				pbre = pbr_event_new(PBR_MAP_MODIFY,
+						     pbrm->name);
+				pbre->seqno = pbrms->seqno;
+
+				pbr_event_enqueue(pbre);
+			}
+
+			if (pbrms->nhg
+			    && (strcmp(nh_group, pbrms->internal_nhg_name)
+				== 0)) {
+				pbrms->nhs_installed = true;
+
+				pbre = pbr_event_new(PBR_MAP_MODIFY,
+						     pbrm->name);
+				pbre->seqno = pbrms->seqno;
+
+				pbr_event_enqueue(pbre);
+			}
+		}
+	}
+}
+
+extern void pbr_map_policy_install(const char *name)
+{
+	struct pbr_map_sequence *pbrms;
+	struct pbr_map *pbrm;
+	struct listnode *node;
+	bool install;
+
+	DEBUGD(&pbr_dbg_map, "%s: for %s", __PRETTY_FUNCTION__, name);
+	pbrm = pbrm_find(name);
+	if (!pbrm)
+		return;
+
+	install = true;
+	for (ALL_LIST_ELEMENTS_RO(pbrm->seqnumbers, node, pbrms)) {
+		DEBUGD(&pbr_dbg_map,
+		       "%s: Looking at what to install %s(%u) %d %d",
+		       __PRETTY_FUNCTION__, name, pbrms->seqno, pbrm->valid,
+		       pbrms->nhs_installed);
+		if (!pbrm->valid || !pbrms->nhs_installed)
+			install = false;
+	}
+
+	if (install && pbrm->incoming->count) {
+		DEBUGD(&pbr_dbg_map, "\tInstalling");
+		pbr_send_pbr_map(pbrm, true);
+	}
+}
+
+extern void pbr_map_policy_delete(const char *ifname)
+{
+	struct listnode *node, *nnode;
+	struct pbr_map_interface *pmi;
+	struct pbr_map *pbrm;
+
+	RB_FOREACH (pbrm, pbr_map_entry_head, &pbr_maps) {
+		for (ALL_LIST_ELEMENTS(pbrm->incoming, node, nnode, pmi)) {
+			DEBUGD(&pbr_dbg_map, "Comparing %s to %s %d",
+			       pmi->ifp->name, ifname, pmi->delete);
+			if (strcmp(ifname, pmi->ifp->name) != 0)
+				continue;
+
+			pbr_send_pbr_map(pbrm, false);
+			listnode_delete(pbrm->incoming, pmi);
+			pmi->pbrm = NULL;
+			XFREE(MTYPE_PBR_MAP_INTERFACE, pmi);
+		}
+	}
+}
+
+/*
+ * For a nexthop group specified, see if any of the pbr-maps
+ * are using it and if so, check to see that we are still
+ * valid for usage.  If we are valid then schedule the installation/deletion
+ * of the pbr-policy.
+ */
+extern void pbr_map_check_nh_group_change(const char *nh_group)
+{
+	struct pbr_map_sequence *pbrms;
+	struct pbr_map *pbrm;
+	struct listnode *node;
+	bool found_name;
+
+	zlog_warn("*** %s for %s ***", __func__, nh_group);
+	RB_FOREACH (pbrm, pbr_map_entry_head, &pbr_maps) {
+		for (ALL_LIST_ELEMENTS_RO(pbrm->seqnumbers, node, pbrms)) {
+			found_name = false;
+			if (pbrms->nhgrp_name)
+				found_name =
+					!strcmp(nh_group, pbrms->nhgrp_name);
+			else if (pbrms->nhg)
+				found_name = !strcmp(nh_group,
+						     pbrms->internal_nhg_name);
+
+			if (found_name) {
+				bool original = pbrm->valid;
+				zlog_warn("*** %s pbrm->valid is %u ***",
+					  __func__, pbrm->valid);
+
+				pbr_map_check_valid_internal(pbrm);
+
+				if (original != pbrm->valid) {
+					struct pbr_event *pbre;
+
+					pbre = pbr_event_new(PBR_MAP_INSTALL,
+							     pbrm->name);
+					pbr_event_enqueue(pbre);
+				}
+				break;
+			}
+		}
+	}
+}
+
+extern void pbr_map_check(const char *name, uint32_t seqno)
+{
+	struct pbr_map_sequence *pbrms;
+	struct listnode *node;
+	struct pbr_map *pbrm;
+
+	DEBUGD(&pbr_dbg_map, "%s: for %s(%u)", __PRETTY_FUNCTION__, name,
+	       seqno);
+	if (pbr_map_check_valid(name))
+		DEBUGD(&pbr_dbg_map, "We are totally valid %s\n", name);
+
+	pbrm = pbrm_find(name);
+	if (!pbrm)
+		return;
+
+	for (ALL_LIST_ELEMENTS_RO(pbrm->seqnumbers, node, pbrms)) {
+		if (seqno != pbrms->seqno)
+			continue;
+
+		DEBUGD(&pbr_dbg_map, "%s: Installing %s(%u) reason: %" PRIu64,
+		       __PRETTY_FUNCTION__, name, seqno, pbrms->reason);
+
+		if (pbrms->reason == PBR_MAP_VALID_SEQUENCE_NUMBER) {
+			struct pbr_event *pbre;
+
+			DEBUGD(&pbr_dbg_map,
+			       "%s: Installing %s(%u) reason: %" PRIu64,
+			       __PRETTY_FUNCTION__, name, seqno, pbrms->reason);
+			DEBUGD(&pbr_dbg_map,
+			       "\tSending PBR_MAP_POLICY_INSTALL event");
+
+			pbre = pbr_event_new(PBR_MAP_POLICY_INSTALL,
+					     pbrm->name);
+			pbre->event = PBR_MAP_POLICY_INSTALL;
+			strcpy(pbre->name, pbrm->name);
+
+			pbr_event_enqueue(pbre);
+
+			break;
+		} else {
+			DEBUGD(&pbr_dbg_map,
+			       "%s: Removing %s(%u) reason: %" PRIu64,
+			       __PRETTY_FUNCTION__, name, seqno, pbrms->reason);
+			pbr_send_pbr_map(pbrm, false);
+			break;
+		}
+	}
+}
+
+extern void pbr_map_install(const char *name)
+{
+	struct pbr_map *pbrm;
+
+	pbrm = pbrm_find(name);
+	if (!pbrm) {
+		DEBUGD(&pbr_dbg_map,
+		       "%s: Specified PBR-MAP(%s) does not exist?",
+		       __PRETTY_FUNCTION__, name);
+		return;
+	}
+
+	if (!pbrm->incoming->count)
+		return;
+
+	pbr_send_pbr_map(pbrm, true);
+	pbrm->installed = true;
+}
+
+extern void pbr_map_add_interfaces(const char *name)
+{
+	struct pbr_map *pbrm;
+	struct interface *ifp;
+	struct pbr_interface *pbr_ifp;
+	struct vrf *vrf;
+
+	pbrm = pbrm_find(name);
+	if (!pbrm) {
+		DEBUGD(&pbr_dbg_map,
+		       "%s: Specified PBR-MAP(%s) does not exist?",
+		       __PRETTY_FUNCTION__, name);
+		return;
+	}
+
+	RB_FOREACH (vrf, vrf_name_head, &vrfs_by_name) {
+                FOR_ALL_INTERFACES (vrf, ifp) {
+			if (ifp->info) {
+				pbr_ifp = ifp->info;
+				if (strcmp(name, pbr_ifp->mapname) == 0)
+					pbr_map_add_interface(pbrm, ifp);
+			}
+		}
+	}
+}
+
+
+extern void pbr_map_check_policy_change(const char *name)
+{
+	struct pbr_map *pbrm;
+
+	pbrm = pbrm_find(name);
+	if (!pbrm) {
+		DEBUGD(&pbr_dbg_map,
+		       "%s: Specified PBR-MAP(%s) does not exist?",
+		       __PRETTY_FUNCTION__, name);
+		return;
+	}
+
+	pbr_map_check_valid(name);
+	if (pbrm->valid && !pbrm->installed) {
+		struct pbr_event *pbre;
+
+		pbre = pbr_event_new(PBR_MAP_INSTALL, name);
+
+		pbr_event_enqueue(pbre);
+	}
+}
+
+extern void pbr_map_init(void)
+{
+	RB_INIT(pbr_map_entry_head, &pbr_maps);
+
+	pbr_map_sequence_unique = 1;
+}
diff --git a/pbrd/pbr_map.h b/pbrd/pbr_map.h
new file mode 100644
index 000000000..fca59951b
--- /dev/null
+++ b/pbrd/pbr_map.h
@@ -0,0 +1,154 @@
+/*
+ * PBR-map Header
+ * Copyright (C) 2018 Cumulus Networks, Inc.
+ *               Donald Sharp
+ *
+ * FRR is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2, or (at your option) any
+ * later version.
+ *
+ * FRR is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; see the file COPYING; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+#ifndef __PBR_MAP_H__
+#define __PBR_MAP_H__
+
+struct pbr_map {
+	/*
+	 * RB Tree of the pbr_maps
+	 */
+	RB_ENTRY(pbr_map) pbr_map_entry;
+
+	/*
+	 * The name of the PBR_MAP
+	 */
+#define PBR_MAP_NAMELEN 100
+	char name[PBR_MAP_NAMELEN];
+
+	struct list *seqnumbers;
+
+	/*
+	 * The list of incoming interfaces that
+	 * we will apply this policy map onto
+	 */
+	struct list *incoming;
+
+	/*
+	 * If valid is true we think the pbr_map is valid,
+	 * If false, look in individual pbrms to see
+	 * what we think is the invalid reason
+	 */
+	bool valid;
+
+	bool installed;
+};
+
+RB_HEAD(pbr_map_entry_head, pbr_map);
+RB_PROTOTYPE(pbr_map_entry_head, pbr_map, pbr_map_entry, pbr_map_compare)
+
+struct pbr_map_interface {
+	struct interface *ifp;
+
+	struct pbr_map *pbrm;
+
+	bool delete;
+};
+
+struct pbr_map_sequence {
+	struct pbr_map *parent;
+
+	/*
+	 * The Unique identifier of this specific pbrms
+	 */
+	uint32_t unique;
+
+	/*
+	 * The sequence of where we are for display
+	 */
+	uint32_t seqno;
+
+	/*
+	 * The rule number to install into
+	 */
+	uint32_t ruleno;
+
+	/*
+	 * Our policy Catchers
+	 */
+	struct prefix *src;
+	struct prefix *dst;
+
+	/*
+	 * The nexthop group we auto create
+	 * for when the user specifies a individual
+	 * nexthop
+	 */
+	struct nexthop_group *nhg;
+	char *internal_nhg_name;
+
+	/*
+	 * The name of the nexthop group
+	 * configured in the pbr-map
+	 */
+	char *nhgrp_name;
+
+	/*
+	 * Do we think are nexthops are installed
+	 */
+	bool nhs_installed;
+
+	bool installed;
+	/*
+	 * A reason of 0 means we think the pbr_map_sequence is good to go
+	 * We can accumuluate multiple failure states
+	 */
+#define PBR_MAP_VALID_SEQUENCE_NUMBER  0
+#define PBR_MAP_INVALID_NEXTHOP_GROUP  (1 << 0)
+#define PBR_MAP_INVALID_NEXTHOP        (1 << 1)
+#define PBR_MAP_INVALID_NO_NEXTHOPS    (1 << 2)
+#define PBR_MAP_INVALID_BOTH_NHANDGRP  (1 << 3)
+#define PBR_MAP_INVALID_SRCDST         (1 << 4)
+#define PBR_MAP_DEL_SEQUENCE_NUMBER    (1 << 5)
+	uint64_t reason;
+
+	QOBJ_FIELDS
+};
+
+DECLARE_QOBJ_TYPE(pbr_map_sequence)
+
+extern struct pbr_map_entry_head pbr_maps;
+
+extern struct pbr_map_sequence *pbrms_get(const char *name, uint32_t seqno);
+extern struct pbr_map_sequence *pbrms_lookup_unique(uint32_t unique,
+						    ifindex_t ifindex);
+
+extern struct pbr_map *pbrm_find(const char *name);
+extern void pbr_map_delete(const char *name, uint32_t seqno);
+extern void pbr_map_add_interface(struct pbr_map *pbrm, struct interface *ifp);
+extern void pbr_map_interface_delete(struct pbr_map *pbrm,
+				     struct interface *ifp);
+extern void pbr_map_write_interfaces(struct vty *vty, struct interface *ifp);
+extern void pbr_map_init(void);
+
+extern bool pbr_map_check_valid(const char *name);
+
+extern void pbr_map_check(const char *name, uint32_t seqno);
+extern void pbr_map_check_nh_group_change(const char *nh_group);
+extern void pbr_map_check_policy_change(const char *name);
+extern void pbr_map_reason_string(unsigned int reason, char *buf, int size);
+extern void pbr_map_add_interfaces(const char *name);
+
+extern void pbr_map_schedule_policy_from_nhg(const char *nh_group);
+
+extern void pbr_map_install(const char *name);
+
+extern void pbr_map_policy_install(const char *name);
+extern void pbr_map_policy_delete(const char *ifname);
+#endif
diff --git a/pbrd/pbr_memory.c b/pbrd/pbr_memory.c
new file mode 100644
index 000000000..febe406ca
--- /dev/null
+++ b/pbrd/pbr_memory.c
@@ -0,0 +1,27 @@
+/*
+ * PBR memory code.
+ * Copyright (C) 2018 Cumulus Networks, Inc.
+ *               Donald Sharp
+ *
+ * FRR is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2, or (at your option) any
+ * later version.
+ *
+ * FRR is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; see the file COPYING; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+#include <zebra.h>
+
+#include <memory.h>
+
+#include "pbrd/pbr_memory.h"
+
+
+DEFINE_MGROUP(PBRD, "pbrd")
diff --git a/pbrd/pbr_memory.h b/pbrd/pbr_memory.h
new file mode 100644
index 000000000..a87d51909
--- /dev/null
+++ b/pbrd/pbr_memory.h
@@ -0,0 +1,24 @@
+/*
+ * pbr memory code.
+ * Copyright (C) 2018 Cumulus Networks, Inc.
+ *               Donald Sharp
+ *
+ * FRR is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2, or (at your option) any
+ * later version.
+ *
+ * FRR is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; see the file COPYING; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+#ifndef __PBR_MEMORY_H__
+
+DECLARE_MGROUP(PBRD)
+
+#endif
diff --git a/pbrd/pbr_nht.c b/pbrd/pbr_nht.c
new file mode 100644
index 000000000..817d8bb5e
--- /dev/null
+++ b/pbrd/pbr_nht.c
@@ -0,0 +1,699 @@
+/*
+ * PBR-nht Code
+ * Copyright (C) 2018 Cumulus Networks, Inc.
+ *               Donald Sharp
+ *
+ * FRR is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2, or (at your option) any
+ * later version.
+ *
+ * FRR is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; see the file COPYING; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+#include <zebra.h>
+
+#include <log.h>
+#include <nexthop.h>
+#include <nexthop_group.h>
+#include <hash.h>
+#include <jhash.h>
+#include <vty.h>
+#include <zclient.h>
+
+#include "pbrd/pbr_nht.h"
+#include "pbrd/pbr_map.h"
+#include "pbrd/pbr_event.h"
+#include "pbrd/pbr_zebra.h"
+#include "pbrd/pbr_memory.h"
+#include "pbrd/pbr_debug.h"
+
+DEFINE_MTYPE_STATIC(PBRD, PBR_NHG, "PBR Nexthop Groups")
+
+static struct hash *pbr_nhg_hash;
+
+static uint32_t pbr_nhg_low_table;
+static uint32_t pbr_nhg_high_table;
+static uint32_t pbr_nhg_low_rule;
+static uint32_t pbr_nhg_high_rule;
+static bool nhg_tableid[65535];
+
+static void *pbr_nh_alloc(void *p)
+{
+	struct pbr_nexthop_cache *new;
+	struct pbr_nexthop_cache *pnhc = (struct pbr_nexthop_cache *)p;
+
+	new = XCALLOC(MTYPE_PBR_NHG, sizeof(*new));
+	memcpy(&new->nexthop, &pnhc->nexthop, sizeof(struct nexthop));
+
+	DEBUGD(&pbr_dbg_nht, "%s: Sending nexthop to Zebra",
+	       __PRETTY_FUNCTION__);
+
+	pbr_send_rnh(&new->nexthop, true);
+
+	new->valid = false;
+	return new;
+}
+
+static void pbr_nh_delete(struct pbr_nexthop_cache **pnhc)
+{
+	pbr_send_rnh(&(*pnhc)->nexthop, false);
+
+	XFREE(MTYPE_PBR_NHG, *pnhc);
+}
+
+static uint32_t pbr_nh_hash_key(void *arg)
+{
+	uint32_t key;
+	struct pbr_nexthop_cache *pbrnc = (struct pbr_nexthop_cache *)arg;
+
+	key = jhash_1word(pbrnc->nexthop.vrf_id, 0x45afe398);
+	key = jhash_1word(pbrnc->nexthop.ifindex, key);
+	key = jhash_1word(pbrnc->nexthop.type, key);
+	key = jhash(&pbrnc->nexthop.gate, sizeof(union g_addr), key);
+
+	return key;
+}
+
+static int pbr_nh_hash_equal(const void *arg1, const void *arg2)
+{
+	const struct pbr_nexthop_cache *pbrnc1 =
+		(const struct pbr_nexthop_cache *)arg1;
+	const struct pbr_nexthop_cache *pbrnc2 =
+		(const struct pbr_nexthop_cache *)arg2;
+
+	if (pbrnc1->nexthop.vrf_id != pbrnc2->nexthop.vrf_id)
+		return 0;
+
+	if (pbrnc1->nexthop.ifindex != pbrnc2->nexthop.ifindex)
+		return 0;
+
+	if (pbrnc1->nexthop.type != pbrnc2->nexthop.type)
+		return 0;
+
+	switch (pbrnc1->nexthop.type) {
+	case NEXTHOP_TYPE_IFINDEX:
+		return 1;
+	case NEXTHOP_TYPE_IPV4_IFINDEX:
+	case NEXTHOP_TYPE_IPV4:
+		return pbrnc1->nexthop.gate.ipv4.s_addr
+		       == pbrnc2->nexthop.gate.ipv4.s_addr;
+	case NEXTHOP_TYPE_IPV6_IFINDEX:
+	case NEXTHOP_TYPE_IPV6:
+		return !memcmp(&pbrnc1->nexthop.gate.ipv6,
+			       &pbrnc2->nexthop.gate.ipv6, 16);
+	case NEXTHOP_TYPE_BLACKHOLE:
+		return pbrnc1->nexthop.bh_type == pbrnc2->nexthop.bh_type;
+	}
+
+	/*
+	 * We should not get here
+	 */
+	return 0;
+}
+
+void pbr_nhgroup_add_cb(const char *name)
+{
+	struct pbr_event *pbre;
+
+	pbre = pbr_event_new(PBR_NHG_NEW, name);
+
+	pbr_event_enqueue(pbre);
+	DEBUGD(&pbr_dbg_nht, "%s: Received ADD cb for %s", __PRETTY_FUNCTION__,
+	       name);
+}
+
+void pbr_nhgroup_add_nexthop_cb(const struct nexthop_group_cmd *nhg,
+				const struct nexthop *nhop)
+{
+	struct pbr_event *pbre;
+
+	pbre = pbr_event_new(PBR_NHG_ADD_NEXTHOP, nhg->name);
+
+	pbr_event_enqueue(pbre);
+	DEBUGD(&pbr_dbg_nht, "%s: Received NEXTHOP_ADD cb for %s",
+	       __PRETTY_FUNCTION__, nhg->name);
+}
+
+void pbr_nhgroup_del_nexthop_cb(const struct nexthop_group_cmd *nhg,
+				const struct nexthop *nhop)
+{
+	struct pbr_event *pbre;
+
+	pbre = pbr_event_new(PBR_NHG_DEL_NEXTHOP, nhg->name);
+
+	pbr_event_enqueue(pbre);
+	DEBUGD(&pbr_dbg_nht, "%s: Received NEXTHOP_DEL cb for %s",
+	       __PRETTY_FUNCTION__, nhg->name);
+}
+
+void pbr_nhgroup_delete_cb(const char *name)
+{
+	struct pbr_event *pbre;
+
+	pbre = pbr_event_new(PBR_NHG_DELETE, name);
+
+	pbr_event_enqueue(pbre);
+	DEBUGD(&pbr_dbg_nht, "%s: Received DELETE cb for %s",
+	       __PRETTY_FUNCTION__, name);
+}
+
+#if 0
+static struct pbr_nexthop_cache *pbr_nht_lookup_nexthop(struct nexthop *nexthop)
+{
+	return NULL;
+}
+#endif
+
+static void pbr_nht_find_nhg_from_table_install(struct hash_backet *b,
+						void *data)
+{
+	struct pbr_nexthop_group_cache *pnhgc =
+		(struct pbr_nexthop_group_cache *)b->data;
+	uint32_t *table_id = (uint32_t *)data;
+
+	if (pnhgc->table_id == *table_id) {
+		DEBUGD(&pbr_dbg_nht, "%s: Table ID (%u) matches %s",
+		       __PRETTY_FUNCTION__, *table_id, pnhgc->name);
+		pnhgc->installed = true;
+		pbr_map_schedule_policy_from_nhg(pnhgc->name);
+	}
+}
+
+void pbr_nht_route_installed_for_table(uint32_t table_id)
+{
+	hash_iterate(pbr_nhg_hash, pbr_nht_find_nhg_from_table_install,
+		     &table_id);
+}
+
+static void pbr_nht_find_nhg_from_table_remove(struct hash_backet *b,
+					       void *data)
+{
+	;
+}
+
+void pbr_nht_route_removed_for_table(uint32_t table_id)
+{
+	hash_iterate(pbr_nhg_hash, pbr_nht_find_nhg_from_table_remove,
+		     &table_id);
+}
+
+/*
+ * Loop through all nexthops in a nexthop group to check that they are all the
+ * same. If they are not all the same, log this peculiarity.
+ *
+ * nhg
+ *    The nexthop group to check
+ *
+ * Returns:
+ *    - AFI of last nexthop in the group
+ *    - AFI_MAX on error
+ */
+static afi_t pbr_nht_which_afi(struct nexthop_group nhg)
+{
+	struct nexthop *nexthop;
+	afi_t install_afi = AFI_MAX;
+	bool v6, v4, bh;
+	v6 = v4 = bh = false;
+
+	for (ALL_NEXTHOPS(nhg, nexthop)) {
+		switch (nexthop->type) {
+		case NEXTHOP_TYPE_IFINDEX:
+			break;
+		case NEXTHOP_TYPE_IPV4:
+		case NEXTHOP_TYPE_IPV4_IFINDEX:
+			v6 = true;
+			install_afi = AFI_IP;
+			break;
+		case NEXTHOP_TYPE_IPV6:
+		case NEXTHOP_TYPE_IPV6_IFINDEX:
+			v4 = true;
+			install_afi = AFI_IP6;
+			break;
+		case NEXTHOP_TYPE_BLACKHOLE:
+			bh = true;
+			install_afi = AFI_MAX;
+			break;
+		}
+	}
+
+	if (!bh && v6 && v4)
+		DEBUGD(&pbr_dbg_nht,
+		       "%s: Saw both V6 and V4 nexthops...using %s",
+		       __PRETTY_FUNCTION__, afi2str(install_afi));
+	if (bh && (v6 || v4))
+		DEBUGD(&pbr_dbg_nht,
+		       "%s: Saw blackhole nexthop(s) with %s%s%s nexthop(s), using AFI_MAX.",
+		       __PRETTY_FUNCTION__, v4 ? "v4" : "",
+		       (v4 && v6) ? " and " : "", v6 ? "v6" : "");
+
+	return install_afi;
+}
+
+static void pbr_nht_install_nexthop_group(struct pbr_nexthop_group_cache *pnhgc,
+					  struct nexthop_group nhg)
+{
+	afi_t install_afi;
+
+	install_afi = pbr_nht_which_afi(nhg);
+
+	pnhgc->installed = false;
+	route_add(pnhgc, nhg, install_afi);
+}
+
+static void
+pbr_nht_uninstall_nexthop_group(struct pbr_nexthop_group_cache *pnhgc,
+				struct nexthop_group nhg)
+{
+	afi_t install_afi;
+
+	install_afi = pbr_nht_which_afi(nhg);
+
+	pnhgc->installed = false;
+	pnhgc->valid = false;
+	route_delete(pnhgc, install_afi);
+}
+
+void pbr_nht_change_group(const char *name)
+{
+	struct nexthop_group_cmd *nhgc;
+	struct pbr_nexthop_group_cache *pnhgc;
+	struct pbr_nexthop_group_cache find;
+	struct nexthop *nhop;
+
+	nhgc = nhgc_find(name);
+	if (!nhgc)
+		return;
+
+	memset(&find, 0, sizeof(find));
+	strcpy(find.name, name);
+	pnhgc = hash_lookup(pbr_nhg_hash, &find);
+
+	if (!pnhgc) {
+		DEBUGD(&pbr_dbg_nht,
+		       "%s: Could not find nexthop-group cache w/ name '%s'",
+		       __PRETTY_FUNCTION__, name);
+		return;
+	}
+
+	for (ALL_NEXTHOPS(nhgc->nhg, nhop)) {
+		struct pbr_nexthop_cache lookup;
+		struct pbr_nexthop_cache *pnhc;
+
+		memcpy(&lookup.nexthop, nhop, sizeof(*nhop));
+		pnhc = hash_lookup(pnhgc->nhh, &lookup);
+		if (!pnhc) {
+			pnhc = hash_get(pnhgc->nhh, &lookup, pbr_nh_alloc);
+			pnhc->parent = pnhgc;
+		}
+	}
+	pbr_nht_install_nexthop_group(pnhgc, nhgc->nhg);
+}
+
+char *pbr_nht_nexthop_make_name(char *name, size_t l,
+				uint32_t seqno, char *buffer)
+{
+	snprintf(buffer, l, "%s%u", name, seqno);
+	return buffer;
+}
+
+static void *pbr_nhgc_alloc(void *p)
+{
+	struct pbr_nexthop_group_cache *new;
+	struct pbr_nexthop_group_cache *pnhgc =
+		(struct pbr_nexthop_group_cache *)p;
+
+	new = XCALLOC(MTYPE_PBR_NHG, sizeof(*new));
+
+	strcpy(new->name, pnhgc->name);
+	new->table_id = pbr_nht_get_next_tableid();
+
+	DEBUGD(&pbr_dbg_nht, "%s: NHT: %s assigned Table ID: %u",
+	       __PRETTY_FUNCTION__, new->name, new->table_id);
+
+	new->nhh = hash_create_size(8, pbr_nh_hash_key, pbr_nh_hash_equal,
+				    "PBR NH Cache Hash");
+	return new;
+}
+
+void pbr_nht_add_individual_nexthop(const char *name, uint32_t seqno)
+{
+	struct pbr_nexthop_group_cache *pnhgc;
+	struct pbr_nexthop_group_cache find;
+	struct pbr_nexthop_cache *pnhc;
+	struct pbr_map_sequence *pbrms;
+	struct pbr_nexthop_cache lookup;
+
+	pbrms = pbrms_get(name, seqno);
+
+	memset(&find, 0, sizeof(find));
+	pbr_nht_nexthop_make_name(pbrms->parent->name, PBR_MAP_NAMELEN,
+				  pbrms->seqno, find.name);
+	if (!pbrms->internal_nhg_name)
+		pbrms->internal_nhg_name = XSTRDUP(MTYPE_TMP, find.name);
+
+	pnhgc = hash_get(pbr_nhg_hash, &find, pbr_nhgc_alloc);
+
+	memcpy(&lookup.nexthop, pbrms->nhg->nexthop, sizeof(struct nexthop));
+	pnhc = hash_get(pnhgc->nhh, &lookup, pbr_nh_alloc);
+	pnhc->parent = pnhgc;
+	pbr_nht_install_nexthop_group(pnhgc, *pbrms->nhg);
+}
+
+void pbr_nht_delete_individual_nexthop(const char *name, uint32_t seqno)
+{
+	struct pbr_nexthop_group_cache *pnhgc;
+	struct pbr_nexthop_group_cache find;
+	struct pbr_nexthop_cache *pnhc;
+	struct pbr_nexthop_cache lup;
+	struct pbr_map_sequence *pbrms;
+	struct nexthop *nh;
+
+	pbrms = pbrms_get(name, seqno);
+
+	memset(&find, 0, sizeof(find));
+	strcpy(&find.name[0], pbrms->internal_nhg_name);
+	pnhgc = hash_lookup(pbr_nhg_hash, &find);
+
+	nh = pbrms->nhg->nexthop;
+	memcpy(&lup.nexthop, nh, sizeof(struct nexthop));
+	pnhc = hash_lookup(pnhgc->nhh, &lup);
+	pnhc->parent = NULL;
+	hash_release(pnhgc->nhh, pnhc);
+	pbr_nh_delete(&pnhc);
+	pbr_nht_uninstall_nexthop_group(pnhgc, *pbrms->nhg);
+
+	hash_release(pbr_nhg_hash, pnhgc);
+
+	nexthop_del(pbrms->nhg, nh);
+	nexthop_free(nh);
+	nexthop_group_delete(&pbrms->nhg);
+	XFREE(MTYPE_TMP, pbrms->internal_nhg_name);
+}
+
+void pbr_nht_add_group(const char *name)
+{
+	struct nexthop *nhop;
+	struct nexthop_group_cmd *nhgc;
+	struct pbr_nexthop_group_cache *pnhgc;
+	struct pbr_nexthop_group_cache lookup;
+
+	nhgc = nhgc_find(name);
+
+	if (!nhgc) {
+		zlog_warn("%s: Could not find group %s to add",
+			  __PRETTY_FUNCTION__, name);
+		return;
+	}
+
+	strcpy(lookup.name, name);
+	pnhgc = hash_get(pbr_nhg_hash, &lookup, pbr_nhgc_alloc);
+	DEBUGD(&pbr_dbg_nht, "%s: Retrieved NHGC @ %p", __PRETTY_FUNCTION__,
+	       pnhgc);
+
+	for (ALL_NEXTHOPS(nhgc->nhg, nhop)) {
+		struct pbr_nexthop_cache lookup;
+		struct pbr_nexthop_cache *pnhc;
+
+		memcpy(&lookup.nexthop, nhop, sizeof(*nhop));
+		pnhc = hash_lookup(pnhgc->nhh, &lookup);
+		if (!pnhc) {
+			pnhc = hash_get(pnhgc->nhh, &lookup, pbr_nh_alloc);
+			pnhc->parent = pnhgc;
+		}
+	}
+}
+
+void pbr_nht_delete_group(const char *name)
+{
+	struct pbr_map_sequence *pbrms;
+	struct listnode *snode;
+	struct pbr_map *pbrm;
+
+	RB_FOREACH (pbrm, pbr_map_entry_head, &pbr_maps) {
+		for (ALL_LIST_ELEMENTS_RO(pbrm->seqnumbers, snode, pbrms)) {
+			if (pbrms->nhgrp_name
+			    && strcmp(pbrms->nhgrp_name, name) == 0) {
+				pbrms->reason |= PBR_MAP_INVALID_NO_NEXTHOPS;
+				pbrm->valid = false;
+			}
+		}
+	}
+}
+
+bool pbr_nht_nexthop_valid(struct nexthop_group *nhg)
+{
+	DEBUGD(&pbr_dbg_nht, "%s: %p", __PRETTY_FUNCTION__, nhg);
+	return true;
+}
+
+bool pbr_nht_nexthop_group_valid(const char *name)
+{
+	struct pbr_nexthop_group_cache *pnhgc;
+	struct pbr_nexthop_group_cache lookup;
+
+	DEBUGD(&pbr_dbg_nht, "%s: %s", __PRETTY_FUNCTION__, name);
+
+	strcpy(lookup.name, name);
+	pnhgc = hash_get(pbr_nhg_hash, &lookup, NULL);
+	if (!pnhgc)
+		return false;
+	DEBUGD(&pbr_dbg_nht, "%s: \t%d %d", __PRETTY_FUNCTION__, pnhgc->valid,
+	       pnhgc->installed);
+	if (pnhgc->valid && pnhgc->installed)
+		return true;
+
+	return false;
+}
+
+struct pbr_nht_individual {
+	struct zapi_route *nhr;
+
+	uint32_t valid;
+};
+
+static void pbr_nht_individual_nexthop_update_lookup(struct hash_backet *b,
+						     void *data)
+{
+	struct pbr_nexthop_cache *pnhc = b->data;
+	struct pbr_nht_individual *pnhi = data;
+	char buf[PREFIX_STRLEN];
+	bool old_valid;
+
+	old_valid = pnhc->valid;
+
+	switch (pnhi->nhr->prefix.family) {
+	case AF_INET:
+		if (pnhc->nexthop.gate.ipv4.s_addr
+		    == pnhi->nhr->prefix.u.prefix4.s_addr)
+			pnhc->valid = !!pnhi->nhr->nexthop_num;
+		break;
+	case AF_INET6:
+		if (memcmp(&pnhc->nexthop.gate.ipv6,
+			   &pnhi->nhr->prefix.u.prefix6, 16) == 0)
+			pnhc->valid = !!pnhi->nhr->nexthop_num;
+		break;
+	}
+
+	DEBUGD(&pbr_dbg_nht, "\tFound %s: old: %d new: %d",
+	       prefix2str(&pnhi->nhr->prefix, buf, sizeof(buf)), old_valid,
+	       pnhc->valid);
+
+	if (old_valid != pnhc->valid) {
+		struct pbr_event *pbre;
+
+		pbre = pbr_event_new(PBR_NH_CHANGED, pnhc->parent->name);
+
+		pbr_event_enqueue(pbre);
+	}
+
+	if (pnhc->valid)
+		pnhi->valid += 1;
+}
+
+static void pbr_nht_nexthop_update_lookup(struct hash_backet *b, void *data)
+{
+	struct pbr_nexthop_group_cache *pnhgc = b->data;
+	struct pbr_nht_individual pnhi;
+
+	pnhi.nhr = (struct zapi_route *)data;
+	pnhi.valid = 0;
+	hash_iterate(pnhgc->nhh, pbr_nht_individual_nexthop_update_lookup,
+		     &pnhi);
+
+	/*
+	 * If any of the specified nexthops are valid we are valid
+	 */
+	pnhgc->valid = !!pnhi.valid;
+}
+
+void pbr_nht_nexthop_update(struct zapi_route *nhr)
+{
+	hash_iterate(pbr_nhg_hash, pbr_nht_nexthop_update_lookup, nhr);
+}
+
+static uint32_t pbr_nhg_hash_key(void *arg)
+{
+	struct pbr_nexthop_group_cache *nhgc =
+		(struct pbr_nexthop_group_cache *)arg;
+
+	return jhash(&nhgc->name, strlen(nhgc->name), 0x52c34a96);
+}
+
+static int pbr_nhg_hash_equal(const void *arg1, const void *arg2)
+{
+	const struct pbr_nexthop_group_cache *nhgc1 =
+		(const struct pbr_nexthop_group_cache *)arg1;
+	const struct pbr_nexthop_group_cache *nhgc2 =
+		(const struct pbr_nexthop_group_cache *)arg2;
+
+	return !strcmp(nhgc1->name, nhgc2->name);
+}
+
+
+uint32_t pbr_nht_get_next_tableid(void)
+{
+	uint32_t i;
+	bool found = false;
+
+	for (i = pbr_nhg_low_table; i <= pbr_nhg_high_table; i++) {
+		if (nhg_tableid[i] == false) {
+			found = true;
+			break;
+		}
+	}
+
+	if (found) {
+		nhg_tableid[i] = true;
+		return i;
+	} else
+		return 0;
+}
+
+void pbr_nht_set_tableid_range(uint32_t low, uint32_t high)
+{
+	pbr_nhg_low_table = low;
+	pbr_nhg_high_table = high;
+}
+
+void pbr_nht_write_table_range(struct vty *vty)
+{
+	if (pbr_nhg_low_table != PBR_NHT_DEFAULT_LOW_TABLEID
+	    || pbr_nhg_high_table != PBR_NHT_DEFAULT_HIGH_TABLEID) {
+		vty_out(vty, "pbr table range %u %u\n", pbr_nhg_low_table,
+			pbr_nhg_high_table);
+	}
+}
+
+uint32_t pbr_nht_get_next_rule(uint32_t seqno)
+{
+	return seqno + pbr_nhg_low_rule - 1;
+}
+void pbr_nht_set_rule_range(uint32_t low, uint32_t high)
+{
+	pbr_nhg_low_rule = low;
+	pbr_nhg_high_rule = high;
+}
+
+void pbr_nht_write_rule_range(struct vty *vty)
+{
+	if (pbr_nhg_low_rule != PBR_NHT_DEFAULT_LOW_RULE
+	    || pbr_nhg_high_rule != PBR_NHT_DEFAULT_HIGH_RULE) {
+		vty_out(vty, "pbr rule range %u %u\n", pbr_nhg_low_rule,
+			pbr_nhg_high_rule);
+	}
+}
+
+uint32_t pbr_nht_get_table(const char *name)
+{
+	struct pbr_nexthop_group_cache find;
+	struct pbr_nexthop_group_cache *pnhgc;
+
+	memset(&find, 0, sizeof(find));
+	strcpy(find.name, name);
+	pnhgc = hash_lookup(pbr_nhg_hash, &find);
+
+	if (!pnhgc) {
+		DEBUGD(&pbr_dbg_nht,
+		       "%s: Could not find nexthop-group cache w/ name '%s'",
+		       __PRETTY_FUNCTION__, name);
+		return 5000;
+	}
+
+	return pnhgc->table_id;
+}
+
+bool pbr_nht_get_installed(const char *name)
+{
+	struct pbr_nexthop_group_cache find;
+	struct pbr_nexthop_group_cache *pnhgc;
+
+	memset(&find, 0, sizeof(find));
+	strcpy(find.name, name);
+
+	pnhgc = hash_lookup(pbr_nhg_hash, &find);
+
+	if (!pnhgc) {
+		return false;
+	}
+
+	return pnhgc->installed;
+}
+
+static void pbr_nht_show_nhg_nexthops(struct hash_backet *b, void *data)
+{
+	struct pbr_nexthop_cache *pnhc = b->data;
+	struct vty *vty = data;
+
+	vty_out(vty, "\tValid: %d", pnhc->valid);
+	nexthop_group_write_nexthop(vty, &pnhc->nexthop);
+}
+
+struct pbr_nht_show {
+	struct vty *vty;
+	const char *name;
+};
+
+static void pbr_nht_show_nhg(struct hash_backet *b, void *data)
+{
+	struct pbr_nexthop_group_cache *pnhgc = b->data;
+	struct pbr_nht_show *pns = data;
+	struct vty *vty;
+
+	if (pns->name && strcmp(pns->name, pnhgc->name) != 0)
+		return;
+
+	vty = pns->vty;
+	vty_out(vty, "Nexthop-Group: %s Table: %u Valid: %d Installed: %d\n",
+		pnhgc->name, pnhgc->table_id, pnhgc->valid, pnhgc->installed);
+
+	hash_iterate(pnhgc->nhh, pbr_nht_show_nhg_nexthops, vty);
+}
+
+void pbr_nht_show_nexthop_group(struct vty *vty, const char *name)
+{
+	struct pbr_nht_show pns;
+
+	pns.vty = vty;
+	pns.name = name;
+
+	hash_iterate(pbr_nhg_hash, pbr_nht_show_nhg, &pns);
+}
+
+void pbr_nht_init(void)
+{
+	pbr_nhg_hash = hash_create_size(
+		16, pbr_nhg_hash_key, pbr_nhg_hash_equal, "PBR NHG Cache Hash");
+
+	pbr_nhg_low_table = PBR_NHT_DEFAULT_LOW_TABLEID;
+	pbr_nhg_high_table = PBR_NHT_DEFAULT_HIGH_TABLEID;
+	pbr_nhg_low_rule = PBR_NHT_DEFAULT_LOW_RULE;
+	pbr_nhg_high_rule = PBR_NHT_DEFAULT_HIGH_RULE;
+	memset(&nhg_tableid, 0, 65535 * sizeof(uint8_t));
+}
diff --git a/pbrd/pbr_nht.h b/pbrd/pbr_nht.h
new file mode 100644
index 000000000..2508f49e4
--- /dev/null
+++ b/pbrd/pbr_nht.h
@@ -0,0 +1,113 @@
+/*
+ * PBR-nht Header
+ * Copyright (C) 2018 Cumulus Networks, Inc.
+ *               Donald Sharp
+ *
+ * FRR is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2, or (at your option) any
+ * later version.
+ *
+ * FRR is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; see the file COPYING; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+#ifndef __PBR_NHT_H__
+#define __PBR_NHT_H__
+
+#include <lib/zclient.h>
+#include <lib/nexthop_group.h>
+
+struct pbr_nexthop_group_cache {
+	char name[PBR_MAP_NAMELEN];
+
+	uint32_t table_id;
+
+	struct hash *nhh;
+
+	/*
+	 * If all nexthops are considered valid
+	 */
+	bool valid;
+
+	bool installed;
+};
+
+struct pbr_nexthop_cache {
+	struct pbr_nexthop_group_cache *parent;
+
+	struct nexthop nexthop;
+
+	bool valid;
+};
+
+extern void pbr_nht_write_table_range(struct vty *vty);
+#define PBR_NHT_DEFAULT_LOW_TABLEID 10000
+#define PBR_NHT_DEFAULT_HIGH_TABLEID 11000
+extern void pbr_nht_set_tableid_range(uint32_t low, uint32_t high);
+
+/*
+ * Get the next tableid to use for installation
+ */
+extern uint32_t pbr_nht_get_next_tableid(void);
+/*
+ * Get the next rule number to use for installation
+ */
+extern void pbr_nht_write_rule_range(struct vty *vty);
+
+#define PBR_NHT_DEFAULT_LOW_RULE 300
+#define PBR_NHT_DEFAULT_HIGH_RULE 1300
+extern void pbr_nht_set_rule_range(uint32_t low, uint32_t high);
+
+extern uint32_t pbr_nht_get_next_rule(uint32_t seqno);
+
+extern void pbr_nhgroup_add_cb(const char *name);
+extern void pbr_nhgroup_add_nexthop_cb(const struct nexthop_group_cmd *nhg,
+				       const struct nexthop *nhop);
+extern void pbr_nhgroup_del_nexthop_cb(const struct nexthop_group_cmd *nhg,
+				       const struct nexthop *nhop);
+extern void pbr_nhgroup_delete_cb(const char *name);
+
+extern bool pbr_nht_nexthop_valid(struct nexthop_group *nhg);
+extern bool pbr_nht_nexthop_group_valid(const char *name);
+
+extern void pbr_nht_add_group(const char *name);
+extern void pbr_nht_change_group(const char *name);
+extern void pbr_nht_delete_group(const char *name);
+
+extern void pbr_nht_add_individual_nexthop(const char *name, uint32_t seqno);
+extern void pbr_nht_delete_individual_nexthop(const char *name, uint32_t seqno);
+/*
+ * Given the tableid of the installed default
+ * route, find the nexthop-group associated with
+ * it, then find all pbr-maps that use it and
+ * install/delete them as well.
+ */
+extern void pbr_nht_route_installed_for_table(uint32_t table_id);
+extern void pbr_nht_route_removed_for_table(uint32_t table_id);
+
+/*
+ * Given the nexthop group name, lookup the associated
+ * tableid with it
+ */
+extern uint32_t pbr_nht_get_table(const char *name);
+
+extern bool pbr_nht_get_installed(const char *name);
+
+extern char *pbr_nht_nexthop_make_name(char *name, size_t l, uint32_t seqno,
+				       char *buffer);
+
+extern void pbr_nht_show_nexthop_group(struct vty *vty, const char *name);
+
+/*
+ * When we get a callback from zebra about a nexthop changing
+ */
+extern void pbr_nht_nexthop_update(struct zapi_route *nhr);
+
+extern void pbr_nht_init(void);
+#endif
diff --git a/pbrd/pbr_vty.c b/pbrd/pbr_vty.c
new file mode 100644
index 000000000..730cfd40f
--- /dev/null
+++ b/pbrd/pbr_vty.c
@@ -0,0 +1,631 @@
+/*
+ * PBR - vty code
+ * Copyright (C) 2018 Cumulus Networks, Inc.
+ *               Donald Sharp
+ *
+ * FRR is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2, or (at your option) any
+ * later version.
+ *
+ * FRR is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; see the file COPYING; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+#include <zebra.h>
+
+#include "vty.h"
+#include "command.h"
+#include "prefix.h"
+#include "vrf.h"
+#include "nexthop.h"
+#include "nexthop_group.h"
+#include "log.h"
+#include "json.h"
+#include "debug.h"
+
+#include "pbrd/pbr_nht.h"
+#include "pbrd/pbr_map.h"
+#include "pbrd/pbr_zebra.h"
+#include "pbrd/pbr_vty.h"
+#include "pbrd/pbr_event.h"
+#include "pbrd/pbr_debug.h"
+#ifndef VTYSH_EXTRACT_PL
+#include "pbrd/pbr_vty_clippy.c"
+#endif
+
+DEFUN_NOSH(pbr_map, pbr_map_cmd, "pbr-map WORD seq (1-1000)",
+	   "Create pbr-map or enter pbr-map command mode\n"
+	   "The name of the PBR MAP\n"
+	   "Sequence to insert in existing pbr-map entry\n"
+	   "Sequence number\n")
+{
+	const char *pbrm_name = argv[1]->arg;
+	uint32_t seqno = atoi(argv[3]->arg);
+	struct pbr_map_sequence *pbrms;
+
+	pbrms = pbrms_get(pbrm_name, seqno);
+	VTY_PUSH_CONTEXT(PBRMAP_NODE, pbrms);
+
+	return CMD_SUCCESS;
+}
+
+DEFUN_NOSH(no_pbr_map, no_pbr_map_cmd, "no pbr-map WORD [seq (1-65535)]",
+	   NO_STR
+	   "Delete pbr-map\n"
+	   "The name of the PBR MAP\n"
+	   "Sequence to delete from existing pbr-map entry\n"
+	   "Sequence number\n")
+{
+	const char *pbrm_name = argv[2]->arg;
+	uint32_t seqno = 0;
+	struct pbr_map *pbrm = pbrm_find(pbrm_name);
+	struct pbr_event *pbre;
+	struct pbr_map_sequence *pbrms;
+	struct listnode *node, *next_node;
+
+	if (argc > 3)
+		seqno = atoi(argv[4]->arg);
+
+	if (!pbrm) {
+		vty_out(vty, "pbr-map %s not found\n", pbrm_name);
+		return CMD_SUCCESS;
+	}
+
+	if (seqno) {
+		pbrms = pbrms_get(pbrm->name, seqno);
+		pbrms->reason |= PBR_MAP_DEL_SEQUENCE_NUMBER;
+	} else {
+		for (ALL_LIST_ELEMENTS(pbrm->seqnumbers, node, next_node,
+				       pbrms)) {
+			if (pbrms)
+				pbrms->reason |= PBR_MAP_DEL_SEQUENCE_NUMBER;
+		}
+	}
+
+	pbre = pbr_event_new(PBR_MAP_DELETE, pbrm_name);
+	pbre->seqno = seqno;
+	pbr_event_enqueue(pbre);
+
+	return CMD_SUCCESS;
+}
+
+DEFPY(pbr_map_match_src, pbr_map_match_src_cmd,
+	"[no] match src-ip <A.B.C.D/M|X:X::X:X/M>$prefix",
+	NO_STR
+	"Match the rest of the command\n"
+	"Choose the src ip or ipv6 prefix to use\n"
+	"v4 Prefix\n"
+	"v6 Prefix\n")
+{
+	struct pbr_map_sequence *pbrms = VTY_GET_CONTEXT(pbr_map_sequence);
+	struct pbr_event *pbre;
+
+	if (!no) {
+		if (!pbrms->src)
+			pbrms->src = prefix_new();
+		prefix_copy(pbrms->src, prefix);
+	} else {
+		prefix_free(pbrms->src);
+		pbrms->src = 0;
+	}
+
+	pbre = pbr_event_new(PBR_MAP_MODIFY, pbrms->parent->name);
+	pbre->seqno = pbrms->seqno;
+	pbr_event_enqueue(pbre);
+
+	return CMD_SUCCESS;
+}
+
+DEFPY(pbr_map_match_dst, pbr_map_match_dst_cmd,
+	"[no] match dst-ip <A.B.C.D/M|X:X::X:X/M>$prefix",
+	NO_STR
+	"Match the rest of the command\n"
+	"Choose the src ip or ipv6 prefix to use\n"
+	"v4 Prefix\n"
+	"v6 Prefix\n")
+{
+	struct pbr_map_sequence *pbrms = VTY_GET_CONTEXT(pbr_map_sequence);
+	struct pbr_event *pbre;
+
+	if (!no) {
+		if (!pbrms->dst)
+			pbrms->dst = prefix_new();
+		prefix_copy(pbrms->dst, prefix);
+	} else {
+		prefix_free(pbrms->dst);
+		pbrms->dst = 0;
+	}
+
+	pbre = pbr_event_new(PBR_MAP_MODIFY, pbrms->parent->name);
+	pbre->seqno = pbrms->seqno;
+	pbr_event_enqueue(pbre);
+
+	return CMD_SUCCESS;
+}
+
+DEFPY(pbr_map_nexthop_group, pbr_map_nexthop_group_cmd,
+	"[no] set nexthop-group NAME$name",
+	NO_STR
+	"Set for the PBR-MAP\n"
+	"nexthop-group to use\n"
+	"The name of the nexthop-group\n")
+{
+	struct pbr_map_sequence *pbrms = VTY_GET_CONTEXT(pbr_map_sequence);
+	struct nexthop_group_cmd *nhgc;
+	struct pbr_event *pbre;
+
+	nhgc = nhgc_find(name);
+	if (!nhgc) {
+		vty_out(vty, "Specified nexthop-group %s does not exist\n",
+			name);
+		vty_out(vty, "PBR-MAP will not be applied until it is created\n");
+	}
+
+	if (no) {
+		if (pbrms->nhgrp_name && strcmp(name, pbrms->nhgrp_name) == 0)
+			pbre = pbr_event_new(PBR_MAP_NHG_DELETE,
+					     pbrms->parent->name);
+		else {
+			vty_out(vty,
+				"Nexthop Group specified: %s does not exist to remove",
+				name);
+			return CMD_WARNING;
+		}
+	} else {
+		if (pbrms->nhgrp_name) {
+			if (strcmp(name, pbrms->nhgrp_name) != 0) {
+				vty_out(vty,
+					"Please delete current nexthop group before modifying current one");
+				return CMD_WARNING;
+			}
+
+			return CMD_SUCCESS;
+		}
+		pbrms->nhgrp_name = XSTRDUP(MTYPE_TMP, name);
+		pbre = pbr_event_new(PBR_MAP_NHG_ADD, pbrms->parent->name);
+	}
+
+	pbre->seqno = pbrms->seqno;
+	pbr_event_enqueue(pbre);
+
+	return CMD_SUCCESS;
+}
+
+DEFPY(pbr_map_nexthop, pbr_map_nexthop_cmd,
+      "[no] set nexthop <A.B.C.D|X:X::X:X>$addr [INTERFACE]$intf [nexthop-vrf NAME$name]",
+      NO_STR
+      "Set for the PBR-MAP\n"
+      "Specify one of the nexthops in this map\n"
+      "v4 Address\n"
+      "v6 Address\n"
+      "Interface to use\n"
+      "If the nexthop is in a different vrf tell us\n"
+      "The nexthop-vrf Name\n")
+{
+	struct pbr_map_sequence *pbrms = VTY_GET_CONTEXT(pbr_map_sequence);
+	struct vrf *vrf;
+	struct nexthop nhop;
+	struct nexthop *nh;
+	struct pbr_event *pbre;
+
+	if (pbrms->nhgrp_name) {
+		vty_out(vty,
+			"Please unconfigure the nexthop group before adding an individual nexthop");
+		return CMD_WARNING;
+	}
+
+	if (name)
+		vrf = vrf_lookup_by_name(name);
+	else
+		vrf = vrf_lookup_by_id(VRF_DEFAULT);
+
+	if (!vrf) {
+		vty_out(vty, "Specified: %s is non-existent\n", name);
+		return CMD_WARNING;
+	}
+
+	memset(&nhop, 0, sizeof(nhop));
+	nhop.vrf_id = vrf->vrf_id;
+
+	if (addr->sa.sa_family == AF_INET) {
+		nhop.gate.ipv4.s_addr = addr->sin.sin_addr.s_addr;
+		if (intf) {
+			nhop.type = NEXTHOP_TYPE_IPV4_IFINDEX;
+			nhop.ifindex = ifname2ifindex(intf, vrf->vrf_id);
+			if (nhop.ifindex == IFINDEX_INTERNAL) {
+				vty_out(vty,
+					"Specified Intf %s does not exist in vrf: %s\n",
+					intf, vrf->name);
+				return CMD_WARNING;
+			}
+		} else
+			nhop.type = NEXTHOP_TYPE_IPV4;
+	} else {
+		memcpy(&nhop.gate.ipv6, &addr->sin6.sin6_addr, 16);
+		if (intf) {
+			nhop.type = NEXTHOP_TYPE_IPV6_IFINDEX;
+			nhop.ifindex = ifname2ifindex(intf, vrf->vrf_id);
+			if (nhop.ifindex == IFINDEX_INTERNAL) {
+				vty_out(vty,
+					"Specified Intf %s does not exist in vrf: %s\n",
+					intf, vrf->name);
+				return CMD_WARNING;
+			}
+		} else
+			nhop.type = NEXTHOP_TYPE_IPV6;
+	}
+
+	if (pbrms->nhg)
+		nh = nexthop_exists(pbrms->nhg, &nhop);
+	else {
+		char buf[100];
+
+		if (no) {
+			vty_out(vty, "No nexthops to delete");
+			return CMD_WARNING;
+		}
+
+		pbrms->nhg = nexthop_group_new();
+		pbrms->internal_nhg_name =
+			XSTRDUP(MTYPE_TMP,
+				pbr_nht_nexthop_make_name(pbrms->parent->name,
+							  PBR_MAP_NAMELEN,
+							  pbrms->seqno,
+							  buf));
+		nh = NULL;
+	}
+
+	if (no) {
+		if (nh) {
+			// nexthop_del(pbrms->nhg, nh);
+			// nexthop_free(nh);
+			pbre = pbr_event_new(PBR_MAP_NEXTHOP_DELETE,
+					     pbrms->parent->name);
+			pbre->seqno = pbrms->seqno;
+			pbr_event_enqueue(pbre);
+		}
+	} else if (!nh) {
+
+		if (pbrms->nhg->nexthop) {
+			vty_out(vty,
+				"If you would like more than one nexthop please use nexthop-groups");
+			return CMD_WARNING;
+		}
+
+		/* must be adding new nexthop since !no and !nexthop_exists */
+		nh = nexthop_new();
+
+		memcpy(nh, &nhop, sizeof(nhop));
+		nexthop_add(&pbrms->nhg->nexthop, nh);
+
+		pbre = pbr_event_new(PBR_MAP_NEXTHOP_ADD, pbrms->parent->name);
+		pbre->seqno = pbrms->seqno;
+		pbr_event_enqueue(pbre);
+	}
+
+	return CMD_SUCCESS;
+}
+
+DEFPY (pbr_table_range,
+       pbr_table_range_cmd,
+       "[no]$no pbr table range (10000-65535)$start (11000-65535)$end",
+       NO_STR
+       "Policy based routing\n"
+       "Policy based routing table\n"
+       "Table range\n"
+       "Initial value of range\n"
+       "Final value of range\n")
+{
+	if (no)
+		pbr_nht_set_tableid_range(PBR_NHT_DEFAULT_LOW_TABLEID,
+					  PBR_NHT_DEFAULT_HIGH_TABLEID);
+	else
+		pbr_nht_set_tableid_range(start, end);
+
+	return CMD_SUCCESS;
+}
+
+DEFPY (pbr_rule_range,
+	pbr_rule_range_cmd,
+	"[no] pbr rule range (300-1300)$start (400-1400)$end",
+	NO_STR
+	"Policy based routing\n"
+	"Policy based routing rule\n"
+	"Rule range\n"
+	"Initial value of range\n"
+	"Final value of range\n")
+{
+	if (no)
+		pbr_nht_set_rule_range(PBR_NHT_DEFAULT_LOW_RULE,
+				       PBR_NHT_DEFAULT_HIGH_RULE);
+	else
+		pbr_nht_set_rule_range(start, end);
+
+	return CMD_SUCCESS;
+}
+
+DEFPY (pbr_policy,
+	pbr_policy_cmd,
+	"[no] pbr-policy NAME$mapname",
+	NO_STR
+	"Policy to use\n"
+	"Name of the pbr-map to apply\n")
+{
+	VTY_DECLVAR_CONTEXT(interface, ifp);
+	struct pbr_map *pbrm, *old_pbrm;
+	struct pbr_interface *pbr_ifp = ifp->info;
+
+	pbrm = pbrm_find(mapname);
+
+	if (no) {
+		if (strcmp(pbr_ifp->mapname, mapname) == 0) {
+			strcpy(pbr_ifp->mapname, "");
+
+			if (pbrm)
+				pbr_map_interface_delete(pbrm, ifp);
+		}
+	} else {
+		if (strcmp(pbr_ifp->mapname, "") == 0) {
+			strcpy(pbr_ifp->mapname, mapname);
+
+			if (pbrm)
+				pbr_map_add_interface(pbrm, ifp);
+		} else {
+			if (!(strcmp(pbr_ifp->mapname, mapname) == 0)) {
+				old_pbrm = pbrm_find(pbr_ifp->mapname);
+				if (old_pbrm)
+					pbr_map_interface_delete(old_pbrm, ifp);
+				strcpy(pbr_ifp->mapname, mapname);
+				if (pbrm)
+					pbr_map_add_interface(pbrm, ifp);
+			}
+		}
+	}
+
+	return CMD_SUCCESS;
+}
+
+DEFPY (show_pbr,
+	show_pbr_cmd,
+	"show pbr [json$json]",
+	SHOW_STR
+	"Policy Based Routing\n"
+	JSON_STR)
+{
+	pbr_nht_write_table_range(vty);
+	pbr_nht_write_rule_range(vty);
+
+	return CMD_SUCCESS;
+}
+
+DEFPY (show_pbr_map,
+	show_pbr_map_cmd,
+	"show pbr map [NAME$name] [detail$detail] [json$json]",
+	SHOW_STR
+	"Policy Based Routing\n"
+	"PBR Map\n"
+	"PBR Map Name\n"
+	"Detailed information\n"
+	JSON_STR)
+{
+	struct pbr_map_sequence *pbrms;
+	struct pbr_map *pbrm;
+	struct listnode *node;
+	char buf[PREFIX_STRLEN];
+	char rbuf[64];
+
+	RB_FOREACH (pbrm, pbr_map_entry_head, &pbr_maps) {
+		if (name && strcmp(name, pbrm->name) != 0)
+			continue;
+
+		vty_out(vty, "  pbr-map %s valid: %d\n", pbrm->name,
+			pbrm->valid);
+
+		for (ALL_LIST_ELEMENTS_RO(pbrm->seqnumbers, node, pbrms)) {
+			if (pbrms->reason)
+				pbr_map_reason_string(pbrms->reason, rbuf,
+						      sizeof(rbuf));
+			vty_out(vty,
+				"    Seq: %u rule: %u Installed: %d(%u) Reason: %s\n",
+				pbrms->seqno, pbrms->ruleno, pbrms->installed,
+				pbrms->unique, pbrms->reason ? rbuf : "Valid");
+
+			if (pbrms->src)
+				vty_out(vty, "\tSRC Match: %s\n",
+					prefix2str(pbrms->src, buf,
+						   sizeof(buf)));
+			if (pbrms->dst)
+				vty_out(vty, "\tDST Match: %s\n",
+					prefix2str(pbrms->dst, buf,
+						   sizeof(buf)));
+
+			if (pbrms->nhgrp_name) {
+				vty_out(vty,
+					"\tNexthop-Group: %s(%u) Installed: %u(%d)\n",
+					pbrms->nhgrp_name,
+					pbr_nht_get_table(pbrms->nhgrp_name),
+					pbrms->nhs_installed,
+					pbr_nht_get_installed(
+						pbrms->nhgrp_name));
+			} else if (pbrms->nhg) {
+				vty_out(vty, "     ");
+				nexthop_group_write_nexthop(
+					vty, pbrms->nhg->nexthop);
+				vty_out(vty,
+					"\tInstalled: %u(%d) Tableid: %d\n",
+					pbrms->nhs_installed,
+					pbr_nht_get_installed(
+						pbrms->internal_nhg_name),
+					pbr_nht_get_table(
+						pbrms->internal_nhg_name));
+			} else {
+				vty_out(vty,
+					"\tNexthop-Group: Unknown Installed: 0(0)\n");
+			}
+		}
+	}
+	return CMD_SUCCESS;
+}
+
+DEFPY(show_pbr_nexthop_group,
+      show_pbr_nexthop_group_cmd,
+      "show pbr nexthop-groups [WORD$word]",
+      SHOW_STR
+      "Policy Based Routing\n"
+      "Nexthop Groups\n"
+      "Optional Name of the nexthop group\n")
+{
+	pbr_nht_show_nexthop_group(vty, word);
+
+	return CMD_SUCCESS;
+}
+
+DEFPY (show_pbr_interface,
+	show_pbr_interface_cmd,
+	"show pbr interface [NAME$name] [json$json]",
+	SHOW_STR
+	"Policy Based Routing\n"
+	"PBR Interface\n"
+	"PBR Interface Name\n"
+	JSON_STR)
+{
+        struct interface *ifp;
+	struct vrf *vrf;
+	struct pbr_interface *pbr_ifp;
+
+	RB_FOREACH(vrf, vrf_name_head, &vrfs_by_name) {
+		FOR_ALL_INTERFACES(vrf, ifp) {
+			struct pbr_map *pbrm;
+
+			if (name && strcmp(ifp->name, name) != 0)
+				continue;
+
+			pbr_ifp = ifp->info;
+
+			if (strcmp(pbr_ifp->mapname, "") == 0)
+				continue;
+
+			pbrm = pbrm_find(pbr_ifp->mapname);
+			vty_out(vty, "  %s(%d) with pbr-policy %s", ifp->name,
+				ifp->ifindex, pbr_ifp->mapname);
+			if (!pbrm)
+				vty_out(vty, " (map doesn't exist)");
+			vty_out(vty, "\n");
+		}
+	}
+
+	return CMD_SUCCESS;
+}
+
+static struct cmd_node interface_node = {
+	INTERFACE_NODE, "%s(config-if)# ", 1 /* vtysh ? yes */
+};
+
+static int pbr_interface_config_write(struct vty *vty)
+{
+	struct interface *ifp;
+	struct vrf *vrf;
+
+	RB_FOREACH (vrf, vrf_name_head, &vrfs_by_name) {
+		FOR_ALL_INTERFACES (vrf, ifp) {
+			if (vrf->vrf_id == VRF_DEFAULT)
+				vty_frame(vty, "interface %s\n", ifp->name);
+			else
+				vty_frame(vty, "interface %s vrf %s\n",
+					  ifp->name, vrf->name);
+
+			pbr_map_write_interfaces(vty, ifp);
+
+			vty_endframe(vty, "!\n");
+		}
+	}
+
+	return 1;
+}
+
+/* PBR map node structure. */
+static struct cmd_node pbr_map_node = {PBRMAP_NODE, "%s(config-pbr-map)# ", 1};
+
+static int pbr_vty_map_config_write_sequence(struct vty *vty,
+					     struct pbr_map *pbrm,
+					     struct pbr_map_sequence *pbrms)
+{
+	char buff[PREFIX_STRLEN];
+
+	vty_out (vty, "pbr-map %s seq %u\n",
+		 pbrm->name, pbrms->seqno);
+
+	if (pbrms->src)
+		vty_out(vty, "  match src-ip %s\n",
+			prefix2str(pbrms->src, buff, sizeof buff));
+
+	if (pbrms->dst)
+		vty_out(vty, "  match dst-ip %s\n",
+			prefix2str(pbrms->dst, buff, sizeof buff));
+
+	if (pbrms->nhgrp_name)
+		vty_out(vty, "  set nexthop-group %s\n", pbrms->nhgrp_name);
+
+	if (pbrms->nhg) {
+		vty_out(vty, "  set");
+		nexthop_group_write_nexthop(vty, pbrms->nhg->nexthop);
+	}
+
+	vty_out (vty, "!\n");
+	return 1;
+}
+
+static int pbr_vty_map_config_write(struct vty *vty)
+{
+	struct pbr_map *pbrm;
+
+	pbr_nht_write_table_range(vty);
+	pbr_nht_write_rule_range(vty);
+
+	RB_FOREACH(pbrm, pbr_map_entry_head, &pbr_maps) {
+		struct pbr_map_sequence *pbrms;
+		struct listnode *node;
+
+		for (ALL_LIST_ELEMENTS_RO(pbrm->seqnumbers, node, pbrms)) {
+			pbr_vty_map_config_write_sequence(vty, pbrm, pbrms);
+		}
+	}
+
+	return 1;
+}
+
+void pbr_vty_init(void)
+{
+	install_node(&interface_node,
+		     pbr_interface_config_write);
+	if_cmd_init();
+
+	install_node(&pbr_map_node,
+		     pbr_vty_map_config_write);
+
+	install_default(PBRMAP_NODE);
+
+	install_element(CONFIG_NODE, &pbr_map_cmd);
+	install_element(CONFIG_NODE, &no_pbr_map_cmd);
+	install_element(INTERFACE_NODE, &pbr_policy_cmd);
+	install_element(CONFIG_NODE, &pbr_table_range_cmd);
+	install_element(CONFIG_NODE, &pbr_rule_range_cmd);
+	install_element(PBRMAP_NODE, &pbr_map_match_src_cmd);
+	install_element(PBRMAP_NODE, &pbr_map_match_dst_cmd);
+	install_element(PBRMAP_NODE, &pbr_map_nexthop_group_cmd);
+	install_element(PBRMAP_NODE, &pbr_map_nexthop_cmd);
+	install_element(VIEW_NODE, &show_pbr_cmd);
+	install_element(VIEW_NODE, &show_pbr_map_cmd);
+	install_element(VIEW_NODE, &show_pbr_interface_cmd);
+	install_element(VIEW_NODE, &show_pbr_nexthop_group_cmd);
+
+	pbr_debug_init_vty();
+
+	return;
+}
diff --git a/pbrd/pbr_vty.h b/pbrd/pbr_vty.h
new file mode 100644
index 000000000..6e345fd7e
--- /dev/null
+++ b/pbrd/pbr_vty.h
@@ -0,0 +1,24 @@
+/*
+ * VTY library for PBR
+ * Copyright (C) 2018 Cumulus Networks, Inc.
+ *               Donald Sharp
+ *
+ * FRR is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2, or (at your option) any
+ * later version.
+ *
+ * FRR is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; see the file COPYING; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+#ifndef __PBR_VTY_H__
+#define __PBR_VTY_H__
+
+extern void pbr_vty_init(void);
+#endif
diff --git a/pbrd/pbr_zebra.c b/pbrd/pbr_zebra.c
new file mode 100644
index 000000000..8c2d470da
--- /dev/null
+++ b/pbrd/pbr_zebra.c
@@ -0,0 +1,579 @@
+/*
+ * Zebra connect code.
+ * Copyright (C) 2018 Cumulus Networks, Inc.
+ *               Donald Sharp
+ *
+ * FRR is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2, or (at your option) any
+ * later version.
+ *
+ * FRR is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; see the file COPYING; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+#include <zebra.h>
+
+#include "thread.h"
+#include "command.h"
+#include "network.h"
+#include "prefix.h"
+#include "routemap.h"
+#include "table.h"
+#include "stream.h"
+#include "memory.h"
+#include "zclient.h"
+#include "filter.h"
+#include "plist.h"
+#include "log.h"
+#include "nexthop.h"
+#include "nexthop_group.h"
+
+#include "pbr_nht.h"
+#include "pbr_map.h"
+#include "pbr_memory.h"
+#include "pbr_zebra.h"
+#include "pbr_debug.h"
+
+DEFINE_MTYPE_STATIC(PBRD, PBR_INTERFACE, "PBR Interface")
+
+/* Zebra structure to hold current status. */
+struct zclient *zclient = NULL;
+
+/* For registering threads. */
+extern struct thread_master *master;
+
+static struct interface *zebra_interface_if_lookup(struct stream *s)
+{
+	char ifname_tmp[INTERFACE_NAMSIZ];
+
+	/* Read interface name. */
+	stream_get(ifname_tmp, s, INTERFACE_NAMSIZ);
+
+	/* And look it up. */
+	return if_lookup_by_name(ifname_tmp, VRF_DEFAULT);
+}
+
+static struct pbr_interface *pbr_if_new(struct interface *ifp)
+{
+	struct pbr_interface *pbr_ifp;
+
+	zassert(ifp);
+	zassert(!ifp->info);
+
+	pbr_ifp = XCALLOC(MTYPE_PBR_INTERFACE, sizeof(*pbr_ifp));
+
+	if (!pbr_ifp) {
+		zlog_err("%s: PBR XCALLOC(%zu) failure", __PRETTY_FUNCTION__,
+			 sizeof(*pbr_ifp));
+		return 0;
+	}
+
+	return (pbr_ifp);
+}
+
+/* Inteface addition message from zebra. */
+static int interface_add(int command, struct zclient *zclient,
+			       zebra_size_t length, vrf_id_t vrf_id)
+{
+	struct interface *ifp;
+
+	ifp = zebra_interface_add_read(zclient->ibuf, vrf_id);
+
+	if (!ifp)
+		return 0;
+
+	if (!ifp->info) {
+		struct pbr_interface *pbr_ifp;
+
+		pbr_ifp = pbr_if_new(ifp);
+		ifp->info = pbr_ifp;
+	}
+
+	return 0;
+}
+
+static int interface_delete(int command, struct zclient *zclient,
+			    zebra_size_t length, vrf_id_t vrf_id)
+{
+	struct interface *ifp;
+	struct stream *s;
+
+	s = zclient->ibuf;
+	/* zebra_interface_state_read () updates interface structure in iflist
+	 */
+	ifp = zebra_interface_state_read(s, vrf_id);
+
+	if (ifp == NULL)
+		return 0;
+
+	if_set_index(ifp, IFINDEX_INTERNAL);
+
+	return 0;
+}
+
+static int interface_address_add(int command, struct zclient *zclient,
+				 zebra_size_t length, vrf_id_t vrf_id)
+{
+	zebra_interface_address_read(command, zclient->ibuf, vrf_id);
+
+	return 0;
+}
+
+static int interface_address_delete(int command, struct zclient *zclient,
+				    zebra_size_t length, vrf_id_t vrf_id)
+{
+	struct connected *c;
+
+	c = zebra_interface_address_read(command, zclient->ibuf, vrf_id);
+
+	if (!c)
+		return 0;
+
+	connected_free(c);
+	return 0;
+}
+
+static int interface_state_up(int command, struct zclient *zclient,
+			      zebra_size_t length, vrf_id_t vrf_id)
+{
+
+	zebra_interface_if_lookup(zclient->ibuf);
+
+	return 0;
+}
+
+static int interface_state_down(int command, struct zclient *zclient,
+				zebra_size_t length, vrf_id_t vrf_id)
+{
+
+	zebra_interface_state_read(zclient->ibuf, vrf_id);
+
+	return 0;
+}
+
+static int route_notify_owner(int command, struct zclient *zclient,
+			      zebra_size_t length, vrf_id_t vrf_id)
+{
+	struct prefix p;
+	enum zapi_route_notify_owner note;
+	uint32_t table_id;
+	char buf[PREFIX_STRLEN];
+
+	prefix2str(&p, buf, sizeof(buf));
+
+	if (!zapi_route_notify_decode(zclient->ibuf, &p, &table_id, &note))
+		return -1;
+
+	switch (note) {
+	case ZAPI_ROUTE_FAIL_INSTALL:
+		DEBUGD(&pbr_dbg_zebra,
+		       "%s: [%s] Route install failure for table: %u",
+		       __PRETTY_FUNCTION__, buf, table_id);
+		break;
+	case ZAPI_ROUTE_BETTER_ADMIN_WON:
+		DEBUGD(&pbr_dbg_zebra,
+		       "%s: [%s] Route better admin distance won for table: %u",
+		       __PRETTY_FUNCTION__, buf, table_id);
+		break;
+	case ZAPI_ROUTE_INSTALLED:
+		DEBUGD(&pbr_dbg_zebra,
+		       "%s: [%s] Route installed succeeded for table: %u",
+		       __PRETTY_FUNCTION__, buf, table_id);
+		pbr_nht_route_installed_for_table(table_id);
+		break;
+	case ZAPI_ROUTE_REMOVED:
+		DEBUGD(&pbr_dbg_zebra,
+		       "%s: [%s] Route Removed succeeded for table: %u",
+		       __PRETTY_FUNCTION__, buf, table_id);
+		pbr_nht_route_removed_for_table(table_id);
+		break;
+	case ZAPI_ROUTE_REMOVE_FAIL:
+		DEBUGD(&pbr_dbg_zebra,
+		       "%s: [%s] Route remove fail for table: %u",
+		       __PRETTY_FUNCTION__, buf, table_id);
+		break;
+	}
+
+	return 0;
+}
+
+static int rule_notify_owner(int command, struct zclient *zclient,
+			     zebra_size_t length, vrf_id_t vrf_id)
+{
+	uint32_t seqno, priority, unique;
+	enum zapi_rule_notify_owner note;
+	struct pbr_map_sequence *pbrms;
+	ifindex_t ifi;
+
+	if (!zapi_rule_notify_decode(zclient->ibuf, &seqno, &priority, &unique,
+				     &ifi, &note))
+		return -1;
+
+	pbrms = pbrms_lookup_unique(unique, ifi);
+	if (!pbrms) {
+		DEBUGD(&pbr_dbg_zebra,
+		       "%s: Failure to lookup pbrms based upon %u",
+		       __PRETTY_FUNCTION__, unique);
+		return 0;
+	}
+
+	switch (note) {
+	case ZAPI_RULE_FAIL_INSTALL:
+		DEBUGD(&pbr_dbg_zebra, "%s: Recieved RULE_FAIL_INSTALL",
+		       __PRETTY_FUNCTION__);
+		pbrms->installed = false;
+		break;
+	case ZAPI_RULE_INSTALLED:
+		pbrms->installed = true;
+		DEBUGD(&pbr_dbg_zebra, "%s: Recived RULE_INSTALLED",
+		       __PRETTY_FUNCTION__);
+		break;
+	case ZAPI_RULE_REMOVED:
+		DEBUGD(&pbr_dbg_zebra, "%s: Received RULE REMOVED",
+		       __PRETTY_FUNCTION__);
+		break;
+	}
+
+	return 0;
+}
+
+static void zebra_connected(struct zclient *zclient)
+{
+	zclient_send_reg_requests(zclient, VRF_DEFAULT);
+}
+
+static void route_add_helper(struct zapi_route *api, struct nexthop_group nhg,
+			     uint8_t install_afi)
+{
+	struct zapi_nexthop *api_nh;
+	struct nexthop *nhop;
+	int i;
+
+	api->prefix.family = install_afi;
+
+	i = 0;
+	for (ALL_NEXTHOPS(nhg, nhop)) {
+		api_nh = &api->nexthops[i];
+		api_nh->vrf_id = nhop->vrf_id;
+		api_nh->type = nhop->type;
+		switch (nhop->type) {
+		case NEXTHOP_TYPE_IPV4:
+			api_nh->gate.ipv4 = nhop->gate.ipv4;
+			break;
+		case NEXTHOP_TYPE_IPV4_IFINDEX:
+			api_nh->gate.ipv4 = nhop->gate.ipv4;
+			api_nh->ifindex = nhop->ifindex;
+			break;
+		case NEXTHOP_TYPE_IFINDEX:
+			api_nh->ifindex = nhop->ifindex;
+			break;
+		case NEXTHOP_TYPE_IPV6:
+			memcpy(&api_nh->gate.ipv6, &nhop->gate.ipv6, 16);
+			break;
+		case NEXTHOP_TYPE_IPV6_IFINDEX:
+			api_nh->ifindex = nhop->ifindex;
+			memcpy(&api_nh->gate.ipv6, &nhop->gate.ipv6, 16);
+			break;
+		case NEXTHOP_TYPE_BLACKHOLE:
+			api_nh->bh_type = nhop->bh_type;
+			break;
+		}
+		i++;
+	}
+	api->nexthop_num = i;
+
+	zclient_route_send(ZEBRA_ROUTE_ADD, zclient, api);
+}
+
+/*
+ * This function assumes a default route is being
+ * installed into the appropriate tableid
+ */
+void route_add(struct pbr_nexthop_group_cache *pnhgc, struct nexthop_group nhg,
+	       afi_t install_afi)
+{
+	struct zapi_route api;
+
+	memset(&api, 0, sizeof(api));
+
+	api.vrf_id = VRF_DEFAULT;
+	api.type = ZEBRA_ROUTE_PBR;
+	api.safi = SAFI_UNICAST;
+	/*
+	 * Sending a default route
+	 */
+	api.tableid = pnhgc->table_id;
+	SET_FLAG(api.flags, ZEBRA_FLAG_ALLOW_RECURSION);
+	SET_FLAG(api.message, ZAPI_MESSAGE_TABLEID);
+	SET_FLAG(api.message, ZAPI_MESSAGE_NEXTHOP);
+	switch (install_afi) {
+	case AFI_MAX:
+		route_add_helper(&api, nhg, AF_INET);
+		route_add_helper(&api, nhg, AF_INET6);
+		break;
+	case AFI_IP:
+		route_add_helper(&api, nhg, AF_INET);
+		break;
+	case AFI_IP6:
+		route_add_helper(&api, nhg, AF_INET6);
+		break;
+	case AFI_L2VPN:
+		DEBUGD(&pbr_dbg_zebra,
+		       "%s: Asked to install unsupported route type: L2VPN",
+		       __PRETTY_FUNCTION__);
+		break;
+	}
+}
+
+/*
+ * This function assumes a default route is being
+ * removed from the appropriate tableid
+ */
+void route_delete(struct pbr_nexthop_group_cache *pnhgc, afi_t afi)
+{
+	struct zapi_route api;
+
+	memset(&api, 0, sizeof(api));
+	api.vrf_id = VRF_DEFAULT;
+	api.type = ZEBRA_ROUTE_PBR;
+	api.safi = SAFI_UNICAST;
+
+	api.tableid = pnhgc->table_id;
+	SET_FLAG(api.message, ZAPI_MESSAGE_TABLEID);
+
+	switch (afi) {
+	case AFI_IP:
+		api.prefix.family = AF_INET;
+		zclient_route_send(ZEBRA_ROUTE_DELETE, zclient, &api);
+		break;
+	case AFI_IP6:
+		api.prefix.family = AF_INET6;
+		zclient_route_send(ZEBRA_ROUTE_DELETE, zclient, &api);
+		break;
+	case AFI_MAX:
+		api.prefix.family = AF_INET;
+		zclient_route_send(ZEBRA_ROUTE_DELETE, zclient, &api);
+		api.prefix.family = AF_INET6;
+		zclient_route_send(ZEBRA_ROUTE_DELETE, zclient, &api);
+		break;
+	case AFI_L2VPN:
+		DEBUGD(&pbr_dbg_zebra,
+		       "%s: Asked to delete unsupported route type: L2VPN",
+		       __PRETTY_FUNCTION__);
+		break;
+	}
+
+	return;
+}
+
+static int pbr_zebra_nexthop_update(int command, struct zclient *zclient,
+				    zebra_size_t length, vrf_id_t vrf_id)
+{
+	struct zapi_route nhr;
+	char buf[PREFIX2STR_BUFFER];
+	uint32_t i;
+
+	zapi_nexthop_update_decode(zclient->ibuf, &nhr);
+
+	if (DEBUG_MODE_CHECK(&pbr_dbg_zebra, DEBUG_MODE_ALL)) {
+
+		DEBUGD(&pbr_dbg_zebra, "%s: Received Nexthop update: %s",
+		       __PRETTY_FUNCTION__,
+		       prefix2str(&nhr.prefix, buf, sizeof(buf)));
+
+		DEBUGD(&pbr_dbg_zebra, "%s: (\tNexthops(%u)",
+		       __PRETTY_FUNCTION__, nhr.nexthop_num);
+
+		for (i = 0; i < nhr.nexthop_num; i++) {
+			DEBUGD(&pbr_dbg_zebra,
+			       "%s: \tType: %d: vrf: %d, ifindex: %d gate: %s",
+			       __PRETTY_FUNCTION__, nhr.nexthops[i].type,
+			       nhr.nexthops[i].vrf_id, nhr.nexthops[i].ifindex,
+			       inet_ntoa(nhr.nexthops[i].gate.ipv4));
+		}
+	}
+
+	pbr_nht_nexthop_update(&nhr);
+	return 1;
+}
+
+extern struct zebra_privs_t pbr_privs;
+
+void pbr_zebra_init(void)
+{
+	struct zclient_options opt = { .receive_notify = true };
+
+	zclient = zclient_new_notify(master, &opt);
+
+	zclient_init(zclient, ZEBRA_ROUTE_PBR, 0, &pbr_privs);
+	zclient->zebra_connected = zebra_connected;
+	zclient->interface_add = interface_add;
+	zclient->interface_delete = interface_delete;
+	zclient->interface_up = interface_state_up;
+	zclient->interface_down = interface_state_down;
+	zclient->interface_address_add = interface_address_add;
+	zclient->interface_address_delete = interface_address_delete;
+	zclient->route_notify_owner = route_notify_owner;
+	zclient->rule_notify_owner = rule_notify_owner;
+	zclient->nexthop_update = pbr_zebra_nexthop_update;
+}
+
+void pbr_send_rnh(struct nexthop *nhop, bool reg)
+{
+	uint32_t command;
+	struct prefix p;
+
+	command = (reg) ?
+		ZEBRA_NEXTHOP_REGISTER : ZEBRA_NEXTHOP_UNREGISTER;
+
+	memset(&p, 0, sizeof(p));
+	switch(nhop->type) {
+	case NEXTHOP_TYPE_IFINDEX:
+	case NEXTHOP_TYPE_BLACKHOLE:
+		return;
+	case NEXTHOP_TYPE_IPV4:
+	case NEXTHOP_TYPE_IPV4_IFINDEX:
+		p.family = AF_INET;
+		p.u.prefix4.s_addr = nhop->gate.ipv4.s_addr;
+		p.prefixlen = 32;
+		break;
+	case NEXTHOP_TYPE_IPV6:
+	case NEXTHOP_TYPE_IPV6_IFINDEX:
+		p.family = AF_INET6;
+		memcpy(&p.u.prefix6, &nhop->gate.ipv6, 16);
+		p.prefixlen = 128;
+		break;
+	}
+
+	if (zclient_send_rnh(zclient, command, &p,
+			     false, nhop->vrf_id) < 0) {
+		zlog_warn("%s: Failure to send nexthop to zebra",
+			  __PRETTY_FUNCTION__);
+	}
+}
+
+static void pbr_encode_pbr_map_sequence_prefix(struct stream *s,
+					       struct prefix *p,
+					       u_char family)
+{
+	struct prefix any;
+
+	if (!p) {
+		memset(&any, 0, sizeof(any));
+		any.family = family;
+		p = &any;
+	}
+
+	stream_putc(s, p->family);
+	stream_putc(s, p->prefixlen);
+	stream_put(s, &p->u.prefix, prefix_blen(p));
+}
+
+static void pbr_encode_pbr_map_sequence(struct stream *s,
+					struct pbr_map_sequence *pbrms,
+					struct interface *ifp)
+{
+	u_char family;
+
+	family = AF_INET;
+	if (pbrms->src)
+		family = pbrms->src->family;
+
+	if (pbrms->dst)
+		family = pbrms->dst->family;
+
+	stream_putl(s, pbrms->seqno);
+	stream_putl(s, pbrms->ruleno);
+	stream_putl(s, pbrms->unique);
+	pbr_encode_pbr_map_sequence_prefix(s, pbrms->src, family);
+	stream_putw(s, 0);  /* src port */
+	pbr_encode_pbr_map_sequence_prefix(s, pbrms->dst, family);
+	stream_putw(s, 0);  /* dst port */
+	if (pbrms->nhgrp_name)
+		stream_putl(s, pbr_nht_get_table(pbrms->nhgrp_name));
+	else if (pbrms->nhg)
+		stream_putl(s, pbr_nht_get_table(pbrms->internal_nhg_name));
+	stream_putl(s, ifp->ifindex);
+}
+
+void pbr_send_pbr_map(struct pbr_map *pbrm, bool install)
+{
+	struct listnode *inode, *snode;
+	struct pbr_map_sequence *pbrms;
+	struct pbr_map_interface *pmi;
+	struct stream *s;
+	uint32_t total;
+	ssize_t tspot;
+
+	DEBUGD(&pbr_dbg_zebra, "%s: for %s %d", __PRETTY_FUNCTION__, pbrm->name,
+	       install);
+
+	s = zclient->obuf;
+	stream_reset(s);
+
+	zclient_create_header(s,
+			      install ? ZEBRA_RULE_ADD : ZEBRA_RULE_DELETE,
+			      VRF_DEFAULT);
+
+	total = 0;
+	tspot = stream_get_endp(s);
+	stream_putl(s, 0);
+	for (ALL_LIST_ELEMENTS_RO(pbrm->incoming, inode, pmi)) {
+
+		DEBUGD(&pbr_dbg_zebra, "%s: \t%s %s %d %s %u",
+		       __PRETTY_FUNCTION__, install ? "Installing" : "Deleting",
+		       pbrm->name, install, pmi->ifp->name, pmi->delete);
+
+		if (!install && pmi->delete) {
+			for (ALL_LIST_ELEMENTS_RO(pbrm->seqnumbers, snode,
+						  pbrms)) {
+				pbr_encode_pbr_map_sequence(s,
+							    pbrms, pmi->ifp);
+				total++;
+			}
+			continue;
+		}
+
+		for (ALL_LIST_ELEMENTS_RO(pbrm->seqnumbers, snode, pbrms)) {
+
+			DEBUGD(&pbr_dbg_zebra, "%s: \tSeqno: %u %ld valid %u",
+			       __PRETTY_FUNCTION__, pbrms->seqno, pbrms->reason,
+			       pbrm->valid);
+
+			if (!install &&
+			    !(pbrms->reason & PBR_MAP_DEL_SEQUENCE_NUMBER))
+				continue;
+
+			if (!install && !pbrms->installed)
+				continue;
+
+			if (install && pbrms->installed)
+				continue;
+
+			DEBUGD(&pbr_dbg_zebra, "%s: \t Seq: %u ifp %s",
+			       __PRETTY_FUNCTION__, pbrms->seqno,
+			       pmi->ifp->name);
+
+			pbr_encode_pbr_map_sequence(s, pbrms, pmi->ifp);
+			total++;
+		}
+	}
+
+	DEBUGD(&pbr_dbg_zebra, "%s: Putting %u at %zu ", __PRETTY_FUNCTION__,
+	       total, tspot);
+
+	stream_putl_at(s, tspot, total);
+	stream_putw_at(s, 0, stream_get_endp(s));
+
+	if (!total) {
+		stream_reset(s);
+		return;
+	}
+	zclient_send_message(zclient);
+}
diff --git a/pbrd/pbr_zebra.h b/pbrd/pbr_zebra.h
new file mode 100644
index 000000000..2ee99ec83
--- /dev/null
+++ b/pbrd/pbr_zebra.h
@@ -0,0 +1,37 @@
+/*
+ * Zebra connect library for PBR
+ * Copyright (C) 2018 Cumulus Networks, Inc.
+ *               Donald Sharp
+ *
+ * FRR is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2, or (at your option) any
+ * later version.
+ *
+ * FRR is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; see the file COPYING; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+#ifndef __PBR_ZEBRA_H__
+#define __PBR_ZEBRA_H__
+
+struct pbr_interface {
+	char mapname[100];
+};
+
+extern void pbr_zebra_init(void);
+
+extern void route_add(struct pbr_nexthop_group_cache *pnhgc,
+		      struct nexthop_group nhg, afi_t install_afi);
+extern void route_delete(struct pbr_nexthop_group_cache *pnhgc,
+			 afi_t install_afi);
+
+extern void pbr_send_rnh(struct nexthop *nhop, bool reg);
+
+extern void pbr_send_pbr_map(struct pbr_map *pbrm, bool install);
+#endif
diff --git a/pbrd/pbrd.conf.sample b/pbrd/pbrd.conf.sample
new file mode 100644
index 000000000..bb1c2edca
--- /dev/null
+++ b/pbrd/pbrd.conf.sample
@@ -0,0 +1,3 @@
+!
+!
+log stdout
diff --git a/pbrd/subdir.am b/pbrd/subdir.am
new file mode 100644
index 000000000..361e6c1fd
--- /dev/null
+++ b/pbrd/subdir.am
@@ -0,0 +1,39 @@
+#
+# pbrd
+#
+
+if PBRD
+noinst_LIBRARIES += pbrd/libpbr.a
+sbin_PROGRAMS += pbrd/pbrd
+dist_examples_DATA += pbrd/pbrd.conf.sample
+endif
+
+pbrd_libpbr_a_SOURCES = \
+	pbrd/pbr_zebra.c \
+	pbrd/pbr_vty.c \
+	pbrd/pbr_map.c \
+	pbrd/pbr_memory.c \
+	pbrd/pbr_nht.c \
+	pbrd/pbr_event.c \
+	pbrd/pbr_debug.c \
+	# end
+
+noinst_HEADERS += \
+	pbrd/pbr_event.h \
+	pbrd/pbr_map.h \
+	pbrd/pbr_memory.h \
+	pbrd/pbr_nht.h \
+	pbrd/pbr_vty.h \
+	pbrd/pbr_zebra.h \
+	pbrd/pbr_debug.h \
+	# end
+
+pbrd/pbr_vty_clippy.c: $(CLIPPY_DEPS)
+pbrd/pbr_vty.$(OBJEXT): pbrd/pbr_vty_clippy.c
+
+pbrd/pbr_debug_clippy.c: $(CLIPPY_DEPS)
+pbrd/pbr_debug.$(OBJEXT): pbrd/pbr_debug_clippy.c
+
+pbrd_pbrd_SOURCES = pbrd/pbr_main.c
+pbrd_pbrd_LDADD = pbrd/libpbr.a lib/libfrr.la @LIBCAP@
+