*: Add and use option for graceful (re)start

Add a new start option "-K" to libfrr to denote a graceful start,
and use it in zebra and bgpd.

zebra will use this option to denote a planned FRR graceful restart
(supporting only bgpd currently) to wait for a route sync completion
from bgpd before cleaning up old stale routes from the FIB. An optional
timer provides an upper-bounds for this cleanup.

bgpd will use this option to denote either a planned FRR graceful
restart or a bgpd-only graceful restart, and this will drive the BGP
GR restarting router procedures.

Signed-off-by: Vivek Venkatraman <vivek@nvidia.com>

16 files changed, 221 insertions, 92 deletions

diff --git a/bgpd/bgp_main.c b/bgpd/bgp_main.c
index bfedb5015..5e6a62c9b 100644
--- a/bgpd/bgp_main.c
+++ b/bgpd/bgp_main.c
@@ -519,7 +519,9 @@ int main(int argc, char **argv)
 		bgp_option_set(BGP_OPT_NO_FIB);
 	if (no_zebra_flag)
 		bgp_option_set(BGP_OPT_NO_ZEBRA);
-	SET_FLAG(bm->flags, BM_FLAG_GRACEFUL_RESTART);
+	if (bgpd_di.graceful_restart)
+		SET_FLAG(bm->flags, BM_FLAG_GRACEFUL_RESTART);
+
 	bgp_error_init();
 	/* Initializations. */
 	libagentx_init();
diff --git a/bgpd/bgp_vty.c b/bgpd/bgp_vty.c
index 8ba167364..e9a79766e 100644
--- a/bgpd/bgp_vty.c
+++ b/bgpd/bgp_vty.c
@@ -3706,9 +3706,10 @@ DEFPY (neighbor_graceful_shutdown,
 	afi_t afi;
 	safi_t safi;
 	struct peer *peer;
-	VTY_DECLVAR_CONTEXT(bgp, bgp);
 	int ret;
 
+	VTY_DECLVAR_CONTEXT(bgp, bgp);
+
 	peer = peer_and_group_lookup_vty(vty, neighbor);
 	if (!peer)
 		return CMD_WARNING_CONFIG_FAILED;
diff --git a/doc/user/bgp.rst b/doc/user/bgp.rst
index 034d57956..98834c7c2 100644
--- a/doc/user/bgp.rst
+++ b/doc/user/bgp.rst
@@ -92,6 +92,12 @@ be specified (:ref:`common-invocation-options`).
    the operator has turned off communication to zebra and is running bgpd
    as a complete standalone process.
 
+.. option:: -K, --graceful_restart
+
+   Bgpd will use this option to denote either a planned FRR graceful
+   restart or a bgpd-only graceful restart, and this will drive the BGP
+   GR restarting router procedures.
+
 LABEL MANAGER
 -------------
 
diff --git a/lib/libfrr.c b/lib/libfrr.c
index 876efe23a..338a7d034 100644
--- a/lib/libfrr.c
+++ b/lib/libfrr.c
@@ -102,23 +102,25 @@ static void opt_extend(const struct optspec *os)
 #define OPTION_SCRIPTDIR 1009
 
 static const struct option lo_always[] = {
-	{"help", no_argument, NULL, 'h'},
-	{"version", no_argument, NULL, 'v'},
-	{"daemon", no_argument, NULL, 'd'},
-	{"module", no_argument, NULL, 'M'},
-	{"profile", required_argument, NULL, 'F'},
-	{"pathspace", required_argument, NULL, 'N'},
-	{"vrfdefaultname", required_argument, NULL, 'o'},
-	{"vty_socket", required_argument, NULL, OPTION_VTYSOCK},
-	{"moduledir", required_argument, NULL, OPTION_MODULEDIR},
-	{"scriptdir", required_argument, NULL, OPTION_SCRIPTDIR},
-	{"log", required_argument, NULL, OPTION_LOG},
-	{"log-level", required_argument, NULL, OPTION_LOGLEVEL},
-	{"command-log-always", no_argument, NULL, OPTION_LOGGING},
-	{"limit-fds", required_argument, NULL, OPTION_LIMIT_FDS},
-	{NULL}};
+	{ "help", no_argument, NULL, 'h' },
+	{ "version", no_argument, NULL, 'v' },
+	{ "daemon", no_argument, NULL, 'd' },
+	{ "module", no_argument, NULL, 'M' },
+	{ "profile", required_argument, NULL, 'F' },
+	{ "pathspace", required_argument, NULL, 'N' },
+	{ "vrfdefaultname", required_argument, NULL, 'o' },
+	{ "graceful_restart", optional_argument, NULL, 'K' },
+	{ "vty_socket", required_argument, NULL, OPTION_VTYSOCK },
+	{ "moduledir", required_argument, NULL, OPTION_MODULEDIR },
+	{ "scriptdir", required_argument, NULL, OPTION_SCRIPTDIR },
+	{ "log", required_argument, NULL, OPTION_LOG },
+	{ "log-level", required_argument, NULL, OPTION_LOGLEVEL },
+	{ "command-log-always", no_argument, NULL, OPTION_LOGGING },
+	{ "limit-fds", required_argument, NULL, OPTION_LIMIT_FDS },
+	{ NULL }
+};
 static const struct optspec os_always = {
-	"hvdM:F:N:o:",
+	"hvdM:F:N:o:K::",
 	"  -h, --help         Display this help and exit\n"
 	"  -v, --version      Print program version\n"
 	"  -d, --daemon       Runs in daemon mode\n"
@@ -126,13 +128,15 @@ static const struct optspec os_always = {
 	"  -F, --profile      Use specified configuration profile\n"
 	"  -N, --pathspace    Insert prefix into config & socket paths\n"
 	"  -o, --vrfdefaultname     Set default VRF name.\n"
+	"  -K, --graceful_restart   FRR starting in Graceful Restart mode, with optional route-cleanup timer\n"
 	"      --vty_socket   Override vty socket path\n"
 	"      --moduledir    Override modules directory\n"
 	"      --scriptdir    Override scripts directory\n"
 	"      --log          Set Logging to stdout, syslog, or file:<name>\n"
 	"      --log-level    Set Logging Level to use, debug, info, warn, etc\n"
 	"      --limit-fds    Limit number of fds supported\n",
-	lo_always};
+	lo_always
+};
 
 static bool logging_to_stdout = false; /* set when --log stdout specified */
 
@@ -358,6 +362,8 @@ void frr_preinit(struct frr_daemon_info *daemon, int argc, char **argv)
 	strlcpy(frr_protonameinst, di->logname, sizeof(frr_protonameinst));
 
 	di->cli_mode = FRR_CLI_CLASSIC;
+	di->graceful_restart = false;
+	di->gr_cleanup_time = 0;
 
 	/* we may be starting with extra FDs open for whatever purpose,
 	 * e.g. logging, some module, etc.  Recording them here allows later
@@ -520,6 +526,11 @@ static int frr_opt(int opt)
 		di->db_file = optarg;
 		break;
 #endif
+	case 'K':
+		di->graceful_restart = true;
+		if (optarg)
+			di->gr_cleanup_time = atoi(optarg);
+		break;
 	case 'C':
 		if (di->flags & FRR_NO_SPLIT_CONFIG)
 			return 1;
diff --git a/lib/libfrr.h b/lib/libfrr.h
index 77d70448a..db9cfbcb1 100644
--- a/lib/libfrr.h
+++ b/lib/libfrr.h
@@ -118,6 +118,8 @@ struct frr_daemon_info {
 	bool dryrun;
 	bool daemon_mode;
 	bool terminal;
+	bool graceful_restart;
+	int gr_cleanup_time;
 	enum frr_cli_mode cli_mode;
 
 	struct event *read_in;
diff --git a/tests/topotests/lib/bgp.py b/tests/topotests/lib/bgp.py
index 3f4ed6e0b..bcd1c7481 100644
--- a/tests/topotests/lib/bgp.py
+++ b/tests/topotests/lib/bgp.py
@@ -3269,29 +3269,24 @@ def verify_graceful_restart(
 
             # Local GR mode
             if "bgp" not in input_dict[dut] and "graceful-restart" in input_dict[dut]:
-                    if (
-                        "graceful-restart" in input_dict[dut]["graceful-restart"]
-                        and input_dict[dut]["graceful-restart"][
-                            "graceful-restart"
-                        ]
-                    ):
-                        lmode = "Restart*"
-                    elif (
-                        "graceful-restart-disable"
-                        in input_dict[dut]["graceful-restart"]
-                        and input_dict[dut]["graceful-restart"][
-                            "graceful-restart-disable"
-                        ]
-                    ):
-                        lmode = "Disable*"
-                    else:
-                        lmode = "Helper*"
+                if (
+                    "graceful-restart" in input_dict[dut]["graceful-restart"]
+                    and input_dict[dut]["graceful-restart"]["graceful-restart"]
+                ):
+                    lmode = "Restart*"
+                elif (
+                    "graceful-restart-disable" in input_dict[dut]["graceful-restart"]
+                    and input_dict[dut]["graceful-restart"]["graceful-restart-disable"]
+                ):
+                    lmode = "Disable*"
+                else:
+                    lmode = "Helper*"
 
             if lmode is None:
                 if "address_family" in input_dict[dut]["bgp"]:
                     bgp_neighbors = input_dict[dut]["bgp"]["address_family"][addr_type][
-                    "unicast"
-                ]["neighbor"][peer]["dest_link"]
+                        "unicast"
+                    ]["neighbor"][peer]["dest_link"]
 
                     for dest_link, data in bgp_neighbors.items():
                         if (
@@ -3336,7 +3331,10 @@ def verify_graceful_restart(
 
             # Remote GR mode
 
-            if "bgp" in input_dict[peer] and "address_family" in input_dict[peer]["bgp"]:
+            if (
+                "bgp" in input_dict[peer]
+                and "address_family" in input_dict[peer]["bgp"]
+            ):
                 bgp_neighbors = input_dict[peer]["bgp"]["address_family"][addr_type][
                     "unicast"
                 ]["neighbor"][dut]["dest_link"]
@@ -3358,7 +3356,10 @@ def verify_graceful_restart(
                         rmode = None
 
             if rmode is None:
-                if "bgp" in input_dict[peer] and  "graceful-restart" in input_dict[peer]["bgp"]:
+                if (
+                    "bgp" in input_dict[peer]
+                    and "graceful-restart" in input_dict[peer]["bgp"]
+                ):
                     if (
                         "graceful-restart"
                         in input_dict[peer]["bgp"]["graceful-restart"]
@@ -3379,13 +3380,13 @@ def verify_graceful_restart(
                         rmode = "Helper"
 
             if rmode is None:
-                if "bgp" not in input_dict[peer] and "graceful-restart" in input_dict[peer]:
+                if (
+                    "bgp" not in input_dict[peer]
+                    and "graceful-restart" in input_dict[peer]
+                ):
                     if (
-                        "graceful-restart"
-                        in input_dict[peer]["graceful-restart"]
-                        and input_dict[peer]["graceful-restart"][
-                            "graceful-restart"
-                        ]
+                        "graceful-restart" in input_dict[peer]["graceful-restart"]
+                        and input_dict[peer]["graceful-restart"]["graceful-restart"]
                     ):
                         rmode = "Restart"
                     elif (
diff --git a/zebra/main.c b/zebra/main.c
index ea1e1cbdb..687da70ca 100644
--- a/zebra/main.c
+++ b/zebra/main.c
@@ -65,8 +65,6 @@ struct mgmt_be_client *mgmt_be_client;
 /* Route retain mode flag. */
 int retain_mode = 0;
 
-int graceful_restart;
-
 /* Receive buffer size for kernel control sockets */
 #define RCVBUFSIZE_MIN 4194304
 #ifdef HAVE_NETLINK
@@ -88,7 +86,6 @@ const struct option longopts[] = {
 	{ "socket", required_argument, NULL, 'z' },
 	{ "ecmp", required_argument, NULL, 'e' },
 	{ "retain", no_argument, NULL, 'r' },
-	{ "graceful_restart", required_argument, NULL, 'K' },
 	{ "asic-offload", optional_argument, NULL, OPTION_ASIC_OFFLOAD },
 	{ "v6-with-v4-nexthops", no_argument, NULL, OPTION_V6_WITH_V4_NEXTHOP },
 #ifdef HAVE_NETLINK
@@ -96,7 +93,7 @@ const struct option longopts[] = {
 	{ "nl-bufsize", required_argument, NULL, 's' },
 	{ "v6-rr-semantics", no_argument, NULL, OPTION_V6_RR_SEMANTICS },
 #endif /* HAVE_NETLINK */
-	{"routing-table", optional_argument, NULL, 'R'},
+	{ "routing-table", optional_argument, NULL, 'R' },
 	{ 0 }
 };
 
@@ -326,7 +323,6 @@ int main(int argc, char **argv)
 	bool v6_with_v4_nexthop = false;
 	bool notify_on_ack = true;
 
-	graceful_restart = 0;
 	vrf_configure_backend(VRF_BACKEND_VRF_LITE);
 
 	frr_preinit(&zebra_di, argc, argv);
@@ -342,7 +338,6 @@ int main(int argc, char **argv)
 		    "  -z, --socket              Set path of zebra socket\n"
 		    "  -e, --ecmp                Specify ECMP to use.\n"
 		    "  -r, --retain              When program terminates, retain added route by zebra.\n"
-		    "  -K, --graceful_restart    Graceful restart at the kernel level, timer in seconds for expiration\n"
 		    "  -A, --asic-offload        FRR is interacting with an asic underneath the linux kernel\n"
 		    "      --v6-with-v4-nexthops Underlying dataplane supports v6 routes with v4 nexthops"
 #ifdef HAVE_NETLINK
@@ -352,8 +347,7 @@ int main(int argc, char **argv)
 #else
 		    "  -s,                       Set kernel socket receive buffer size\n"
 #endif /* HAVE_NETLINK */
-		    "  -R, --routing-table       Set kernel routing table\n"
-	);
+		    "  -R, --routing-table       Set kernel routing table\n");
 
 	while (1) {
 		int opt = frr_getopt(argc, argv, NULL);
@@ -397,9 +391,6 @@ int main(int argc, char **argv)
 		case 'r':
 			retain_mode = 1;
 			break;
-		case 'K':
-			graceful_restart = atoi(optarg);
-			break;
 		case 's':
 			rcvbufsize = atoi(optarg);
 			if (rcvbufsize < RCVBUFSIZE_MIN)
@@ -488,11 +479,25 @@ int main(int argc, char **argv)
 	*  Clean up zebra-originated routes. The requests will be sent to OS
 	*  immediately, so originating PID in notifications from kernel
 	*  will be equal to the current getpid(). To know about such routes,
-	* we have to have route_read() called before.
+	*  we have to have route_read() called before.
+	*  If FRR is gracefully restarting, we either wait for clients
+	*  (e.g., BGP) to signal GR is complete else we wait for specified
+	*  duration.
 	*/
 	zrouter.startup_time = monotime(NULL);
-	event_add_timer(zrouter.master, rib_sweep_route, NULL, graceful_restart,
-			&zrouter.sweeper);
+	zrouter.rib_sweep_time = 0;
+	zrouter.graceful_restart = zebra_di.graceful_restart;
+	if (!zrouter.graceful_restart)
+		event_add_timer(zrouter.master, rib_sweep_route, NULL, 0, NULL);
+	else {
+		int gr_cleanup_time;
+
+		gr_cleanup_time = zebra_di.gr_cleanup_time
+					  ? zebra_di.gr_cleanup_time
+					  : ZEBRA_GR_DEFAULT_RIB_SWEEP_TIME;
+		event_add_timer(zrouter.master, rib_sweep_route, NULL,
+				gr_cleanup_time, &zrouter.t_rib_sweep);
+	}
 
 	/* Needed for BSD routing socket. */
 	pid = getpid();
diff --git a/zebra/rib.h b/zebra/rib.h
index a721f4bac..84ea766c4 100644
--- a/zebra/rib.h
+++ b/zebra/rib.h
@@ -622,10 +622,10 @@ static inline struct nexthop_group *rib_get_fib_backup_nhg(
 }
 
 extern void zebra_gr_process_client(afi_t afi, vrf_id_t vrf_id, uint8_t proto,
-				    uint8_t instance);
+				    uint8_t instance, time_t restart_time);
 
 extern int rib_add_gr_run(afi_t afi, vrf_id_t vrf_id, uint8_t proto,
-			  uint8_t instance);
+			  uint8_t instance, time_t restart_time);
 
 extern void zebra_vty_init(void);
 
diff --git a/zebra/zapi_msg.c b/zebra/zapi_msg.c
index 2a1eea959..654ade806 100644
--- a/zebra/zapi_msg.c
+++ b/zebra/zapi_msg.c
@@ -2414,6 +2414,7 @@ static void zsend_capabilities(struct zserv *client, struct zebra_vrf *zvrf)
 	stream_putl(s, zrouter.multipath_num);
 	stream_putc(s, zebra_mlag_get_role());
 	stream_putc(s, zrouter.v6_with_v4_nexthop);
+	stream_putc(s, zrouter.graceful_restart);
 	stream_putw_at(s, 0, stream_get_endp(s));
 	zserv_send_message(client, s);
 }
diff --git a/zebra/zebra_gr.c b/zebra/zebra_gr.c
index cee66cc05..d84f680e2 100644
--- a/zebra/zebra_gr.c
+++ b/zebra/zebra_gr.c
@@ -103,6 +103,7 @@ static struct client_gr_info *zebra_gr_client_info_create(struct zserv *client)
 	info->stale_client_ptr = client;
 
 	TAILQ_INSERT_TAIL(&(client->gr_info_queue), info, gr_info);
+	info->client_ptr = client;
 	return info;
 }
 
@@ -290,6 +291,7 @@ struct zebra_gr_afi_clean {
 	afi_t afi;
 	uint8_t proto;
 	uint8_t instance;
+	time_t restart_time;
 
 	struct event *t_gac;
 };
@@ -420,7 +422,7 @@ void zread_client_capabilities(ZAPI_HANDLER_ARGS)
 		 * Schedule for after anything already in the meta Q
 		 */
 		rib_add_gr_run(api.afi, api.vrf_id, client->proto,
-			       client->instance);
+			       client->instance, client->restart_time);
 		zebra_gr_process_client_stale_routes(client, info);
 		break;
 	case ZEBRA_CLIENT_ROUTE_UPDATE_PENDING:
@@ -455,7 +457,11 @@ static void zebra_gr_route_stale_delete_timer_expiry(struct event *thread)
 	struct zserv *client;
 	struct vrf *vrf = vrf_lookup_by_id(info->vrf_id);
 
-	client = (struct zserv *)info->stale_client_ptr;
+	info->t_stale_removal = NULL;
+	if (zrouter.graceful_restart)
+		client = (struct zserv *)info->client_ptr;
+	else
+		client = (struct zserv *)info->stale_client_ptr;
 
 	cnt = zebra_gr_delete_stale_routes(info);
 
@@ -486,16 +492,24 @@ static void zebra_gr_route_stale_delete_timer_expiry(struct event *thread)
  *
  * Returns true when a node is deleted else false
  */
-static bool zebra_gr_process_route_entry(struct zserv *client,
-					 struct route_node *rn,
-					 struct route_entry *re)
+static bool zebra_gr_process_route_entry(struct route_node *rn,
+					 struct route_entry *re,
+					 time_t compare_time, uint8_t proto)
 {
+	struct nexthop *nexthop;
+	char buf[PREFIX2STR_BUFFER];
+
 	/* If the route is not refreshed after restart, delete the entry */
-	if (re->uptime < client->restart_time) {
-		if (IS_ZEBRA_DEBUG_RIB)
-			zlog_debug("%s: Client %s stale route %pFX is deleted",
-				   __func__, zebra_route_string(client->proto),
-				   &rn->p);
+	if (re->uptime < compare_time) {
+		if (IS_ZEBRA_DEBUG_RIB) {
+			prefix2str(&rn->p, buf, sizeof(buf));
+			zlog_debug("%s: Client %s stale route %s is deleted",
+				   __func__, zebra_route_string(proto), buf);
+		}
+		SET_FLAG(re->status, ROUTE_ENTRY_INSTALLED);
+		for (ALL_NEXTHOPS(re->nhe->nhg, nexthop))
+			SET_FLAG(nexthop->flags, NEXTHOP_FLAG_FIB);
+
 		rib_delnode(rn, re);
 
 		return true;
@@ -532,8 +546,9 @@ static void zebra_gr_delete_stale_route_table_afi(struct event *event)
 
 			if (re->type == gac->proto &&
 			    re->instance == gac->instance &&
-			    zebra_gr_process_route_entry(
-				    gac->info->stale_client_ptr, rn, re))
+			    zebra_gr_process_route_entry(rn, re,
+							 gac->restart_time,
+							 gac->proto))
 				n++;
 
 			/* If the max route count is reached
@@ -567,28 +582,42 @@ static int32_t zebra_gr_delete_stale_route(struct client_gr_info *info,
 	uint8_t proto;
 	uint16_t instance;
 	struct zserv *s_client;
+	struct zserv *client;
+	time_t restart_time = time(NULL);
 
-	s_client = info->stale_client_ptr;
-	if (s_client == NULL) {
-		LOG_GR("%s: Stale client %s(%u) not present", __func__,
-		       zvrf->vrf->name, zvrf->vrf->vrf_id);
+	if ((info == NULL) || (zvrf == NULL))
 		return -1;
-	}
 
-	proto = s_client->proto;
-	instance = s_client->instance;
+	if (zrouter.graceful_restart) {
+		client = info->client_ptr;
+		if (client == NULL) {
+			LOG_GR("%s: client not present", __func__);
+			return -1;
+		}
+		proto = client->proto;
+		instance = client->instance;
+		restart_time = zrouter.startup_time;
+	} else {
+		s_client = info->stale_client_ptr;
+		if (s_client == NULL) {
+			LOG_GR("%s: Stale client not present", __func__);
+			return -1;
+		}
+		proto = s_client->proto;
+		instance = s_client->instance;
+		restart_time = s_client->restart_time;
+	}
 
 	LOG_GR("%s: Client %s %s(%u) stale routes are being deleted", __func__,
 	       zebra_route_string(proto), zvrf->vrf->name, zvrf->vrf->vrf_id);
 
 	/* Process routes for all AFI */
 	for (afi = AFI_IP; afi < AFI_MAX; afi++) {
-
 		/*
 		 * Schedule for immediately after anything in the
 		 * meta-Q
 		 */
-		rib_add_gr_run(afi, info->vrf_id, proto, instance);
+		rib_add_gr_run(afi, info->vrf_id, proto, instance, restart_time);
 	}
 	return 0;
 }
@@ -641,12 +670,13 @@ static void zebra_gr_process_client_stale_routes(struct zserv *client,
 
 	/*
 	 * Route update completed for all AFI, SAFI
-	 * Cancel the stale timer, routes are already being processed
+	 * Also perform the cleanup if FRR itself is gracefully restarting.
 	 */
-	if (info->t_stale_removal) {
+	info->route_sync_done_time = monotime(NULL);
+	if (info->t_stale_removal || zrouter.graceful_restart) {
 		struct vrf *vrf = vrf_lookup_by_id(info->vrf_id);
 
-		LOG_GR("%s: Client %s canceled stale delete timer vrf %s(%d)",
+		LOG_GR("%s: Client %s route update complete for all AFI/SAFI in vrf %s(%d)",
 		       __func__, zebra_route_string(client->proto),
 		       VRF_LOGNAME(vrf), info->vrf_id);
 		EVENT_OFF(info->t_stale_removal);
@@ -654,7 +684,7 @@ static void zebra_gr_process_client_stale_routes(struct zserv *client,
 }
 
 void zebra_gr_process_client(afi_t afi, vrf_id_t vrf_id, uint8_t proto,
-			     uint8_t instance)
+			     uint8_t instance, time_t restart_time)
 {
 	struct zserv *client = zserv_find_client(proto, instance);
 	struct client_gr_info *info = NULL;
@@ -676,6 +706,7 @@ void zebra_gr_process_client(afi_t afi, vrf_id_t vrf_id, uint8_t proto,
 	gac->afi = afi;
 	gac->proto = proto;
 	gac->instance = instance;
+	gac->restart_time = restart_time;
 
 	event_add_event(zrouter.master, zebra_gr_delete_stale_route_table_afi,
 			gac, 0, &gac->t_gac);
diff --git a/zebra/zebra_rib.c b/zebra/zebra_rib.c
index 5b95d8668..b176ea2fe 100644
--- a/zebra/zebra_rib.c
+++ b/zebra/zebra_rib.c
@@ -3159,6 +3159,7 @@ struct meta_q_gr_run {
 	vrf_id_t vrf_id;
 	uint8_t proto;
 	uint8_t instance;
+	time_t restart_time;
 };
 
 static void process_subq_gr_run(struct listnode *lnode)
@@ -3166,7 +3167,7 @@ static void process_subq_gr_run(struct listnode *lnode)
 	struct meta_q_gr_run *gr_run = listgetdata(lnode);
 
 	zebra_gr_process_client(gr_run->afi, gr_run->vrf_id, gr_run->proto,
-				gr_run->instance);
+				gr_run->instance, gr_run->restart_time);
 
 	XFREE(MTYPE_WQ_WRAPPER, gr_run);
 }
@@ -4266,7 +4267,8 @@ static int rib_meta_queue_early_route_add(struct meta_queue *mq, void *data)
 	return 0;
 }
 
-int rib_add_gr_run(afi_t afi, vrf_id_t vrf_id, uint8_t proto, uint8_t instance)
+int rib_add_gr_run(afi_t afi, vrf_id_t vrf_id, uint8_t proto, uint8_t instance,
+		   time_t restart_time)
 {
 	struct meta_q_gr_run *gr_run;
 
@@ -4276,6 +4278,7 @@ int rib_add_gr_run(afi_t afi, vrf_id_t vrf_id, uint8_t proto, uint8_t instance)
 	gr_run->proto = proto;
 	gr_run->vrf_id = vrf_id;
 	gr_run->instance = instance;
+	gr_run->restart_time = restart_time;
 
 	return mq_add_handler(gr_run, rib_meta_queue_gr_run_add);
 }
@@ -4694,6 +4697,10 @@ void rib_sweep_route(struct event *t)
 	struct vrf *vrf;
 	struct zebra_vrf *zvrf;
 
+	zrouter.rib_sweep_time = monotime(NULL);
+	/* TODO: Change to debug */
+	zlog_info("Sweeping the RIB for stale routes...");
+
 	RB_FOREACH (vrf, vrf_id_head, &vrfs_by_id) {
 		if ((zvrf = vrf->info) == NULL)
 			continue;
diff --git a/zebra/zebra_router.c b/zebra/zebra_router.c
index 3fd4e6eb1..8d6b2f347 100644
--- a/zebra/zebra_router.c
+++ b/zebra/zebra_router.c
@@ -238,7 +238,7 @@ void zebra_router_terminate(void)
 {
 	struct zebra_router_table *zrt, *tmp;
 
-	EVENT_OFF(zrouter.sweeper);
+	EVENT_OFF(zrouter.t_rib_sweep);
 
 	RB_FOREACH_SAFE (zrt, zebra_router_table_head, &zrouter.tables, tmp)
 		zebra_router_free_table(zrt);
diff --git a/zebra/zebra_router.h b/zebra/zebra_router.h
index 304170743..c86c6be1e 100644
--- a/zebra/zebra_router.h
+++ b/zebra/zebra_router.h
@@ -191,10 +191,16 @@ struct zebra_router {
 	enum multicast_mode ipv4_multicast_mode;
 
 	/*
-	 * Time for when we sweep the rib from old routes
+	 * zebra start time and time of sweeping RIB of old routes
 	 */
 	time_t startup_time;
-	struct event *sweeper;
+	time_t rib_sweep_time;
+
+	/* FRR fast/graceful restart info */
+	bool graceful_restart;
+	int gr_cleanup_time;
+#define ZEBRA_GR_DEFAULT_RIB_SWEEP_TIME 500
+	struct event *t_rib_sweep;
 
 	/*
 	 * The hash of nexthop groups associated with this router
diff --git a/zebra/zebra_vty.c b/zebra/zebra_vty.c
index c31218a7c..858e50303 100644
--- a/zebra/zebra_vty.c
+++ b/zebra/zebra_vty.c
@@ -3825,6 +3825,20 @@ DEFUN (show_zebra,
 	struct vrf *vrf;
 	struct ttable *table = ttable_new(&ttable_styles[TTSTYLE_BLANK]);
 	char *out;
+	char timebuf[MONOTIME_STRLEN];
+
+	time_to_string(zrouter.startup_time, timebuf);
+	vty_out(vty, "Zebra started%s at time %s",
+		zrouter.graceful_restart ? " gracefully" : "", timebuf);
+
+	if (zrouter.t_rib_sweep)
+		vty_out(vty,
+			"Zebra RIB sweep timer running, remaining time %lds\n",
+			event_timer_remain_second(zrouter.t_rib_sweep));
+	else {
+		time_to_string(zrouter.rib_sweep_time, timebuf);
+		vty_out(vty, "Zebra RIB sweep happened at %s", timebuf);
+	}
 
 	ttable_rowseps(table, 0, BOTTOM, true, '-');
 	ttable_add_row(table, "OS|%s(%s)", cmd_system_get(), cmd_release_get());
diff --git a/zebra/zserv.c b/zebra/zserv.c
index 27668534e..a731f7f27 100644
--- a/zebra/zserv.c
+++ b/zebra/zserv.c
@@ -1031,6 +1031,7 @@ static char *zserv_time_buf(time_t *time1, char *buf, int buflen)
 /* Display client info details */
 static void zebra_show_client_detail(struct vty *vty, struct zserv *client)
 {
+	struct client_gr_info *info;
 	char cbuf[ZEBRA_TIME_BUF], rbuf[ZEBRA_TIME_BUF];
 	char wbuf[ZEBRA_TIME_BUF], nhbuf[ZEBRA_TIME_BUF], mbuf[ZEBRA_TIME_BUF];
 	time_t connect_time, last_read_time, last_write_time;
@@ -1125,6 +1126,45 @@ static void zebra_show_client_detail(struct vty *vty, struct zserv *client)
 	vty_out(vty, "ES-EVI      %-12u%-12u%-12u\n",
 		client->local_es_evi_add_cnt, 0, client->local_es_evi_del_cnt);
 	vty_out(vty, "Errors: %u\n", client->error_cnt);
+
+	TAILQ_FOREACH (info, &client->gr_info_queue, gr_info) {
+		afi_t afi;
+		bool route_sync_done = true;
+		char timebuf[MONOTIME_STRLEN];
+
+		vty_out(vty, "VRF : %s\n", vrf_id_to_name(info->vrf_id));
+		vty_out(vty, "Capabilities : ");
+		switch (info->capabilities) {
+		case ZEBRA_CLIENT_GR_CAPABILITIES:
+			vty_out(vty, "Graceful Restart\n");
+			break;
+		case ZEBRA_CLIENT_ROUTE_UPDATE_COMPLETE:
+		case ZEBRA_CLIENT_ROUTE_UPDATE_PENDING:
+		case ZEBRA_CLIENT_GR_DISABLE:
+		case ZEBRA_CLIENT_RIB_STALE_TIME:
+			vty_out(vty, "None\n");
+			break;
+		}
+		for (afi = AFI_IP; afi < AFI_MAX; afi++) {
+			if (info->af_enabled[afi]) {
+				if (info->route_sync[afi])
+					vty_out(vty,
+						"AFI %d enabled, route sync DONE\n",
+						afi);
+				else {
+					vty_out(vty,
+						"AFI %d enabled, route sync NOT DONE\n",
+						afi);
+					route_sync_done = false;
+				}
+			}
+		}
+		if (route_sync_done) {
+			time_to_string(info->route_sync_done_time, timebuf);
+			vty_out(vty, "Route sync finished at %s", timebuf);
+		}
+	}
+
 	vty_out(vty, "Input Fifo: %zu:%zu Output Fifo: %zu:%zu\n",
 		client->ibuf_fifo->count, client->ibuf_fifo->max_count,
 		client->obuf_fifo->count, client->obuf_fifo->max_count);
diff --git a/zebra/zserv.h b/zebra/zserv.h
index 57d673060..87d2b4adb 100644
--- a/zebra/zserv.h
+++ b/zebra/zserv.h
@@ -64,6 +64,8 @@ struct client_gr_info {
 	/* Book keeping */
 	void *stale_client_ptr;
 	struct event *t_stale_removal;
+	void *client_ptr;
+	time_t route_sync_done_time;
 
 	TAILQ_ENTRY(client_gr_info) gr_info;
 };