Multi-Instance OSPF Summary

——————————————-------------

- etc/init.d/quagga is modified to support creating separate ospf daemon
  process for each instance. Each individual instance is monitored by
  watchquagga just like any protocol daemons.(requires initd-mi.patch).

- Vtysh is modified to able to connect to multiple daemons of the same
  protocol (supported for OSPF only for now).

- ospfd is modified to remember the Instance-ID that its invoked with. For
  the entire life of the process it caters to any command request that
  matches that instance-ID (unless its a non instance specific command).
  Routes/messages to zebra are tagged with instance-ID.

- zebra route/redistribute mechanisms are modified to work with
  [protocol type + instance-id]

- bgpd now has ability to have multiple instance specific redistribution
  for a protocol (OSPF only supported/tested for now).

- zlog ability to display instance-id besides the protocol/daemon name.

- Changes in other daemons are to because of the needed integration with
  some of the modified APIs/routines. (Didn’t prefer replicating too many
  separate instance specific APIs.)

- config/show/debug commands are modified to take instance-id argument
  as appropriate.

Guidelines to start using multi-instance ospf
---------------------------------------------

The patch is backward compatible, i.e for any previous way of single ospf
deamon(router ospf <cr>) will continue to work as is, including all the
show commands etc.

To enable multiple instances, do the following:

     1. service quagga stop
     2. Modify /etc/quagga/daemons to add instance-ids of each desired
        instance in the following format:
        ospfd=“yes"
        ospfd_instances="1,2,3"
	assuming you want to enable 3 instances with those instance ids.
     3. Create corresponding ospfd config files as ospfd-1.conf, ospfd-2.conf
        and ospfd-3.conf.
     4. service quagga start/restart
     5. Verify that the deamons are started as expected. You should see
        ospfd started with -n <instance-id> option.
     	ps –ef | grep quagga
     	With that /var/run/quagga/ should have ospfd-<instance-id>.pid and
	ospfd-<instance-id>/vty to each instance.
     6. vtysh to work with instances as you would with any other deamons.
     7. Overall most quagga semantics are the same working with the instance
     	deamon, like it is for any other daemon.

NOTE:
     To safeguard against errors leading to too many processes getting invoked,
     a hard limit on number of instance-ids is in place, currently its 5.
     Allowed instance-id range is <1-65535>
     Once daemons are up, show running from vtysh should show the instance-id
     of  each daemon as 'router ospf <instance-id>’  (without needing explicit
     configuration)
     Instance-id can not be changed via vtysh, other router ospf configuration
     is allowed as before.

Signed-off-by: Vipin Kumar <vipin@cumulusnetworks.com>
Reviewed-by: Daniel Walton <dwalton@cumulusnetworks.com>
Reviewed-by: Dinesh G Dutt <ddutt@cumulusnetworks.com>

5 files changed, 119 insertions, 25 deletions

diff --git a/lib/log.c b/lib/log.c
index 06f1171ae..cdca478ac 100644
--- a/lib/log.c
+++ b/lib/log.c
@@ -150,6 +150,7 @@ time_print(FILE *fp, struct timestamp_control *ctl)
 static void
 vzlog (struct zlog *zl, int priority, const char *format, va_list args)
 {
+  char proto_str[32];
   struct timestamp_control tsctl;
   tsctl.already_rendered = 0;
 
@@ -181,6 +182,11 @@ vzlog (struct zlog *zl, int priority, const char *format, va_list args)
       va_end(ac);
     }
 
+  if (zl->instance)
+   sprintf (proto_str, "%s[%d]: ", zlog_proto_names[zl->protocol], zl->instance);
+  else
+   sprintf (proto_str, "%s: ", zlog_proto_names[zl->protocol]);
+
   /* File output. */
   if ((priority <= zl->maxlvl[ZLOG_DEST_FILE]) && zl->fp)
     {
@@ -188,7 +194,7 @@ vzlog (struct zlog *zl, int priority, const char *format, va_list args)
       time_print (zl->fp, &tsctl);
       if (zl->record_priority)
 	fprintf (zl->fp, "%s: ", zlog_priority[priority]);
-      fprintf (zl->fp, "%s: ", zlog_proto_names[zl->protocol]);
+      fprintf (zl->fp, "%s", proto_str);
       va_copy(ac, args);
       vfprintf (zl->fp, format, ac);
       va_end(ac);
@@ -203,7 +209,7 @@ vzlog (struct zlog *zl, int priority, const char *format, va_list args)
       time_print (stdout, &tsctl);
       if (zl->record_priority)
 	fprintf (stdout, "%s: ", zlog_priority[priority]);
-      fprintf (stdout, "%s: ", zlog_proto_names[zl->protocol]);
+      fprintf (stdout, "%s", proto_str);
       va_copy(ac, args);
       vfprintf (stdout, format, ac);
       va_end(ac);
@@ -214,7 +220,7 @@ vzlog (struct zlog *zl, int priority, const char *format, va_list args)
   /* Terminal monitor. */
   if (priority <= zl->maxlvl[ZLOG_DEST_MONITOR])
     vty_log ((zl->record_priority ? zlog_priority[priority] : NULL),
-	     zlog_proto_names[zl->protocol], format, &tsctl, args);
+	     proto_str, format, &tsctl, args);
 }
 
 static char *
@@ -600,7 +606,7 @@ _zlog_assert_failed (const char *assertion, const char *file,
 
 /* Open log stream */
 struct zlog *
-openzlog (const char *progname, zlog_proto_t protocol,
+openzlog (const char *progname, zlog_proto_t protocol, u_short instance,
 	  int syslog_flags, int syslog_facility)
 {
   struct zlog *zl;
@@ -610,6 +616,7 @@ openzlog (const char *progname, zlog_proto_t protocol,
 
   zl->ident = progname;
   zl->protocol = protocol;
+  zl->instance = instance;
   zl->facility = syslog_facility;
   zl->syslog_options = syslog_flags;
 
diff --git a/lib/log.h b/lib/log.h
index b27ba2530..28fc3736f 100644
--- a/lib/log.h
+++ b/lib/log.h
@@ -73,6 +73,7 @@ struct zlog
 {
   const char *ident;	/* daemon name (first arg to openlog) */
   zlog_proto_t protocol;
+  u_short      instance;
   int maxlvl[ZLOG_NUM_DESTS];	/* maximum priority to send to associated
   				   logging destination */
   int default_lvl;	/* maxlvl to use if none is specified */
@@ -97,7 +98,7 @@ extern struct zlog *zlog_default;
 
 /* Open zlog function */
 extern struct zlog *openzlog (const char *progname, zlog_proto_t protocol,
-		              int syslog_options, int syslog_facility);
+		              u_short instance, int syslog_options, int syslog_facility);
 
 /* Close zlog function. */
 extern void closezlog (struct zlog *zl);
diff --git a/lib/routemap.c b/lib/routemap.c
index b884fed2a..40ba87da4 100644
--- a/lib/routemap.c
+++ b/lib/routemap.c
@@ -327,8 +327,10 @@ vty_show_route_map_entry (struct vty *vty, struct route_map *map)
 
   /* Print the name of the protocol */
   if (zlog_default)
-    vty_out (vty, "%s:%s", zlog_proto_names[zlog_default->protocol],
-             VTY_NEWLINE);
+    vty_out (vty, "%s", zlog_proto_names[zlog_default->protocol]);
+  if (zlog_default->instance)
+    vty_out (vty, " %d", zlog_default->instance);
+  vty_out (vty, ":%s", VTY_NEWLINE);
 
   for (index = map->head; index; index = index->next)
     {
diff --git a/lib/zclient.c b/lib/zclient.c
index 24962bac3..f2f910f6b 100644
--- a/lib/zclient.c
+++ b/lib/zclient.c
@@ -78,10 +78,66 @@ zclient_free (struct zclient *zclient)
   XFREE (MTYPE_ZCLIENT, zclient);
 }
 
+int
+redist_check_instance (struct redist_proto *red, u_short instance)
+{
+  struct listnode *node;
+  u_short *id;
+
+  if (!red->instances)
+    return 0;
+
+  for (ALL_LIST_ELEMENTS_RO (red->instances, node, id))
+    if (*id == instance)
+      return 1;
+
+  return 0;
+}
+
+void
+redist_add_instance (struct redist_proto *red, u_short instance)
+{
+  u_short *in;
+
+  red->enabled = 1;
+
+  if (!red->instances)
+    red->instances = list_new();
+
+  in = (u_short *)calloc(1, sizeof(u_short));
+  *in = instance;
+  listnode_add (red->instances, in);
+}
+
+void
+redist_del_instance (struct redist_proto *red, u_short instance)
+{
+  struct listnode *node;
+  u_short *id = NULL;
+
+  if (!red->instances)
+    return 0;
+
+  for (ALL_LIST_ELEMENTS_RO (red->instances, node, id))
+    if (*id == instance)
+        break;
+
+  if (id)
+    {
+      listnode_delete(red->instances, id);
+      if (!red->instances->count)
+        {
+          red->enabled = 0;
+          list_free(red->instances);
+          red->instances = NULL;
+        }
+    }
+}
+
 /* Initialize zebra client.  Argument redist_default is unwanted
    redistribute route type. */
 void
-zclient_init (struct zclient *zclient, int redist_default)
+zclient_init (struct zclient *zclient, int redist_default, u_short instance)
 {
   int i;
   
@@ -93,12 +149,13 @@ zclient_init (struct zclient *zclient, int redist_default)
 
   /* Clear redistribution flags. */
   for (i = 0; i < ZEBRA_ROUTE_MAX; i++)
-    zclient->redist[i] = 0;
+    memset(&zclient->redist[i], 0, sizeof(struct redist_proto));
 
   /* Set unwanted redistribute route.  bgpd does not need BGP route
      redistribution. */
   zclient->redist_default = redist_default;
-  zclient->redist[redist_default] = 1;
+  zclient->instance = instance;
+  redist_add_instance (&zclient->redist[redist_default], instance);
 
   /* Set default-information redistribute to zero. */
   zclient->default_information = 0;
@@ -142,7 +199,7 @@ void
 zclient_reset (struct zclient *zclient)
 {
   zclient_stop (zclient);
-  zclient_init (zclient, zclient->redist_default);
+  zclient_init (zclient, zclient->redist_default, zclient->instance);
 }
 
 #ifdef HAVE_TCP_ZEBRA
@@ -330,6 +387,7 @@ zebra_hello_send (struct zclient *zclient)
 
       zclient_create_header (s, ZEBRA_HELLO);
       stream_putc (s, zclient->redist_default);
+      stream_putw (s, zclient->instance);
       stream_putw_at (s, 0, stream_get_endp (s));
       return zclient_send_message(zclient);
     }
@@ -388,8 +446,15 @@ zclient_start (struct zclient *zclient)
 
   /* Flush all redistribute request. */
   for (i = 0; i < ZEBRA_ROUTE_MAX; i++)
-    if (i != zclient->redist_default && zclient->redist[i])
-      zebra_redistribute_send (ZEBRA_REDISTRIBUTE_ADD, zclient, i);
+    if (zclient->redist[i].enabled)
+      {
+        struct listnode *node;
+        u_short *id;
+
+        for (ALL_LIST_ELEMENTS_RO(zclient->redist[i].instances, node, id))
+          if (!(i == zclient->redist_default && *id == zclient->instance))
+            zebra_redistribute_send (ZEBRA_REDISTRIBUTE_ADD, zclient, i, *id);
+      }
 
   /* If default information is needed. */
   if (zclient->default_information)
@@ -491,6 +556,7 @@ zapi_ipv4_route (u_char cmd, struct zclient *zclient, struct prefix_ipv4 *p,
   
   /* Put type and nexthop. */
   stream_putc (s, api->type);
+  stream_putw (s, api->instance);
   stream_putc (s, api->flags);
   stream_putc (s, api->message);
   stream_putw (s, api->safi);
@@ -572,6 +638,7 @@ zapi_ipv6_route (u_char cmd, struct zclient *zclient, struct prefix_ipv6 *p,
 
   /* Put type and nexthop. */
   stream_putc (s, api->type);
+  stream_putw (s, api->instance);
   stream_putc (s, api->flags);
   stream_putc (s, api->message);
   stream_putw (s, api->safi);
@@ -627,7 +694,8 @@ zapi_ipv6_route (u_char cmd, struct zclient *zclient, struct prefix_ipv6 *p,
  * sending client
  */
 int
-zebra_redistribute_send (int command, struct zclient *zclient, int type)
+zebra_redistribute_send (int command, struct zclient *zclient, int type,
+                         u_short instance)
 {
   struct stream *s;
 
@@ -636,6 +704,7 @@ zebra_redistribute_send (int command, struct zclient *zclient, int type)
   
   zclient_create_header (s, command);
   stream_putc (s, type);
+  stream_putw (s, instance);
   
   stream_putw_at (s, 0, stream_get_endp (s));
   
@@ -1153,24 +1222,25 @@ zclient_read (struct thread *thread)
 }
 
 void
-zclient_redistribute (int command, struct zclient *zclient, int type)
+zclient_redistribute (int command, struct zclient *zclient, int type,
+                      u_short instance)
 {
 
   if (command == ZEBRA_REDISTRIBUTE_ADD) 
     {
-      if (zclient->redist[type])
+      if (redist_check_instance(&zclient->redist[type], instance))
          return;
-      zclient->redist[type] = 1;
+      redist_add_instance(&zclient->redist[type], instance);
     }
   else
     {
-      if (!zclient->redist[type])
+      if (!redist_check_instance(&zclient->redist[type], instance))
          return;
-      zclient->redist[type] = 0;
+      redist_del_instance(&zclient->redist[type], instance);
     }
 
   if (zclient->sock > 0)
-    zebra_redistribute_send (command, zclient, type);
+    zebra_redistribute_send (command, zclient, type, instance);
 }
 
 
diff --git a/lib/zclient.h b/lib/zclient.h
index e1bb2000c..3c214d278 100644
--- a/lib/zclient.h
+++ b/lib/zclient.h
@@ -34,6 +34,12 @@
 /* Zebra header size. */
 #define ZEBRA_HEADER_SIZE             6
 
+struct redist_proto
+{
+  u_char enabled;
+  struct list *instances;
+};
+
 /* Structure for the zebra client. */
 struct zclient
 {
@@ -64,8 +70,9 @@ struct zclient
   struct thread *t_write;
 
   /* Redistribute information. */
-  u_char redist_default;
-  u_char redist[ZEBRA_ROUTE_MAX];
+  u_char redist_default; /* clients protocol */
+  u_short instance;
+  struct redist_proto redist[ZEBRA_ROUTE_MAX];
 
   /* Redistribute defauilt. */
   u_char default_information;
@@ -112,6 +119,7 @@ struct zserv_header
 struct zapi_ipv4
 {
   u_char type;
+  u_short instance;
 
   u_char flags;
 
@@ -134,7 +142,7 @@ struct zapi_ipv4
 
 /* Prototypes of zebra client service functions. */
 extern struct zclient *zclient_new (void);
-extern void zclient_init (struct zclient *, int);
+extern void zclient_init (struct zclient *, int, u_short);
 extern int zclient_start (struct zclient *);
 extern void zclient_stop (struct zclient *);
 extern void zclient_reset (struct zclient *);
@@ -143,11 +151,16 @@ extern void zclient_free (struct zclient *);
 extern int  zclient_socket_connect (struct zclient *);
 extern void zclient_serv_path_set  (char *path);
 
+extern int redist_check_instance (struct redist_proto *, u_short);
+extern void redist_add_instance (struct redist_proto *, u_short);
+extern void redist_del_instance (struct redist_proto *, u_short);
+
 /* Send redistribute command to zebra daemon. Do not update zclient state. */
-extern int zebra_redistribute_send (int command, struct zclient *, int type);
+extern int zebra_redistribute_send (int command, struct zclient *, int type, u_short instance);
 
 /* If state has changed, update state and call zebra_redistribute_send. */
-extern void zclient_redistribute (int command, struct zclient *, int type);
+extern void zclient_redistribute (int command, struct zclient *, int type,
+                                  u_short instance);
 
 /* If state has changed, update state and send the command to zebra. */
 extern void zclient_redistribute_default (int command, struct zclient *);
@@ -175,6 +188,7 @@ extern int zapi_ipv4_route (u_char, struct zclient *, struct prefix_ipv4 *,
 struct zapi_ipv6
 {
   u_char type;
+  u_short instance;
 
   u_char flags;