[ovs-dev,v3,ovn,1/2] Forwarding group to load balance l2 traffic with liveness detection
diff mbox series

Message ID 20200122200845.61083-2-manoj.sharma@nutanix.com
State New
Headers show
Series
  • Forwarding group to load balance l2 traffic with liveness detection
Related show

Commit Message

Manoj Sharma Jan. 22, 2020, 8:09 p.m. UTC
Add a forwarding group table and a reference to the logical switch
it is configured on. The forwarding group is configured with a virtual
IP, virtual MAC and a number of logical switch ports from a logical
switch. Unit tests for the forwarding group.

Signed-off-by: Manoj Sharma <manoj.sharma@nutanix.com>
---
 NEWS                      |   1 +
 northd/ovn-northd.8.xml   |  49 +++++++++
 ovn-nb.ovsschema          |  18 +++-
 ovn-nb.xml                |  35 +++++++
 ovn-sb.xml                |  19 ++++
 tests/ovn-nbctl.at        |  37 +++++++
 tests/ovn.at              | 124 +++++++++++++++++++++++
 utilities/ovn-nbctl.8.xml |  38 +++++++
 utilities/ovn-nbctl.c     | 253 ++++++++++++++++++++++++++++++++++++++++++++++
 9 files changed, 572 insertions(+), 2 deletions(-)

Patch
diff mbox series

diff --git a/NEWS b/NEWS
index 0ad9677..9e7d601 100644
--- a/NEWS
+++ b/NEWS
@@ -4,6 +4,7 @@  Post-OVS-v2.12.0
      independently.
    - Added IPv6 NAT support for OVN routers.
    - Added Stateless Floating IP support in OVN.
+   - Added Forwarding Group support in OVN.
 
 v2.12.0 - 03 Sep 2019
 ---------------------
diff --git a/northd/ovn-northd.8.xml b/northd/ovn-northd.8.xml
index 4b227ca..47ec328 100644
--- a/northd/ovn-northd.8.xml
+++ b/northd/ovn-northd.8.xml
@@ -767,6 +767,45 @@  output;
       </li>
 
       <li>
+        <p>
+          For each <var>VIP</var> configured in the table
+          <ref table="Forwarding_Group" db="OVN_Northbound"/>
+          a priority-50 logical flow is added with the match
+          <code>arp.tpa == <var>vip</var> &amp;&amp; &amp;&amp; arp.op == 1
+          </code> and applies the action
+        </p>
+
+        <pre>
+eth.dst = eth.src;
+eth.src = <var>E</var>;
+arp.op = 2; /* ARP reply. */
+arp.tha = arp.sha;
+arp.sha = <var>E</var>;
+arp.tpa = arp.spa;
+arp.spa = <var>A</var>;
+outport = inport;
+flags.loopback = 1;
+output;
+        </pre>
+
+        <p>
+          where <var>E</var> is the forwarding group's mac defined in
+          the <ref column="vmac" table="Forwarding_Group"
+          db="OVN_Northbound"/>.
+        </p>
+
+        <p>
+          <var>A</var> is used as either the destination ip for load balancing
+          traffic to child ports or as nexthop to hosts behind the child ports.
+        </p>
+
+        <p>
+          These flows are required to respond to an ARP request if an ARP
+          request is sent for the IP <var>vip</var>.
+        </p>
+      </li>
+
+      <li>
         One priority-0 fallback flow that matches all packets and advances to
         the next table.
       </li>
@@ -1153,6 +1192,16 @@  output;
             address is only programmed on the <code>redirect-chassis</code>.
           </li>
         </ul>
+
+        <p>
+          For each forwarding group configured on the logical switch datapath,
+          a priority-50 flow that matches on <code>eth.dst == <var>VIP</var>
+          </code> with an action of <code>fwd_group(childports=<var>args
+          </var>)</code>, where <var>args</var> contains comma separated
+          logical switch child ports to load balance to.
+          If <code>liveness</code> is enabled, then action also includes
+          <code> liveness=true</code>.
+        </p>
       </li>
 
       <li>
diff --git a/ovn-nb.ovsschema b/ovn-nb.ovsschema
index 12999a4..99b6285 100644
--- a/ovn-nb.ovsschema
+++ b/ovn-nb.ovsschema
@@ -1,7 +1,7 @@ 
 {
     "name": "OVN_Northbound",
     "version": "5.18.0",
-    "cksum": "2806349485 24196",
+    "cksum": "63300136 24879",
     "tables": {
         "NB_Global": {
             "columns": {
@@ -59,7 +59,12 @@ 
                              "min": 0, "max": "unlimited"}},
                 "external_ids": {
                     "type": {"key": "string", "value": "string",
-                             "min": 0, "max": "unlimited"}}},
+                             "min": 0, "max": "unlimited"}},
+               "forwarding_groups": {
+                    "type": {"key": {"type": "uuid",
+                                     "refTable": "Forwarding_Group",
+                                     "refType": "strong"},
+                                     "min": 0, "max": "unlimited"}}},
             "isRoot": true},
         "Logical_Switch_Port": {
             "columns": {
@@ -113,6 +118,15 @@ 
                              "min": 0, "max": "unlimited"}}},
             "indexes": [["name"]],
             "isRoot": false},
+        "Forwarding_Group": {
+            "columns": {
+                "name": {"type": "string"},
+                "vip": {"type": "string"},
+                "vmac": {"type": "string"},
+                "liveness": {"type": "boolean"},
+                "child_port": {"type": {"key": "string",
+                                        "min": 1, "max": "unlimited"}}},
+            "isRoot": false},
         "Address_Set": {
             "columns": {
                 "name": {"type": "string"},
diff --git a/ovn-nb.xml b/ovn-nb.xml
index 5ae52bb..decb4ae 100644
--- a/ovn-nb.xml
+++ b/ovn-nb.xml
@@ -197,6 +197,11 @@ 
       Please see the <ref table="DNS"/> table.
     </column>
 
+    <column name="forwarding_groups">
+      Groups a set of logical port endpoints for traffic going out of the
+      logical switch.
+    </column>
+
     <group title="Naming">
       <p>
         These columns provide names for the logical switch.  From OVN's
@@ -1152,6 +1157,36 @@ 
     </group>
   </table>
 
+  <table name="Forwarding_Group" title="forwarding group">
+    <p>
+      Each row represents one forwarding group.
+    </p>
+
+    <column name="name">
+      A name for the forwarding group.  This name has no special meaning or
+      purpose other than to provide convenience for human interaction with
+      the ovn-nb database.
+    </column>
+
+    <column name="vip">
+      The virtual IP address assigned to the forwarding group. It will respond
+      with vmac when an ARP request is sent for vip.
+    </column>
+
+    <column name="vmac">
+      The virtual MAC address assigned to the forwarding group.
+    </column>
+
+    <column name="liveness">
+      If set to <code>true</code>, liveness is enabled for child ports
+      otherwise it is disabled.
+    </column>
+
+    <column name="child_port">
+      List of child ports in the forwarding group.
+    </column>
+  </table>
+
   <table name="Address_Set" title="Address Sets">
     <p>
       Each row in this table represents a named set of addresses.
diff --git a/ovn-sb.xml b/ovn-sb.xml
index 82167c4..ea33481 100644
--- a/ovn-sb.xml
+++ b/ovn-sb.xml
@@ -2114,6 +2114,25 @@  tcp.flags = RST;
 
           <p><b>Example:</b> <code>handle_svc_check(inport);</code></p>
         </dd>
+
+        <dt><code>fwd_group(<var>P</var>);</code></dt>
+        <dd>
+          <p>
+            <b>Parameters</b>: liveness, list of child ports <var>P</var>.
+          </p>
+
+          <p>
+            It load balances traffic to one or more child ports in a
+            logical switch. <code>ovn-controller</code> translates the
+            <code>fwd_group</code> into openflow group with one bucket
+            for each child port. If liveness is set to true, it also
+            integrates the bucket selection with BFD status on the tunnel
+            interface corresponding to child port.
+          </p>
+
+          <p><b>Example:</b> <code>fwd_group(liveness=true, childports=p1,p2
+            </code></p>
+        </dd>
       </dl>
     </column>
 
diff --git a/tests/ovn-nbctl.at b/tests/ovn-nbctl.at
index 2679f1f..077c41e 100644
--- a/tests/ovn-nbctl.at
+++ b/tests/ovn-nbctl.at
@@ -1735,6 +1735,43 @@  $SW1P2
 AT_CHECK([ovn-nbctl pg-del pg1], [0], [ignore])
 AT_CHECK([ovn-nbctl list port_group], [0], [])
 ])
+dnl ---------------------------------------------------------------------
+
+OVN_NBCTL_TEST([ovn_nbctl_fwd_groups], [fwd groups], [
+
+dnl Add fwd-group to a non-existent logical switch
+AT_CHECK([ovn-nbctl fwd-group-add fwd_grp1 ls0 10.1.1.11 00:11:22:33:44:55 lsp1 lsp2], [1], [],
+  [ovn-nbctl: ls0: switch name not found
+])
+
+AT_CHECK([ovn-nbctl ls-add ls0])
+
+dnl Add fwd-group with non-existent logical switch ports
+AT_CHECK([ovn-nbctl fwd-group-add fwd_grp1 ls0 10.1.1.11 00:11:22:33:44:55 lsp1 lsp2], [1], [],
+  [ovn-nbctl: lsp1: logical switch port does not exist
+])
+
+AT_CHECK([ovn-nbctl lsp-add ls0 lsp1])
+AT_CHECK([ovn-nbctl lsp-add ls0 lsp2])
+AT_CHECK([ovn-nbctl fwd-group-add fwd_grp1 ls0 10.1.1.11 00:11:22:33:44:55 lsp1 lsp2])
+AT_CHECK([ovn-nbctl fwd-group-list ls0], [0], [dnl
+FWD_GROUP       LS            VIP             VMAC                  CHILD_PORTS
+fwd_grp1        ls0           10.1.1.11      00:11:22:33:44:55      lsp1 lsp2
+])
+AT_CHECK([ovn-nbctl --bare --columns=name list forwarding_group], [0],
+[fwd_grp1
+])
+
+dnl Add duplicate fwd-group
+AT_CHECK([ovn-nbctl fwd-group-add fwd_grp1 ls0 10.1.1.11 00:11:22:33:44:55 lsp1 lsp2], [1], [],
+  [ovn-nbctl: fwd_grp1: a forwarding group by this name already exists
+])
+
+dnl Delete fwd-group
+AT_CHECK([ovn-nbctl fwd-group-del fwd_grp1], [0], [ignore])
+AT_CHECK([ovn-nbctl list forwarding_group], [0], [])
+
+])
 
 AT_SETUP([ovn-nbctl - daemon retry connection])
 OVN_NBCTL_TEST_START daemon
diff --git a/tests/ovn.at b/tests/ovn.at
index adb677c..f128c7e 100644
--- a/tests/ovn.at
+++ b/tests/ovn.at
@@ -17374,3 +17374,127 @@  AT_CHECK([ovn-trace --ovs lsw0 'inport == "lp1" && eth.type == 0x1234' | grep "d
 OVN_CLEANUP([hv1])
 
 AT_CLEANUP
+
+AT_SETUP([ovn -- forwarding group: 3 HVs, 1 LR, 2 LS])
+AT_KEYWORDS([forwarding-group])
+ovn_start
+
+# Logical network:
+# One LR - R1 has a logical switch ls1 and ls2 connected to it.
+# Logical switch ls1 has one port while ls2 has two logical switch ports as
+# child ports.
+ovn-nbctl lr-add R1
+ovn-nbctl ls-add ls1
+ovn-nbctl ls-add ls2
+
+# Logical switch ls1 to R1 connectivity
+ovn-nbctl lrp-add R1 R1-ls1 00:00:00:01:02:f1 192.168.1.1/24
+ovn-nbctl lsp-add ls1 ls1-R1 -- set Logical_Switch_Port ls1-R1 \
+    type=router options:router-port=R1-ls1 -- lsp-set-addresses ls1-R1 router
+ovn-nbctl lsp-add ls1 lsp11 \
+    -- lsp-set-addresses lsp11 "00:00:00:01:02:01 192.168.1.2"
+
+# Logical switch ls2 to R1 connectivity
+ovn-nbctl lrp-add R1 R1-ls2 00:00:00:01:02:f2 172.16.1.1/24
+ovn-nbctl lsp-add ls2 ls2-R1 -- set Logical_Switch_Port ls2-R1 \
+    type=router options:router-port=R1-ls2 -- lsp-set-addresses ls2-R1 router
+ovn-nbctl lsp-add ls2 lsp21 \
+    -- lsp-set-addresses lsp21 "00:00:00:01:02:01 172.16.1.2"
+ovn-nbctl lsp-add ls2 lsp22 \
+    -- lsp-set-addresses lsp22 "00:00:00:01:02:02 172.16.1.3"
+
+# Create a network
+net_add n1
+
+# Create hypervisor hv1 connected to n1
+sim_add hv1
+as hv1
+ovs-vsctl add-br br-phys
+ovn_attach n1 br-phys 192.168.0.1
+ovs-vsctl add-port br-int vif1 -- set Interface vif1 external-ids:iface-id=lsp11 options:tx_pcap=hv1/vif1-tx.pcap options:rxq_pcap=hv1/vif1-rx.pcap ofport-request=1
+
+# Create hypervisor hv2 connected to n1
+sim_add hv2
+as hv2
+ovs-vsctl add-br br-phys
+ovn_attach n1 br-phys 192.168.0.2
+ovs-vsctl add-port br-int vif2 -- set Interface vif2 external-ids:iface-id=lsp21 options:tx_pcap=hv2/vif2-tx.pcap options:rxq_pcap=hv2/vif2-rx.pcap ofport-request=1
+
+# Create hypervisor hv3 connected to n1
+sim_add hv3
+as hv3
+ovs-vsctl add-br br-phys
+ovn_attach n1 br-phys 192.168.0.3
+ovs-vsctl add-port br-int vif3 -- set Interface vif3 external-ids:iface-id=lsp22 options:tx_pcap=hv3/vif3-tx.pcap options:rxq_pcap=hv3/vif3-rx.pcap ofport-request=1
+
+# Add a forwarding group on ls2 with lsp21 and lsp22 as child ports
+# virtual IP - 172.16.1.11, virtual MAC - 00:11:de:ad:be:ef
+ovn-nbctl fwd-group-add fwd_grp1 ls2 172.16.1.11 00:11:de:ad:be:ef lsp21 lsp22
+
+# Allow some time for ovn-northd and ovn-controller to catch up.
+sleep 1
+
+# Check logical flow
+AT_CHECK([ovn-sbctl dump-flows | grep ls_in_l2_lkup | grep fwd_group | wc -l], [0], [dnl
+1
+])
+
+# Check openflow rule with "group" on hypervisor
+AT_CHECK([as hv1 ovs-ofctl dump-flows br-int | \
+    grep "dl_dst=00:11:de:ad:be:ef actions=group" | wc -l], [0], [dnl
+1
+])
+
+# Verify openflow group members
+for child_port in lsp21 lsp22; do
+    tunnel_key=`ovn-sbctl --bare --column tunnel_key find port_binding logical_port=$child_port`
+    AT_CHECK([as hv1 ovs-ofctl -O OpenFlow13 dump-groups br-int | \
+        grep "bucket=actions=load:0x"$tunnel_key | wc -l], [0], [dnl
+1
+])
+done
+
+# Send a packet to virtual IP
+src_mac=00:00:00:01:02:01
+dst_mac=00:00:00:01:02:f1
+src_ip=192.168.1.2
+dst_ip=172.16.1.11
+packet="inport==\"lsp11\" && eth.src==$src_mac && eth.dst==$dst_mac &&
+        ip4 && ip.ttl==64 && ip4.src==$src_ip && ip4.dst==$dst_ip &&
+        udp && udp.src==53 && udp.dst==4369"
+as hv1 ovs-appctl -t ovn-controller inject-pkt "$packet"
+
+# Check if the packet hit the forwarding group policy
+AT_CHECK([as hv1 ovs-ofctl dump-flows br-int | \
+    grep "dl_dst=00:11:de:ad:be:ef actions=group" | \
+    grep "n_packets=1" | wc -l], [0], [dnl
+1
+])
+
+# Delete the forwarding group
+ovn-nbctl fwd-group-del fwd_grp1
+
+# Add a forwarding group with liveness on ls2 with lsp21 and lsp22 as child
+# ports virtual IP - 172.16.1.11, virtual MAC - 00:11:de:ad:be:ef
+ovn-nbctl --liveness fwd-group-add fwd_grp1 ls2 172.16.1.11 00:11:de:ad:be:ef lsp21 lsp22
+
+# Allow some time for ovn-northd and ovn-controller to catch up.
+sleep 1
+
+# Verify openflow group members
+ofport_lsp21=$(as hv1 ovs-vsctl --bare --columns ofport find Interface name=ovn-hv2-0)
+tunnel_key=`ovn-sbctl --bare --column tunnel_key find port_binding logical_port=lsp21`
+AT_CHECK([as hv1 ovs-ofctl -O OpenFlow13 dump-groups br-int | \
+    grep "bucket=watch_port:$ofport_lsp21,actions=load:0x"$tunnel_key | wc -l], [0], [dnl
+1
+])
+
+ofport_lsp22=$(as hv1 ovs-vsctl --bare --columns ofport find Interface name=ovn-hv3-0)
+tunnel_key=`ovn-sbctl --bare --column tunnel_key find port_binding logical_port=lsp22`
+AT_CHECK([as hv1 ovs-ofctl -O OpenFlow13 dump-groups br-int | \
+    grep "bucket=watch_port:$ofport_lsp22,actions=load:0x"$tunnel_key | wc -l], [0], [dnl
+1
+])
+
+OVN_CLEANUP([hv1], [hv2], [hv3])
+AT_CLEANUP
diff --git a/utilities/ovn-nbctl.8.xml b/utilities/ovn-nbctl.8.xml
index 88ebd13..e8572f7 100644
--- a/utilities/ovn-nbctl.8.xml
+++ b/utilities/ovn-nbctl.8.xml
@@ -483,6 +483,44 @@ 
 
     </dl>
 
+    <h1>Forwarding Group Commands</h1>
+
+    <dl>
+      <dt>[<code>--liveness</code>]<code>fwd-group-add</code> <var>group</var> <var>switch</var> <var>vip</var> <var>vmac</var> <var>ports</var></dt>
+      <dd>
+        <p>
+          Creates a new forwarding group named <var>group</var> as the name
+          with the provided <var>vip</var> and <var>vmac</var>. <var>vip</var>
+          should be a virtual IP address and <var>vmac</var> should be a
+          virtual MAC address to access the forwarding group. <var>ports</var>
+          are the logical switch port names that are put in the forwarding
+          group. Example for <var>ports</var> is lsp1 lsp2 ...
+          Traffic destined to virtual IP of the forwarding group will be load
+          balanced to all the child ports.
+        </p>
+        <p>
+          When <code>--liveness</code> is specified then child ports are
+          expected to be bound to external devices like routers. BFD should
+          be configured between hypervisors and the external devices.
+          The child port selection will become dependent on BFD status with
+          its external device.
+        </p>
+      </dd>
+
+      <dt>[<code>--if-exists</code>] <code>fwd-group-del</code> <var>group
+          </var></dt>
+      <dd>
+        Deletes <var>group</var>.  It is an error if <var>group</var> does
+        not exist, unless <code>--if-exists</code> is specified.
+      </dd>
+
+      <dt><code>fwd-group-list</code> [<var>switch</var>]</dt>
+      <dd>
+        Lists all existing forwarding groups, If <var>switch</var> is specified
+        then only the forwarding groups configured for <var>switch</var> will
+        be listed.
+      </dd>
+    </dl>
     <h1>Logical Router Commands</h1>
 
     <dl>
diff --git a/utilities/ovn-nbctl.c b/utilities/ovn-nbctl.c
index 46ba3a9..39f53da 100644
--- a/utilities/ovn-nbctl.c
+++ b/utilities/ovn-nbctl.c
@@ -648,6 +648,13 @@  Logical switch port commands:\n\
   lsp-get-dhcpv6-options PORT  get the dhcpv6 options for PORT\n\
   lsp-get-ls PORT           get the logical switch which the port belongs to\n\
 \n\
+Forwarding group commands:\n\
+  [--liveness]\n\
+  fwd-group-add GROUP SWITCH VIP VMAC PORTS...\n\
+                            add a forwarding group on SWITCH\n\
+  fwd-group-del GROUP       delete a forwarding group\n\
+  fwd-group-list [SWITCH]   print forwarding groups\n\
+\n\
 Logical router commands:\n\
   lr-add [ROUTER]           create a logical router named ROUTER\n\
   lr-del ROUTER             delete ROUTER and all its ports\n\
@@ -4720,6 +4727,244 @@  nbctl_lrp_get_redirect_type(struct ctl_context *ctx)
                   !redirect_type ? "overlay": redirect_type);
 }
 
+static const struct nbrec_forwarding_group *
+fwd_group_by_name_or_uuid(struct ctl_context *ctx, const char *id)
+{
+    const struct nbrec_forwarding_group *fwd_group = NULL;
+    struct uuid fwd_uuid;
+
+    bool is_uuid = uuid_from_string(&fwd_uuid, id);
+    if (is_uuid) {
+        fwd_group = nbrec_forwarding_group_get_for_uuid(ctx->idl, &fwd_uuid);
+    }
+
+    if (!fwd_group) {
+        NBREC_FORWARDING_GROUP_FOR_EACH(fwd_group, ctx->idl) {
+            if (!strcmp(fwd_group->name, id)) {
+                break;
+            }
+        }
+    }
+
+    return fwd_group;
+}
+
+static const struct nbrec_logical_switch *
+fwd_group_to_logical_switch(struct ctl_context *ctx,
+                            const struct nbrec_forwarding_group *fwd_group)
+{
+    if (!fwd_group) {
+        return NULL;
+    }
+
+    const struct nbrec_logical_switch_port *lsp;
+    char *error = lsp_by_name_or_uuid(ctx, fwd_group->child_port[0],
+                                      false, &lsp);
+    if (error) {
+        ctx->error = error;
+        return NULL;
+    }
+    if (!lsp) {
+        return NULL;
+    }
+
+    const struct nbrec_logical_switch *ls;
+    error = lsp_to_ls(ctx->idl, lsp, &ls);
+    if (error) {
+        ctx->error = error;
+        return NULL;
+    }
+
+    if (!ls) {
+        return NULL;
+    }
+
+    return ls;
+}
+
+static void
+nbctl_fwd_group_add(struct ctl_context *ctx)
+{
+    if (ctx->argc <= 5) {
+        ctl_error(ctx, "Usage : ovn-nbctl fwd-group-add group switch vip vmac "
+                  "child_ports...");
+        return;
+    }
+
+    /* Check if the forwarding group already exists */
+    const char *fwd_group_name = ctx->argv[1];
+    if (fwd_group_by_name_or_uuid(ctx, fwd_group_name)) {
+        ctl_error(ctx, "%s: a forwarding group by this name already exists",
+                  fwd_group_name);
+        return;
+    }
+
+    /* Check if the logical switch exists */
+    const char *ls_name = ctx->argv[2];
+    const struct nbrec_logical_switch *ls = NULL;
+    char *error = ls_by_name_or_uuid(ctx, ls_name, true, &ls);
+    if (error) {
+        ctx->error = error;
+        return;
+    }
+
+    /* Virtual IP for the group */
+    ovs_be32 ipv4 = 0;
+    const char *fwd_group_vip = ctx->argv[3];
+    if (!ip_parse(fwd_group_vip, &ipv4)) {
+        ctl_error(ctx, "invalid ip address %s", fwd_group_vip);
+        return;
+    }
+
+    /* Virtual MAC for the group */
+    const char *fwd_group_vmac = ctx->argv[4];
+    struct eth_addr ea;
+    if (!eth_addr_from_string(fwd_group_vmac, &ea)) {
+        ctl_error(ctx, "invalid mac address %s", fwd_group_vmac);
+        return;
+    }
+
+    /* Create the forwarding group */
+    struct nbrec_forwarding_group *fwd_group = NULL;
+    fwd_group = nbrec_forwarding_group_insert(ctx->txn);
+    nbrec_forwarding_group_set_name(fwd_group, fwd_group_name);
+    nbrec_forwarding_group_set_vip(fwd_group, fwd_group_vip);
+    nbrec_forwarding_group_set_vmac(fwd_group, fwd_group_vmac);
+
+    int n_child_port = ctx->argc - 5;
+    const char **child_port = (const char **)&ctx->argv[5];
+
+    /* Verify that child ports belong to the logical switch specified */
+    for (int i = 5; i < ctx->argc; ++i) {
+        const struct nbrec_logical_switch_port *lsp;
+        const char *lsp_name = ctx->argv[i];
+        error = lsp_by_name_or_uuid(ctx, lsp_name, false, &lsp);
+        if (error) {
+            ctx->error = error;
+            return;
+        }
+        if (lsp) {
+            error = lsp_to_ls(ctx->idl, lsp, &ls);
+            if (error) {
+                ctx->error = error;
+                return;
+            }
+            if (strcmp(ls->name, ls_name)) {
+                ctl_error(ctx, "%s: port already exists but in logical "
+                          "switch %s", lsp_name, ls->name);
+                return;
+            }
+        } else {
+            ctl_error(ctx, "%s: logical switch port does not exist", lsp_name);
+            return;
+        }
+    }
+    nbrec_forwarding_group_set_child_port(fwd_group, child_port, n_child_port);
+
+    /* Liveness option */
+    bool liveness = shash_find(&ctx->options, "--liveness") != NULL;
+    if (liveness) {
+      nbrec_forwarding_group_set_liveness(fwd_group, true);
+    }
+
+    struct nbrec_forwarding_group **new_fwd_groups =
+            xmalloc(sizeof(*new_fwd_groups) * (ls->n_forwarding_groups + 1));
+    memcpy(new_fwd_groups, ls->forwarding_groups,
+           sizeof *new_fwd_groups * ls->n_forwarding_groups);
+    new_fwd_groups[ls->n_forwarding_groups] = fwd_group;
+    nbrec_logical_switch_set_forwarding_groups(ls, new_fwd_groups,
+                                               (ls->n_forwarding_groups + 1));
+    free(new_fwd_groups);
+
+}
+
+static void
+nbctl_fwd_group_del(struct ctl_context *ctx)
+{
+    const char *id = ctx->argv[1];
+    const struct nbrec_forwarding_group *fwd_group = NULL;
+
+    fwd_group = fwd_group_by_name_or_uuid(ctx, id);
+    if (!fwd_group) {
+        return;
+    }
+
+    const struct nbrec_logical_switch *ls = NULL;
+    ls = fwd_group_to_logical_switch(ctx, fwd_group);
+    if (!ls) {
+      return;
+    }
+
+    for (int i = 0; i < ls->n_forwarding_groups; ++i) {
+        if (!strcmp(ls->forwarding_groups[i]->name, fwd_group->name)) {
+            struct nbrec_forwarding_group **new_fwd_groups =
+                xmemdup(ls->forwarding_groups,
+                        sizeof *new_fwd_groups * ls->n_forwarding_groups);
+            new_fwd_groups[i] =
+                ls->forwarding_groups[ls->n_forwarding_groups - 1];
+            nbrec_logical_switch_set_forwarding_groups(ls, new_fwd_groups,
+                (ls->n_forwarding_groups - 1));
+            free(new_fwd_groups);
+            nbrec_forwarding_group_delete(fwd_group);
+            return;
+        }
+    }
+}
+
+static void
+fwd_group_list_all(struct ctl_context *ctx, const char *ls_name)
+{
+    const struct nbrec_logical_switch *ls;
+    struct ds *s = &ctx->output;
+    const struct nbrec_forwarding_group *fwd_group = NULL;
+
+    if (ls_name) {
+        char *error = ls_by_name_or_uuid(ctx, ls_name, true, &ls);
+        if (error) {
+            ctx->error = error;
+            return;
+        }
+        if (!ls) {
+            ctl_error(
+                ctx, "%s: a logical switch with this name does not exist",
+                ls_name);
+            return;
+        }
+    }
+
+    ds_put_format(s, "%-16.16s%-14.16s%-16.7s%-22.21s%s\n",
+                  "FWD_GROUP", "LS", "VIP", "VMAC", "CHILD_PORTS");
+
+    NBREC_FORWARDING_GROUP_FOR_EACH(fwd_group, ctx->idl) {
+        ls = fwd_group_to_logical_switch(ctx, fwd_group);
+        if (!ls) {
+            continue;
+        }
+
+        if (ls_name && (strcmp(ls->name, ls_name))) {
+            continue;
+        }
+
+        ds_put_format(s, "%-16.16s%-14.18s%-15.16s%-9.18s     ",
+                      fwd_group->name, ls->name,
+                      fwd_group->vip, fwd_group->vmac);
+        for (int i = 0; i < fwd_group->n_child_port; ++i) {
+            ds_put_format(s, " %s", fwd_group->child_port[i]);
+        }
+        ds_put_char(s, '\n');
+    }
+}
+
+static void
+nbctl_fwd_group_list(struct ctl_context *ctx)
+{
+    if (ctx->argc == 1) {
+        fwd_group_list_all(ctx, NULL);
+    } else if (ctx->argc == 2) {
+        fwd_group_list_all(ctx, ctx->argv[1]);
+    }
+}
+
 
 struct ipv4_route {
     int priority;
@@ -5704,6 +5949,14 @@  static const struct ctl_command_syntax nbctl_commands[] = {
       nbctl_lsp_get_dhcpv6_options, NULL, "", RO },
     { "lsp-get-ls", 1, 1, "PORT", NULL, nbctl_lsp_get_ls, NULL, "", RO },
 
+    /* forwarding group commands. */
+    { "fwd-group-add", 4, INT_MAX, "SWITCH GROUP VIP VMAC PORT...",
+      NULL, nbctl_fwd_group_add, NULL, "--liveness", RW },
+    { "fwd-group-del", 1, 1, "GROUP", NULL, nbctl_fwd_group_del, NULL,
+      "--if-exists", RW },
+    { "fwd-group-list", 0, 1, "[GROUP]", NULL, nbctl_fwd_group_list, NULL,
+      "", RO },
+
     /* logical router commands. */
     { "lr-add", 0, 1, "[ROUTER]", NULL, nbctl_lr_add, NULL,
       "--may-exist,--add-duplicate", RW },