diff mbox

[ovs-dev] ovn: SFC Patch V3

Message ID 1493063784-36666-1-git-send-email-jmcdowall@paloaltonetworks.com
State Changes Requested
Headers show

Commit Message

John McDowall April 24, 2017, 7:56 p.m. UTC
From: John McDowall <jmcdowall@paloaltonetworks.com>


Fixed changes from Mickey's last review.

Changes

1) Fixed re-circulation rules
2) Fixed match statement - match is only applied to beginnning of chain in
   each direction.
3) Fixed array length of chain of VNFs. I have tested thi sup to three VNFs
   in a chain and it looks like it works in both directions. 

Areas to review

1) The logic now supports hair-pinnign the flow back to the original source to
   ensure that the MAC learnign problem is addressed. I tested this using 
   ovn-trace - any better testing that I should do?

Current todo list

1) I have standalone tests need to add tests to ovs/ovn framework.
2) Load-balancing support for port-pair-groups
3) Publish more detailed examples.
4) Submit suggestions to change and shorted the CLI names.

Simple example using ovn-trace

#!/bin/sh
#
clear
ovn-nbctl ls-add swt1

ovn-nbctl lsp-add swt1 swt1-appc
ovn-nbctl lsp-add swt1 swt1-apps
ovn-nbctl lsp-add swt1 swt1-vnfp1
ovn-nbctl lsp-add swt1 swt1-vnfp2

ovn-nbctl lsp-set-addresses swt1-appc "00:00:00:00:00:01 192.168.33.1"
ovn-nbctl lsp-set-addresses swt1-apps "00:00:00:00:00:02 192.168.33.2"
ovn-nbctl lsp-set-addresses swt1-vnfp1 00:00:00:00:00:03
ovn-nbctl lsp-set-addresses swt1-vnfp2 00:00:00:00:00:04
#
# Configure Service chain
#
ovn-nbctl lsp-pair-add swt1 swt1-vnfp1 swt1-vnfp2 pp1
ovn-nbctl lsp-chain-add swt1 pc1
ovn-nbctl lsp-pair-group-add pc1 ppg1
ovn-nbctl lsp-pair-group-add-port-pair ppg1 pp1
ovn-nbctl lsp-chain-classifier-add swt1 pc1 swt1-appc "entry-lport" "bi-directional" pcc1
#
ovn-sbctl dump-flows
#
# Run trace command
printf "\n---------Flow 1 -------------\n\n"
ovn-trace --detailed  swt1 'inport == "swt1-appc" && eth.src == 00:00:00:00:00:01 && eth.dst == 00:00:00:00:00:02'
printf "\n---------Flow 2 -------------\n\n"
ovn-trace --detailed  swt1 'inport == "swt1-vnfp1" && eth.src == 00:00:00:00:00:01 && eth.dst == 00:00:00:00:00:02'
printf "\n---------Flow 3 -------------\n\n"
ovn-trace --detailed  swt1 'inport == "swt1-apps" && eth.dst == 00:00:00:00:00:01 && eth.src == 00:00:00:00:00:02'
printf "\n---------Flow 4 -------------\n\n"
ovn-trace --detailed  swt1 'inport == "swt1-vnfp2" && eth.dst == 00:00:00:00:00:01 && eth.src == 00:00:00:00:00:02'
#
# Cleanup
#
ovn-nbctl lsp-chain-classifier-del pcc1
ovn-nbctl lsp-pair-group-del ppg1
ovn-nbctl lsp-chain-del pc1
ovn-nbctl lsp-pair-del pp1
ovn-nbctl ls-del swt1

Reported at: https://mail.openvswitch.org/pipermail/ovs-discuss/2016-March/040381.html
Reported at: https://mail.openvswitch.org/pipermail/ovs-discuss/2016-May/041359.html

Signed-off-by: John McDowall <jmcdowall@paloaltonetworks.com>
Signed-off-by: Flavio Fernandes <flavio@flaviof.com>
Co-authored-by: Flavio Fernandes <flavio@flaviof.com>
---
 ovn/northd/ovn-northd.8.xml   |   69 ++-
 ovn/northd/ovn-northd.c       |  382 ++++++++++++-
 ovn/ovn-architecture.7.xml    |   91 ++++
 ovn/ovn-nb.ovsschema          |   87 ++-
 ovn/ovn-nb.xml                |  188 ++++++-
 ovn/utilities/ovn-nbctl.8.xml |  231 ++++++++
 ovn/utilities/ovn-nbctl.c     | 1208 +++++++++++++++++++++++++++++++++++++++++
 7 files changed, 2227 insertions(+), 29 deletions(-)

Comments

Mickey Spiegel April 25, 2017, 1:39 a.m. UTC | #1
On Mon, Apr 24, 2017 at 12:56 PM, <jmcdowall@paloaltonetworks.com> wrote:

> From: John McDowall <jmcdowall@paloaltonetworks.com>
>
>
> Fixed changes from Mickey's last review.
>
> Changes
>
> 1) Fixed re-circulation rules
>

Still a few modifications required. See comments inline. I just typed some
stuff out, have not run, built, or tested anything.


> 2) Fixed match statement - match is only applied to beginnning of chain in
>    each direction.
> 3) Fixed array length of chain of VNFs. I have tested thi sup to three VNFs
>    in a chain and it looks like it works in both directions.
>
> Areas to review
>
> 1) The logic now supports hair-pinnign the flow back to the original
> source to
>    ensure that the MAC learnign problem is addressed. I tested this using
>    ovn-trace - any better testing that I should do?
>
> Current todo list
>
> 1) I have standalone tests need to add tests to ovs/ovn framework.
> 2) Load-balancing support for port-pair-groups
> 3) Publish more detailed examples.
> 4) Submit suggestions to change and shorted the CLI names.
>
> Simple example using ovn-trace
>
> #!/bin/sh
> #
> clear
> ovn-nbctl ls-add swt1
>
> ovn-nbctl lsp-add swt1 swt1-appc
> ovn-nbctl lsp-add swt1 swt1-apps
> ovn-nbctl lsp-add swt1 swt1-vnfp1
> ovn-nbctl lsp-add swt1 swt1-vnfp2
>
> ovn-nbctl lsp-set-addresses swt1-appc "00:00:00:00:00:01 192.168.33.1"
> ovn-nbctl lsp-set-addresses swt1-apps "00:00:00:00:00:02 192.168.33.2"
> ovn-nbctl lsp-set-addresses swt1-vnfp1 00:00:00:00:00:03
> ovn-nbctl lsp-set-addresses swt1-vnfp2 00:00:00:00:00:04
> #
> # Configure Service chain
> #
> ovn-nbctl lsp-pair-add swt1 swt1-vnfp1 swt1-vnfp2 pp1
> ovn-nbctl lsp-chain-add swt1 pc1
> ovn-nbctl lsp-pair-group-add pc1 ppg1
> ovn-nbctl lsp-pair-group-add-port-pair ppg1 pp1
> ovn-nbctl lsp-chain-classifier-add swt1 pc1 swt1-appc "entry-lport"
> "bi-directional" pcc1
> #
> ovn-sbctl dump-flows
> #
> # Run trace command
> printf "\n---------Flow 1 -------------\n\n"
> ovn-trace --detailed  swt1 'inport == "swt1-appc" && eth.src ==
> 00:00:00:00:00:01 && eth.dst == 00:00:00:00:00:02'
> printf "\n---------Flow 2 -------------\n\n"
> ovn-trace --detailed  swt1 'inport == "swt1-vnfp1" && eth.src ==
> 00:00:00:00:00:01 && eth.dst == 00:00:00:00:00:02'
> printf "\n---------Flow 3 -------------\n\n"
> ovn-trace --detailed  swt1 'inport == "swt1-apps" && eth.dst ==
> 00:00:00:00:00:01 && eth.src == 00:00:00:00:00:02'
> printf "\n---------Flow 4 -------------\n\n"
> ovn-trace --detailed  swt1 'inport == "swt1-vnfp2" && eth.dst ==
> 00:00:00:00:00:01 && eth.src == 00:00:00:00:00:02'
> #
> # Cleanup
> #
> ovn-nbctl lsp-chain-classifier-del pcc1
> ovn-nbctl lsp-pair-group-del ppg1
> ovn-nbctl lsp-chain-del pc1
> ovn-nbctl lsp-pair-del pp1
> ovn-nbctl ls-del swt1
>
> Reported at: https://mail.openvswitch.org/pipermail/ovs-discuss/2016-
> March/040381.html
> Reported at: https://mail.openvswitch.org/pipermail/ovs-discuss/2016-
> May/041359.html
>
> Signed-off-by: John McDowall <jmcdowall@paloaltonetworks.com>
> Signed-off-by: Flavio Fernandes <flavio@flaviof.com>
> Co-authored-by: Flavio Fernandes <flavio@flaviof.com>
> ---
>  ovn/northd/ovn-northd.8.xml   |   69 ++-
>  ovn/northd/ovn-northd.c       |  382 ++++++++++++-
>  ovn/ovn-architecture.7.xml    |   91 ++++
>  ovn/ovn-nb.ovsschema          |   87 ++-
>  ovn/ovn-nb.xml                |  188 ++++++-
>  ovn/utilities/ovn-nbctl.8.xml |  231 ++++++++
>  ovn/utilities/ovn-nbctl.c     | 1208 ++++++++++++++++++++++++++++++
> +++++++++++
>  7 files changed, 2227 insertions(+), 29 deletions(-)
>
>
<snip>


>
> diff --git ovn/northd/ovn-northd.c ovn/northd/ovn-northd.c
> index d0a5ba2..090f768 100644
> --- ovn/northd/ovn-northd.c
> +++ ovn/northd/ovn-northd.c
> @@ -106,13 +106,14 @@ enum ovn_stage {
>      PIPELINE_STAGE(SWITCH, IN,  PRE_LB,         4, "ls_in_pre_lb")
> \
>      PIPELINE_STAGE(SWITCH, IN,  PRE_STATEFUL,   5, "ls_in_pre_stateful")
> \
>      PIPELINE_STAGE(SWITCH, IN,  ACL,            6, "ls_in_acl")
>  \
> -    PIPELINE_STAGE(SWITCH, IN,  QOS_MARK,       7, "ls_in_qos_mark")
> \
> -    PIPELINE_STAGE(SWITCH, IN,  LB,             8, "ls_in_lb")
> \
> -    PIPELINE_STAGE(SWITCH, IN,  STATEFUL,       9, "ls_in_stateful")
> \
> -    PIPELINE_STAGE(SWITCH, IN,  ARP_ND_RSP,    10, "ls_in_arp_rsp")
>  \
> -    PIPELINE_STAGE(SWITCH, IN,  DHCP_OPTIONS,  11, "ls_in_dhcp_options")
> \
> -    PIPELINE_STAGE(SWITCH, IN,  DHCP_RESPONSE, 12, "ls_in_dhcp_response")
> \
> -    PIPELINE_STAGE(SWITCH, IN,  L2_LKUP,       13, "ls_in_l2_lkup")
>  \
> +    PIPELINE_STAGE(SWITCH, IN,  CHAIN,          7, "ls_in_chain")        \
> +    PIPELINE_STAGE(SWITCH, IN,  QOS_MARK,       8, "ls_in_qos_mark")    \
> +    PIPELINE_STAGE(SWITCH, IN,  LB,             9, "ls_in_lb")
> \
> +    PIPELINE_STAGE(SWITCH, IN,  STATEFUL,      10, "ls_in_stateful")
> \
> +    PIPELINE_STAGE(SWITCH, IN,  ARP_ND_RSP,    11, "ls_in_arp_rsp")
>  \
> +    PIPELINE_STAGE(SWITCH, IN,  DHCP_OPTIONS,  12, "ls_in_dhcp_options")
> \
> +    PIPELINE_STAGE(SWITCH, IN,  DHCP_RESPONSE, 13, "ls_in_dhcp_response")
> \
> +    PIPELINE_STAGE(SWITCH, IN,  L2_LKUP,       14, "ls_in_l2_lkup")
>  \
>                                                                        \
>      /* Logical switch egress stages. */                               \


You need a stage in the egress pipeline to do SFC egress loopback:

    PIPELINE_STAGE(SWITCH, OUT, SFC_LOOPBACK, 0, "ls_out_sfc_loopback")  \
    PIPELINE_STAGE(SWITCH, OUT, PRE_LB,       1, "ls_out_pre_lb")     \
    ...

     PIPELINE_STAGE(SWITCH, OUT, PRE_LB,       0, "ls_out_pre_lb")     \
> @@ -160,7 +161,6 @@ enum ovn_stage {
>  #define REGBIT_CONNTRACK_COMMIT "reg0[1]"
>  #define REGBIT_CONNTRACK_NAT    "reg0[2]"
>  #define REGBIT_DHCP_OPTS_RESULT "reg0[3]"
> -
>

Whitespace change should be removed.


>  /* Register definitions for switches and routers. */
>  #define REGBIT_NAT_REDIRECT     "reg9[0]"
>  /* Indicate that this packet has been recirculated using egress
> @@ -3014,6 +3014,348 @@ build_acls(struct ovn_datapath *od, struct hmap
> *lflows)
>      }
>  }
>
> +static int
> +cmp_port_pair_groups(const void *ppg1_, const void *ppg2_)
> +{
> +    const struct nbrec_logical_port_pair_group *const *ppg1p = ppg1_;
> +    const struct nbrec_logical_port_pair_group *const *ppg2p = ppg2_;
> +    const struct nbrec_logical_port_pair_group *ppg1 = *ppg1p;
> +    const struct nbrec_logical_port_pair_group *ppg2 = *ppg2p;
> +
> +
> +    return ( (int)ppg1->sortkey - (int)ppg2->sortkey);
> +}
> +
> +static void
> +build_chain(struct ovn_datapath *od, struct hmap *lflows, struct hmap
> *ports)
> +{
> +   /*
> +    * TODO Items
> +    *     * IPV6 support
> +    *     * Load balancing support
> +    *     * Bidirectional parameter support
> +    *     * Support modes of VNF (BitW, L2, L3)
> +    *     * Remove port-security on VNF Ports (if set by CMS)
> +    *     * Add some state to allow match that does not require 'inport'
> +    *     * Support visiting the same VNF more than once
> +    *     * Unit tests!
> +    */
> +    const uint16_t ingress_inner_priority = 150;
> +    const uint16_t ingress_outer_priority = 100;
> +    const uint16_t egress_inner_priority = 150;
> +    const uint16_t egress_outer_priority = 100;
> +
> +    struct ovn_port **input_port_array = NULL;
> +    struct ovn_port **output_port_array = NULL;
> +
> +    struct ovn_port *dst_port = NULL;
> +    struct ovn_port *src_port = NULL;
> +
> +    struct nbrec_logical_port_chain *lpc;
> +    struct nbrec_logical_port_pair_group *lppg;
> +    struct nbrec_logical_port_pair *lpp;
> +    struct nbrec_logical_port_chain_classifier *lcc;
> +
> +    char *lcc_match = NULL;
> +    char *lcc_action = NULL;
> +    struct ovn_port *traffic_port;
> +    unsigned int chain_direction = 2;
> +    unsigned int chain_path = 2;
> +    char * chain_match = NULL;
> +
> +    /* Ingress table ls_in_chain: default to passing through to the next
> table
> +     * (priority 0)
> +     */
> +    ovn_lflow_add(lflows, od, S_SWITCH_IN_CHAIN, 0, "1", "next;");
>

At some point also need:

    ovn_lflow_add(lflows, od, S_SWITCH_OUT_SFC_LOOPBACK, 0, "1", "next;");


> +
> +    /* No port chains defined therefore, no further work to do here. */
> +    if (!od->nbs->port_chain_classifiers) {
> +        return;
> +    }
> +    /* Iterate through all the port-chains defined for this datapath. */
> +    for (size_t i = 0; i < od->nbs->n_port_chain_classifiers; i++) {
> +        lcc = od->nbs->port_chain_classifiers[i];
> +        /* Get the parameters from the classifier */
> +        lpc = lcc->chain;
> +        //traffic_port = lcc->port;
> +        traffic_port =  ovn_port_find(ports,lcc->port->name);
> +        if (traffic_port == NULL) {
> +            static struct vlog_rate_limit rl =
> +                        VLOG_RATE_LIMIT_INIT(1, 1);
> +            VLOG_WARN_RL(&rl,
> +                        "Traffic port %s does not exist\n",
> +                              lcc->port->name);
> +            break;
> +        }
> +        /*
> +        * Check chain has one or more groups
> +        */
> +        if (lpc->n_port_pair_groups == 0) {
> +            static struct vlog_rate_limit rl =
> +                        VLOG_RATE_LIMIT_INIT(1, 1);
> +            VLOG_WARN_RL(&rl,
> +                "SFC Chain: %s used in classifier: %s does not not "
> +                 "have any port pair groups\n",
> +                              lcc->chain->name, lcc->name);
> +            break;
> +
> +        }
> +        /* TODO Check port exists. */
> +        struct eth_addr traffic_logical_port_ea;
> +        ovs_be32 traffic_logical_port_ip;
> +        ovs_scan(traffic_port->nbsp->addresses[0],
> +                    ETH_ADDR_SCAN_FMT" "IP_SCAN_FMT,
> +                    ETH_ADDR_SCAN_ARGS(traffic_logical_port_ea),
> +                    IP_SCAN_ARGS(&traffic_logical_port_ip));
> +        /* Set the port to use as source or destination. */
> +        if (strcmp(lcc->direction,"entry-lport")==0) {
> +            chain_path = 0;
> +        } else {
> +            chain_path = 1;
> +        }
> +        /* Set the direction of the port-chain. */
> +        if (strcmp(lcc->path,"uni-directional") == 0) {
> +            chain_direction = 0;
> +        } else {
> +            chain_direction = 1;
> +        }
> +        /* Set the match parameters. */
> +        chain_match = lcc->match;
> +        /*
> +         * Allocate space for port-pairs + 1. The Extra 1 represents the
> +         * final hop to reach desired destination.
> +         * TODO: We are going to allocate enough space to hold all the
> hops:
> +         *  1 x portGroups + 1. This needs
> +         *  to enhanced to: SUM(port pairs of each port group) + 1
> +         */
> +        input_port_array = xmalloc(sizeof *src_port *
> +                                   lpc->n_port_pair_groups);
> +        output_port_array = xmalloc(sizeof *dst_port *
> +                                  (lpc->n_port_pair_groups));
> +        /* Copy port groups from chain and sort them according to
> sortkey.*/
> +        struct nbrec_logical_port_pair_group **port_pair_groups =
> +                                 xmemdup(lpc->port_pair_groups,
> +                          sizeof *port_pair_groups *
> lpc->n_port_pair_groups);
> +        if (lpc->n_port_pair_groups > 1) {
> +            qsort(port_pair_groups, lpc->n_port_pair_groups,
> +              sizeof *port_pair_groups, cmp_port_pair_groups);
> +        }
> +        /* For each port-pair-group in a port chain pull out the
> port-pairs.*/
> +        for (size_t j = 0; j < lpc->n_port_pair_groups; j++) {
> +            lppg = port_pair_groups[j];
> +            for (size_t k = 0; k < lppg->n_port_pairs; k++) {
> +                 /* TODO: Need to add load balancing logic when LB becomes
> +                 * available. Until LB is available just take the first
> +                 * PP in the PPG. */
> +                if (k > 0) {
> +                    static struct vlog_rate_limit rl =
> +                        VLOG_RATE_LIMIT_INIT(1, 1);
> +                    VLOG_WARN_RL(&rl,
> +                        "Currently lacking support for more than \
> +                            one port-pair %"PRIuSIZE"\n",
> +                              lppg->n_port_pairs);
> +                    break;
> +                }
> +                lpp = lppg->port_pairs[k];
> +                input_port_array[j] = lpp->inport ? ovn_port_find(ports,
> +                                       lpp->inport->name) : NULL;
> +                output_port_array[j] = lpp->outport ? ovn_port_find(ports,
> +                                        lpp->outport->name) : NULL;
> +            }
> +        /* At this point we need to determine the final hop port to add to
> +         * the chain. This defines the complete path for packets through
> +         * the chain. */
> +        }
>

I find the change in indentation below confusing.


> +    /*
> +    * Insert the lowest priorty rule dest is src-logical-port. These are
> the
> +    * entry points into the chain in either direction. The match statement
> +    * is used to filter the entry port to provide higher granularity of
> +    * filtering.
> +    */
> +    if (chain_path == 1) { /* Path starting from entry port */
> +        if (strcmp(chain_match,"")!=0) {
> +           lcc_match =  xasprintf(
> +            "eth.src == "ETH_ADDR_FMT" && %s",
> +             ETH_ADDR_ARGS(traffic_logical_port_ea), chain_match);
> +        } else {
> +           lcc_match =  xasprintf(
> +            "eth.src == "ETH_ADDR_FMT,
> +             ETH_ADDR_ARGS(traffic_logical_port_ea));
> +        }
> +           lcc_action = xasprintf("outport = %s; output;",
> +                                input_port_array[0]->json_key);
> +    } else {  /* Path going to the entry port */
> +        if (strcmp(chain_match,"")!=0) {
> +           lcc_match =  xasprintf(
> +            "eth.dst == "ETH_ADDR_FMT" && %s",
> +             ETH_ADDR_ARGS(traffic_logical_port_ea), chain_match);
> +        } else {
> +           lcc_match =  xasprintf(
> +            "eth.dst == "ETH_ADDR_FMT,
> +             ETH_ADDR_ARGS(traffic_logical_port_ea));
> +        }
> +           lcc_action = xasprintf("outport = %s; output;",
> +                    output_port_array[lpc->n_port_
> pair_groups-1]->json_key);
> +    }
> +    ovn_lflow_add(lflows, od, S_SWITCH_IN_CHAIN, ingress_outer_priority,
> +                       lcc_match, lcc_action);
> +    free(lcc_match);
> +    free(lcc_action);
> +    /*
> +    * For last VNF send the flow back to teh original chassis and exit
> from
> +    * there.
> +    */
> +    if (chain_path == 1) { /* Path starting from entry port */
> +           lcc_match = xasprintf(
> +                    "eth.src == "ETH_ADDR_FMT" && inport == %s",
> +                     ETH_ADDR_ARGS(traffic_logical_port_ea),
> +                     output_port_array[lpc->n_port_
> pair_groups-1]->json_key);
> +    } else { /* Path starting from destination port. */
> +            lcc_match = xasprintf(
> +                    "eth.dst == "ETH_ADDR_FMT" && inport == %s",
> +                     ETH_ADDR_ARGS(traffic_logical_port_ea),
> +                     input_port_array[0]->json_key);
> +    }
>

    lcc_action = xasprintf("outport = %s; output;", traffic_port->json_key);
    ovn_lflow_add(lflows, od, S_SWITCH_IN_CHAIN, ingress_inner_priority,
                        lcc_match, lcc_action);
    free(lcc_match);
    free(lcc_action);

Temporarily ignoring cases where there are multiple port_chain_classifiers
with the same traffic_port. You should do something to avoid duplicate
flows in this case.

    /* SFC egress loopback on traffic_port. */

    lcc_match = xasprintf(
                    "eth.src == "ETH_ADDR_FMT" && outport == %s",
                    ETH_ADDR_ARGS(traffic_logical_port_ea),
                    traffic_port->json_key);

+             //lcc_action = xasprintf("next;");
> +             //lcc_action = xasprintf("flags.loopback = 1; "
> +             //    REGBIT_CHAIN_LOOPBACK" = 1;"
> +             //   "next(pipeline=ingress, table=0);");
> +    struct ds actions = DS_EMPTY_INITIALIZER;
> +    ds_put_format(&actions, "clone { ct_clear; "
> +                              "inport = outport; outport = \"\"; "
> +                              "flags = 0; flags.loopback = 1; ");


Probably should not set flags.loopback, see comments
below.

+    for (int ii = 0; ii < MFF_N_LOG_REGS; ii++) {
> +            ds_put_format(&actions, "reg%d = 0; ", ii);
> +    }
> +    ds_put_format(&actions, "next(pipeline=ingress, table=%d); };",
> +                            ovn_stage_get_table(S_SWITCH_IN_CHAIN) + 1);
> +    ovn_lflow_add(lflows, od, S_SWITCH_IN_CHAIN, ingress_inner_priority,
> +                        lcc_match, ds_cstr(&actions));
>

Replace the line above by:

    ovn_lflow_add(lflows, od, S_SWITCH_OUT_SFC_LOOPBACK, 100,
                        lcc_match, ds_cstr(&actions));


> +    free(lcc_match);
> +        //free(lcc_action);
> +    ds_destroy(&actions);
> +
>

Some thought needs to be put behind loop avoidance, for
example when eth.dst == eth.src == traffic_logical_port_ea.
This may be an issue since we do not have any definitive
method to distinguish packets returning from chains, from
packets that were switched to traffic_port according to
normal L2 processing.

One option is to add higher priority ( > 100) flows. This is
messy, since there is already a fair amount of code to
write flows to send traffic to this port in
S_SWITCH_IN_L2_LKUP. This needs some thought to
determine the match conditions.
The action is simple, just "next;".

Perhaps more promising is not to set flags.loopback to 1.
This would cause any packets with
eth.src == traffic_logical_port_ea that are switched to
traffic_port to go back through traffic_port's ingress
pipeline, then be dropped if outport == traffic_port at the
end of traffic_port's ingress pipeline.
However, if anything after S_SWITCH_IN_CHAIN sets
flags.loopback, that might not be enough.

There seems to be another issue, if anything before
S_SWITCH_IN_CHAIN sets flags.loopback, that will not be
remembered when the packet comes back, so the packet
will not be processed properly.

Right now it looks like flags.loopback is only set for a
logical switch in tables 11, 12, 13 which are after
S_SWITCH_IN_CHAIN. So, at the moment, remembering
flags.loopback is not an issue. These flows generate
ARP and DCHP responses, in all cases changing
eth.src, so the packets will not loop indefinitely even
though these rules set flags.loopback. There are
already flows to avoid generating ARP responses for
requests on the port that owns the address, which is
the dangerous case for SFC loops.

In summary, it looks like leaving flags.loopback at 0
should work for now. The question is what would
happen if more cases are added in the future that
set flags.loopback in a logical switch.

Mickey

+    /*
> +    * Loop over all VNFs and create flow rules for each
> +    * Only get here when there is more than one VNF in the chain.
> +    */
> +    for (size_t j = 1; j < lpc->n_port_pair_groups; j++) {
> +
> +        /* Apply inner rules flows */
> +        if (chain_path == 1) { /* Path starting from entry port */
> +            lcc_match = xasprintf(
> +                    "eth.src == "ETH_ADDR_FMT" && inport == %s",
> +                                 ETH_ADDR_ARGS(traffic_logical_port_ea),
> +                                 output_port_array[j-1]->json_key);
> +            lcc_action = xasprintf("outport = %s; output;",
> +                                input_port_array[j]->json_key);
> +        } else { /* Path going to entry port. */
> +            lcc_match = xasprintf(
> +                    "eth.dst == "ETH_ADDR_FMT" && inport == %s",
> +                    ETH_ADDR_ARGS(traffic_logical_port_ea),
> +                    input_port_array[lpc->n_port_
> pair_groups-j]->json_key);
> +             lcc_action = xasprintf("outport = %s; output;",
> +                  output_port_array[lpc->n_port_
> pair_groups-(j+1)]->json_key);
> +        }
> +        ovn_lflow_add(lflows, od, S_SWITCH_IN_CHAIN,
> ingress_inner_priority,
> +                        lcc_match, lcc_action);
> +        free(lcc_match);
> +        free(lcc_action);
> +    }
> +    /* bi-directional chain (Response)*/
> +    if (chain_direction == 1) {
> +        /*
> +         * Insert the lowest priorty rule dest is src-logical-port
> +         */
> +        if (chain_path == 1) { /* Path from source port. */
> +            if (strcmp(chain_match,"")!=0) {
> +                 lcc_match =  xasprintf("eth.dst == "ETH_ADDR_FMT" && %s",
> +                        ETH_ADDR_ARGS(traffic_logical_port_ea),
> chain_match);
> +            } else { /* Path to source port */
> +
> +                 lcc_match =  xasprintf("eth.dst == "ETH_ADDR_FMT,
> +                                    ETH_ADDR_ARGS(traffic_logical_
> port_ea));
> +            }
> +            lcc_action = xasprintf("outport = %s; output;",
> +                    output_port_array[lpc->n_port_
> pair_groups-1]->json_key);
> +        } else { /* Path from destination port. */
> +           if (strcmp(chain_match,"")!=0) {
> +                lcc_match =  xasprintf("eth.src == "ETH_ADDR_FMT" && %s",
> +                        ETH_ADDR_ARGS(traffic_logical_port_ea),
> chain_match);
> +            } else {
> +                lcc_match =  xasprintf("eth.src == "ETH_ADDR_FMT,
> +                        ETH_ADDR_ARGS(traffic_logical_port_ea));
> +            }
> +            lcc_action = xasprintf("outport = %s; output;",
> +                    input_port_array[0]->json_key);
> +        }
> +        ovn_lflow_add(lflows, od, S_SWITCH_IN_CHAIN,
> egress_outer_priority,
> +                       lcc_match, lcc_action);
> +        free(lcc_match);
> +        free(lcc_action);
> +        /*
> +        * End Entry Flow to classification chain entry point.
> +        */
> +        /* Apply last rule to exit from chain */
> +        if (chain_path == 1) { /* Path from source port. */
> +             lcc_match = xasprintf(
> +                    "eth.dst == "ETH_ADDR_FMT" && inport == %s",
> +                                 ETH_ADDR_ARGS(traffic_logical_port_ea),
> +                                 input_port_array[0]->json_key);
> +
> +        } else { /* Path to source port. */
> +             lcc_match = xasprintf(
> +                "eth.src == "ETH_ADDR_FMT" && inport == %s",
> +                    ETH_ADDR_ARGS(traffic_logical_port_ea),
> +                    output_port_array[lpc->n_port_
> pair_groups-1]->json_key);
> +        }
> +        struct ds actions = DS_EMPTY_INITIALIZER;
> +        ds_put_format(&actions,
> +                              "clone { ct_clear; "
> +                              "inport = outport; outport = \"\"; "
> +                              "flags = 0; flags.loopback = 1; ");
> +        for (int ii = 0; ii < MFF_N_LOG_REGS; ii++) {
> +                    ds_put_format(&actions, "reg%d = 0; ", ii);
> +        }
> +        ds_put_format(&actions, "next(pipeline=ingress, table=%d); };",
> +                              ovn_stage_get_table(S_SWITCH_IN_CHAIN) +
> 1);
> +        ovn_lflow_add(lflows, od, S_SWITCH_IN_CHAIN,
> +                        egress_inner_priority, lcc_match,
> ds_cstr(&actions));
> +        ds_destroy(&actions);
> +        free(lcc_match);
> +
> +        for (int j = 1; j< lpc->n_port_pair_groups; j++) {
> +
> +            /* Completed first catch all rule for this port-chain. */
> +
> +            /* Apply inner rules flows */
> +            if (chain_path == 1) { /* Path from source port. */
> +                lcc_match = xasprintf(
> +                    "eth.dst == "ETH_ADDR_FMT" && inport == %s",
> +                    ETH_ADDR_ARGS(traffic_logical_port_ea),
> +                    input_port_array[lpc->n_port_
> pair_groups-j]->json_key);
> +                 lcc_action = xasprintf("outport = %s; output;",
> +                  output_port_array[lpc->n_port_
> pair_groups-(j+1)]->json_key);
> +
> +            } else { /* Path to source port. */
> +                lcc_match = xasprintf(
> +                    "eth.src == "ETH_ADDR_FMT" && inport == %s",
> +                        ETH_ADDR_ARGS(traffic_logical_port_ea),
> +                        output_port_array[j-1]->json_key);
> +                 lcc_action = xasprintf("outport = %s; output;",
> +                        input_port_array[j]->json_key);
> +            }
> +            ovn_lflow_add(lflows, od, S_SWITCH_IN_CHAIN,
> +                          egress_inner_priority, lcc_match, lcc_action);
> +            free(lcc_match);
> +            free(lcc_action);
> +        }
> +    }
> +    free(input_port_array);
> +    free(output_port_array);
> +    free(port_pair_groups);
> +    }
> +}
>  static void
>  build_qos(struct ovn_datapath *od, struct hmap *lflows) {
>      ovn_lflow_add(lflows, od, S_SWITCH_IN_QOS_MARK, 0, "1", "next;");
> @@ -3142,7 +3484,7 @@ build_lswitch_flows(struct hmap *datapaths, struct
> hmap *ports,
>      struct ds actions = DS_EMPTY_INITIALIZER;
>
>      /* Build pre-ACL and ACL tables for both ingress and egress.
> -     * Ingress tables 3 through 9.  Egress tables 0 through 6. */
> +     * Ingress tables 3 through 10.  Egress tables 0 through 6. */
>      struct ovn_datapath *od;
>      HMAP_FOR_EACH (od, key_node, datapaths) {
>          if (!od->nbs) {
> @@ -3153,6 +3495,7 @@ build_lswitch_flows(struct hmap *datapaths, struct
> hmap *ports,
>          build_pre_lb(od, lflows);
>          build_pre_stateful(od, lflows);
>          build_acls(od, lflows);
> +        build_chain(od, lflows, ports);
>          build_qos(od, lflows);
>          build_lb(od, lflows);
>          build_stateful(od, lflows);
> @@ -3225,9 +3568,9 @@ build_lswitch_flows(struct hmap *datapaths, struct
> hmap *ports,
>          ovn_lflow_add(lflows, od, S_SWITCH_IN_PORT_SEC_IP, 0, "1",
> "next;");
>      }
>
> -    /* Ingress table 10: ARP/ND responder, skip requests coming from
> localnet
> -     * and vtep ports. (priority 100); see ovn-northd.8.xml for the
> -     * rationale. */
> +    /* Ingress table 11: ARP/ND responder, skip requests coming from
> localnet
> +     * ports. (priority 100). */
> +
>      HMAP_FOR_EACH (op, key_node, ports) {
>          if (!op->nbsp) {
>              continue;
> @@ -3242,7 +3585,7 @@ build_lswitch_flows(struct hmap *datapaths, struct
> hmap *ports,
>          }
>      }
>
> -    /* Ingress table 10: ARP/ND responder, reply for known IPs.
> +    /* Ingress table 11: ARP/ND responder, reply for known IPs.
>       * (priority 50). */
>      HMAP_FOR_EACH (op, key_node, ports) {
>          if (!op->nbsp) {
> @@ -3335,7 +3678,7 @@ build_lswitch_flows(struct hmap *datapaths, struct
> hmap *ports,
>          }
>      }
>
> -    /* Ingress table 10: ARP/ND responder, by default goto next.
> +    /* Ingress table 11: ARP/ND responder, by default goto next.
>       * (priority 0)*/
>      HMAP_FOR_EACH (od, key_node, datapaths) {
>          if (!od->nbs) {
> @@ -3345,7 +3688,7 @@ build_lswitch_flows(struct hmap *datapaths, struct
> hmap *ports,
>          ovn_lflow_add(lflows, od, S_SWITCH_IN_ARP_ND_RSP, 0, "1",
> "next;");
>      }
>
> -    /* Logical switch ingress table 11 and 12: DHCP options and response
> +    /* Logical switch ingress table 12 and 13: DHCP options and response
>           * priority 100 flows. */
>      HMAP_FOR_EACH (op, key_node, ports) {
>          if (!op->nbsp) {
> @@ -3449,7 +3792,7 @@ build_lswitch_flows(struct hmap *datapaths, struct
> hmap *ports,
>          }
>      }
>
> -    /* Ingress table 11 and 12: DHCP options and response, by default
> goto next.
> +    /* Ingress table 12 and 13: DHCP options and response, by default
> goto next.
>       * (priority 0). */
>
>      HMAP_FOR_EACH (od, key_node, datapaths) {
> @@ -3461,7 +3804,8 @@ build_lswitch_flows(struct hmap *datapaths, struct
> hmap *ports,
>          ovn_lflow_add(lflows, od, S_SWITCH_IN_DHCP_RESPONSE, 0, "1",
> "next;");
>      }
>
> -    /* Ingress table 13: Destination lookup, broadcast and multicast
> handling
> +
> +    /* Ingress table 14: Destination lookup, broadcast and multicast
> handling
>       * (priority 100). */
>      HMAP_FOR_EACH (op, key_node, ports) {
>          if (!op->nbsp) {
> @@ -3481,7 +3825,7 @@ build_lswitch_flows(struct hmap *datapaths, struct
> hmap *ports,
>                        "outport = \""MC_FLOOD"\"; output;");
>      }
>
> -    /* Ingress table 13: Destination lookup, unicast handling (priority
> 50), */
> +    /* Ingress table 14: Destination lookup, unicast handling (priority
> 50), */
>      HMAP_FOR_EACH (op, key_node, ports) {
>          if (!op->nbsp) {
>              continue;
> @@ -3581,7 +3925,7 @@ build_lswitch_flows(struct hmap *datapaths, struct
> hmap *ports,
>          }
>      }
>
> -    /* Ingress table 13: Destination lookup for unknown MACs (priority
> 0). */
> +    /* Ingress table 14: Destination lookup for unknown MACs (priority
> 0). */
>      HMAP_FOR_EACH (od, key_node, datapaths) {
>          if (!od->nbs) {
>              continue;
>
>
<snip>
diff mbox

Patch

diff --git ovn/northd/ovn-northd.8.xml ovn/northd/ovn-northd.8.xml
index ab8fd88..7788311 100644
--- ovn/northd/ovn-northd.8.xml
+++ ovn/northd/ovn-northd.8.xml
@@ -362,7 +362,62 @@ 
       </li>
     </ul>
 
-    <h3>Ingress Table 7: <code>from-lport</code> QoS marking</h3>
+     <h3>Ingress Table 7: <code>from-lport</code> Port Chaining</h3>
+
+    <p>
+      Logical flows in this table closely reproduce those in the
+      <code>QoS</code> table in the <code>OVN_Northbound</code> database
+      for the <code>from-lport</code> direction.
+    </p>
+
+    <ul>
+      <li>
+        For every port-chain a set of rules will be added to direct traffic
+        through the port pairs defined in the port-chain. A port chain
+        is composed of an ordered set of port-pair-groups that contain one or
+        more port-pairs. Traffic is directed into the port-chain by creating a
+        port-chain-classifier. A port-chain can be reused by different
+        port-chain-classifier instances allowing a port chain to be
+        applied to multiple traffic paths and application traffic types.
+
+        The port-chain-classifier defines a starting port or ending port and
+        a direction for the traffic, either uni-directional or bi-directional.
+        In addition a match expression can be defined to further filter
+        traffic.
+
+        Service insertion is implemented by adding 4 new flow rules into the
+        OVN northbound database for each VNF inserted. The service
+        insertion rules have a higher priority than the standard forwarding
+        rules. This means that they override the existing forwarding rules.
+        There are four new rules added for each insertion. Two ingress and two
+        egress, The first ingress rule sends all traffic destined for the
+        application into the VNF ingress port and the second rule takes all
+        traffic destined to the application from the VNF egress port to the
+        application, the priorities are such that the second rule is always
+        checked first.
+
+        The final rule in a chain re-circulates the packet back to the original
+        entry point to the chain. This ensures that the packet exiting to a
+        physcal network always enters the physical network from the appropriate
+        location preserving correct operation of upstream L2 learning. Within
+        the chain the OVN Geneve tunnel between hypervisors ensures that the
+        inner packet is not seen by physical networking devices.
+
+        In the egress direction the rules are similar if the
+        traffic is from the application it is sent to the VNF egress port
+        and if if is from the application and is from the VNF ingress port it
+        is delivered to the destination. Additional VNFs can be chained
+        together to create a sequence of operations.
+
+      </li>
+
+      <li>
+        One priority-0 fallback flow that matches all packets and advances to
+        the next table.
+      </li>
+    </ul>
+
+    <h3>Ingress Table 8: <code>from-lport</code> QoS marking</h3>
 
     <p>
       Logical flows in this table closely reproduce those in the
@@ -382,7 +437,7 @@ 
       </li>
     </ul>
 
-    <h3>Ingress Table 8: LB</h3>
+    <h3>Ingress Table 9: LB</h3>
 
     <p>
       It contains a priority-0 flow that simply moves traffic to the next
@@ -395,7 +450,7 @@ 
       connection.)
     </p>
 
-    <h3>Ingress Table 9: Stateful</h3>
+    <h3>Ingress Table 10: Stateful</h3>
 
     <ul>
       <li>
@@ -432,7 +487,7 @@ 
       </li>
     </ul>
 
-    <h3>Ingress Table 10: ARP/ND responder</h3>
+    <h3>Ingress Table 11: ARP/ND responder</h3>
 
     <p>
       This table implements ARP/ND responder in a logical switch for known
@@ -582,7 +637,7 @@  nd_na {
       </li>
     </ul>
 
-    <h3>Ingress Table 11: DHCP option processing</h3>
+    <h3>Ingress Table 12: DHCP option processing</h3>
 
     <p>
       This table adds the DHCPv4 options to a DHCPv4 packet from the
@@ -642,7 +697,7 @@  next;
       </li>
     </ul>
 
-    <h3>Ingress Table 12: DHCP responses</h3>
+    <h3>Ingress Table 13: DHCP responses</h3>
 
     <p>
       This table implements DHCP responder for the DHCP replies generated by
@@ -724,7 +779,7 @@  output;
       </li>
     </ul>
 
-    <h3>Ingress Table 13 Destination Lookup</h3>
+    <h3>Ingress Table 14 Destination Lookup</h3>
 
     <p>
       This table implements switching behavior.  It contains these logical
diff --git ovn/northd/ovn-northd.c ovn/northd/ovn-northd.c
index d0a5ba2..090f768 100644
--- ovn/northd/ovn-northd.c
+++ ovn/northd/ovn-northd.c
@@ -106,13 +106,14 @@  enum ovn_stage {
     PIPELINE_STAGE(SWITCH, IN,  PRE_LB,         4, "ls_in_pre_lb")        \
     PIPELINE_STAGE(SWITCH, IN,  PRE_STATEFUL,   5, "ls_in_pre_stateful")  \
     PIPELINE_STAGE(SWITCH, IN,  ACL,            6, "ls_in_acl")           \
-    PIPELINE_STAGE(SWITCH, IN,  QOS_MARK,       7, "ls_in_qos_mark")      \
-    PIPELINE_STAGE(SWITCH, IN,  LB,             8, "ls_in_lb")            \
-    PIPELINE_STAGE(SWITCH, IN,  STATEFUL,       9, "ls_in_stateful")      \
-    PIPELINE_STAGE(SWITCH, IN,  ARP_ND_RSP,    10, "ls_in_arp_rsp")       \
-    PIPELINE_STAGE(SWITCH, IN,  DHCP_OPTIONS,  11, "ls_in_dhcp_options")  \
-    PIPELINE_STAGE(SWITCH, IN,  DHCP_RESPONSE, 12, "ls_in_dhcp_response") \
-    PIPELINE_STAGE(SWITCH, IN,  L2_LKUP,       13, "ls_in_l2_lkup")       \
+    PIPELINE_STAGE(SWITCH, IN,  CHAIN,          7, "ls_in_chain")        \
+    PIPELINE_STAGE(SWITCH, IN,  QOS_MARK,       8, "ls_in_qos_mark")    \
+    PIPELINE_STAGE(SWITCH, IN,  LB,             9, "ls_in_lb")            \
+    PIPELINE_STAGE(SWITCH, IN,  STATEFUL,      10, "ls_in_stateful")      \
+    PIPELINE_STAGE(SWITCH, IN,  ARP_ND_RSP,    11, "ls_in_arp_rsp")       \
+    PIPELINE_STAGE(SWITCH, IN,  DHCP_OPTIONS,  12, "ls_in_dhcp_options")  \
+    PIPELINE_STAGE(SWITCH, IN,  DHCP_RESPONSE, 13, "ls_in_dhcp_response") \
+    PIPELINE_STAGE(SWITCH, IN,  L2_LKUP,       14, "ls_in_l2_lkup")       \
                                                                       \
     /* Logical switch egress stages. */                               \
     PIPELINE_STAGE(SWITCH, OUT, PRE_LB,       0, "ls_out_pre_lb")     \
@@ -160,7 +161,6 @@  enum ovn_stage {
 #define REGBIT_CONNTRACK_COMMIT "reg0[1]"
 #define REGBIT_CONNTRACK_NAT    "reg0[2]"
 #define REGBIT_DHCP_OPTS_RESULT "reg0[3]"
-
 /* Register definitions for switches and routers. */
 #define REGBIT_NAT_REDIRECT     "reg9[0]"
 /* Indicate that this packet has been recirculated using egress
@@ -3014,6 +3014,348 @@  build_acls(struct ovn_datapath *od, struct hmap *lflows)
     }
 }
 
+static int
+cmp_port_pair_groups(const void *ppg1_, const void *ppg2_)
+{
+    const struct nbrec_logical_port_pair_group *const *ppg1p = ppg1_;
+    const struct nbrec_logical_port_pair_group *const *ppg2p = ppg2_;
+    const struct nbrec_logical_port_pair_group *ppg1 = *ppg1p;
+    const struct nbrec_logical_port_pair_group *ppg2 = *ppg2p;
+
+
+    return ( (int)ppg1->sortkey - (int)ppg2->sortkey);
+}
+
+static void
+build_chain(struct ovn_datapath *od, struct hmap *lflows, struct hmap *ports)
+{
+   /*
+    * TODO Items
+    *     * IPV6 support
+    *     * Load balancing support
+    *     * Bidirectional parameter support
+    *     * Support modes of VNF (BitW, L2, L3)
+    *     * Remove port-security on VNF Ports (if set by CMS)
+    *     * Add some state to allow match that does not require 'inport'
+    *     * Support visiting the same VNF more than once
+    *     * Unit tests!
+    */
+    const uint16_t ingress_inner_priority = 150;
+    const uint16_t ingress_outer_priority = 100;
+    const uint16_t egress_inner_priority = 150;
+    const uint16_t egress_outer_priority = 100;
+
+    struct ovn_port **input_port_array = NULL;
+    struct ovn_port **output_port_array = NULL;
+
+    struct ovn_port *dst_port = NULL;
+    struct ovn_port *src_port = NULL;
+
+    struct nbrec_logical_port_chain *lpc;
+    struct nbrec_logical_port_pair_group *lppg;
+    struct nbrec_logical_port_pair *lpp;
+    struct nbrec_logical_port_chain_classifier *lcc;
+
+    char *lcc_match = NULL;
+    char *lcc_action = NULL;
+    struct ovn_port *traffic_port;
+    unsigned int chain_direction = 2;
+    unsigned int chain_path = 2;
+    char * chain_match = NULL;
+
+    /* Ingress table ls_in_chain: default to passing through to the next table
+     * (priority 0)
+     */
+    ovn_lflow_add(lflows, od, S_SWITCH_IN_CHAIN, 0, "1", "next;");
+
+    /* No port chains defined therefore, no further work to do here. */
+    if (!od->nbs->port_chain_classifiers) {
+        return;
+    }
+    /* Iterate through all the port-chains defined for this datapath. */
+    for (size_t i = 0; i < od->nbs->n_port_chain_classifiers; i++) {
+        lcc = od->nbs->port_chain_classifiers[i];
+        /* Get the parameters from the classifier */
+        lpc = lcc->chain;
+        //traffic_port = lcc->port;
+        traffic_port =  ovn_port_find(ports,lcc->port->name);
+        if (traffic_port == NULL) {
+            static struct vlog_rate_limit rl =
+                        VLOG_RATE_LIMIT_INIT(1, 1);
+            VLOG_WARN_RL(&rl,
+                        "Traffic port %s does not exist\n",
+                              lcc->port->name);
+            break;
+        }
+        /*
+        * Check chain has one or more groups
+        */
+        if (lpc->n_port_pair_groups == 0) {
+            static struct vlog_rate_limit rl =
+                        VLOG_RATE_LIMIT_INIT(1, 1);
+            VLOG_WARN_RL(&rl,
+                "SFC Chain: %s used in classifier: %s does not not "
+                 "have any port pair groups\n",
+                              lcc->chain->name, lcc->name);
+            break;
+
+        }
+        /* TODO Check port exists. */
+        struct eth_addr traffic_logical_port_ea;
+        ovs_be32 traffic_logical_port_ip;
+        ovs_scan(traffic_port->nbsp->addresses[0],
+                    ETH_ADDR_SCAN_FMT" "IP_SCAN_FMT,
+                    ETH_ADDR_SCAN_ARGS(traffic_logical_port_ea),
+                    IP_SCAN_ARGS(&traffic_logical_port_ip));
+        /* Set the port to use as source or destination. */
+        if (strcmp(lcc->direction,"entry-lport")==0) {
+            chain_path = 0;
+        } else {
+            chain_path = 1;
+        }
+        /* Set the direction of the port-chain. */
+        if (strcmp(lcc->path,"uni-directional") == 0) {
+            chain_direction = 0;
+        } else {
+            chain_direction = 1;
+        }
+        /* Set the match parameters. */
+        chain_match = lcc->match;
+        /*
+         * Allocate space for port-pairs + 1. The Extra 1 represents the
+         * final hop to reach desired destination.
+         * TODO: We are going to allocate enough space to hold all the hops:
+         *  1 x portGroups + 1. This needs
+         *  to enhanced to: SUM(port pairs of each port group) + 1
+         */
+        input_port_array = xmalloc(sizeof *src_port *
+                                   lpc->n_port_pair_groups);
+        output_port_array = xmalloc(sizeof *dst_port *
+                                  (lpc->n_port_pair_groups));
+        /* Copy port groups from chain and sort them according to sortkey.*/
+        struct nbrec_logical_port_pair_group **port_pair_groups =
+                                 xmemdup(lpc->port_pair_groups,
+                          sizeof *port_pair_groups * lpc->n_port_pair_groups);
+        if (lpc->n_port_pair_groups > 1) {
+            qsort(port_pair_groups, lpc->n_port_pair_groups,
+              sizeof *port_pair_groups, cmp_port_pair_groups);
+        }
+        /* For each port-pair-group in a port chain pull out the port-pairs.*/
+        for (size_t j = 0; j < lpc->n_port_pair_groups; j++) {
+            lppg = port_pair_groups[j];
+            for (size_t k = 0; k < lppg->n_port_pairs; k++) {
+                 /* TODO: Need to add load balancing logic when LB becomes
+                 * available. Until LB is available just take the first
+                 * PP in the PPG. */
+                if (k > 0) {
+                    static struct vlog_rate_limit rl =
+                        VLOG_RATE_LIMIT_INIT(1, 1);
+                    VLOG_WARN_RL(&rl,
+                        "Currently lacking support for more than \
+                            one port-pair %"PRIuSIZE"\n",
+                              lppg->n_port_pairs);
+                    break;
+                }
+                lpp = lppg->port_pairs[k];
+                input_port_array[j] = lpp->inport ? ovn_port_find(ports,
+                                       lpp->inport->name) : NULL;
+                output_port_array[j] = lpp->outport ? ovn_port_find(ports,
+                                        lpp->outport->name) : NULL;
+            }
+        /* At this point we need to determine the final hop port to add to
+         * the chain. This defines the complete path for packets through
+         * the chain. */
+        }
+    /*
+    * Insert the lowest priorty rule dest is src-logical-port. These are the
+    * entry points into the chain in either direction. The match statement
+    * is used to filter the entry port to provide higher granularity of
+    * filtering.
+    */
+    if (chain_path == 1) { /* Path starting from entry port */
+        if (strcmp(chain_match,"")!=0) {
+           lcc_match =  xasprintf(
+            "eth.src == "ETH_ADDR_FMT" && %s",
+             ETH_ADDR_ARGS(traffic_logical_port_ea), chain_match);
+        } else {
+           lcc_match =  xasprintf(
+            "eth.src == "ETH_ADDR_FMT,
+             ETH_ADDR_ARGS(traffic_logical_port_ea));
+        }
+           lcc_action = xasprintf("outport = %s; output;",
+                                input_port_array[0]->json_key);
+    } else {  /* Path going to the entry port */
+        if (strcmp(chain_match,"")!=0) {
+           lcc_match =  xasprintf(
+            "eth.dst == "ETH_ADDR_FMT" && %s",
+             ETH_ADDR_ARGS(traffic_logical_port_ea), chain_match);
+        } else {
+           lcc_match =  xasprintf(
+            "eth.dst == "ETH_ADDR_FMT,
+             ETH_ADDR_ARGS(traffic_logical_port_ea));
+        }
+           lcc_action = xasprintf("outport = %s; output;",
+                    output_port_array[lpc->n_port_pair_groups-1]->json_key);
+    }
+    ovn_lflow_add(lflows, od, S_SWITCH_IN_CHAIN, ingress_outer_priority,
+                       lcc_match, lcc_action);
+    free(lcc_match);
+    free(lcc_action);
+    /*
+    * For last VNF send the flow back to teh original chassis and exit from
+    * there.
+    */
+    if (chain_path == 1) { /* Path starting from entry port */
+           lcc_match = xasprintf(
+                    "eth.src == "ETH_ADDR_FMT" && inport == %s",
+                     ETH_ADDR_ARGS(traffic_logical_port_ea),
+                     output_port_array[lpc->n_port_pair_groups-1]->json_key);
+    } else { /* Path starting from destination port. */
+            lcc_match = xasprintf(
+                    "eth.dst == "ETH_ADDR_FMT" && inport == %s",
+                     ETH_ADDR_ARGS(traffic_logical_port_ea),
+                     input_port_array[0]->json_key);
+    }
+             //lcc_action = xasprintf("next;");
+             //lcc_action = xasprintf("flags.loopback = 1; "
+             //    REGBIT_CHAIN_LOOPBACK" = 1;"
+             //   "next(pipeline=ingress, table=0);");
+    struct ds actions = DS_EMPTY_INITIALIZER;
+    ds_put_format(&actions, "clone { ct_clear; "
+                              "inport = outport; outport = \"\"; "
+                              "flags = 0; flags.loopback = 1; ");
+    for (int ii = 0; ii < MFF_N_LOG_REGS; ii++) {
+            ds_put_format(&actions, "reg%d = 0; ", ii);
+    }
+    ds_put_format(&actions, "next(pipeline=ingress, table=%d); };",
+                            ovn_stage_get_table(S_SWITCH_IN_CHAIN) + 1);
+    ovn_lflow_add(lflows, od, S_SWITCH_IN_CHAIN, ingress_inner_priority,
+                        lcc_match, ds_cstr(&actions));
+    free(lcc_match);
+        //free(lcc_action);
+    ds_destroy(&actions);
+
+    /*
+    * Loop over all VNFs and create flow rules for each
+    * Only get here when there is more than one VNF in the chain.
+    */
+    for (size_t j = 1; j < lpc->n_port_pair_groups; j++) {
+
+        /* Apply inner rules flows */
+        if (chain_path == 1) { /* Path starting from entry port */
+            lcc_match = xasprintf(
+                    "eth.src == "ETH_ADDR_FMT" && inport == %s",
+                                 ETH_ADDR_ARGS(traffic_logical_port_ea),
+                                 output_port_array[j-1]->json_key);
+            lcc_action = xasprintf("outport = %s; output;",
+                                input_port_array[j]->json_key);
+        } else { /* Path going to entry port. */
+            lcc_match = xasprintf(
+                    "eth.dst == "ETH_ADDR_FMT" && inport == %s",
+                    ETH_ADDR_ARGS(traffic_logical_port_ea),
+                    input_port_array[lpc->n_port_pair_groups-j]->json_key);
+             lcc_action = xasprintf("outport = %s; output;",
+                  output_port_array[lpc->n_port_pair_groups-(j+1)]->json_key);
+        }
+        ovn_lflow_add(lflows, od, S_SWITCH_IN_CHAIN, ingress_inner_priority,
+                        lcc_match, lcc_action);
+        free(lcc_match);
+        free(lcc_action);
+    }
+    /* bi-directional chain (Response)*/
+    if (chain_direction == 1) {
+        /*
+         * Insert the lowest priorty rule dest is src-logical-port
+         */
+        if (chain_path == 1) { /* Path from source port. */
+            if (strcmp(chain_match,"")!=0) {
+                 lcc_match =  xasprintf("eth.dst == "ETH_ADDR_FMT" && %s",
+                        ETH_ADDR_ARGS(traffic_logical_port_ea), chain_match);
+            } else { /* Path to source port */
+
+                 lcc_match =  xasprintf("eth.dst == "ETH_ADDR_FMT,
+                                    ETH_ADDR_ARGS(traffic_logical_port_ea));
+            }
+            lcc_action = xasprintf("outport = %s; output;",
+                    output_port_array[lpc->n_port_pair_groups-1]->json_key);
+        } else { /* Path from destination port. */
+           if (strcmp(chain_match,"")!=0) {
+                lcc_match =  xasprintf("eth.src == "ETH_ADDR_FMT" && %s",
+                        ETH_ADDR_ARGS(traffic_logical_port_ea), chain_match);
+            } else {
+                lcc_match =  xasprintf("eth.src == "ETH_ADDR_FMT,
+                        ETH_ADDR_ARGS(traffic_logical_port_ea));
+            }
+            lcc_action = xasprintf("outport = %s; output;",
+                    input_port_array[0]->json_key);
+        }
+        ovn_lflow_add(lflows, od, S_SWITCH_IN_CHAIN, egress_outer_priority,
+                       lcc_match, lcc_action);
+        free(lcc_match);
+        free(lcc_action);
+        /*
+        * End Entry Flow to classification chain entry point.
+        */
+        /* Apply last rule to exit from chain */
+        if (chain_path == 1) { /* Path from source port. */
+             lcc_match = xasprintf(
+                    "eth.dst == "ETH_ADDR_FMT" && inport == %s",
+                                 ETH_ADDR_ARGS(traffic_logical_port_ea),
+                                 input_port_array[0]->json_key);
+
+        } else { /* Path to source port. */
+             lcc_match = xasprintf(
+                "eth.src == "ETH_ADDR_FMT" && inport == %s",
+                    ETH_ADDR_ARGS(traffic_logical_port_ea),
+                    output_port_array[lpc->n_port_pair_groups-1]->json_key);
+        }
+        struct ds actions = DS_EMPTY_INITIALIZER;
+        ds_put_format(&actions,
+                              "clone { ct_clear; "
+                              "inport = outport; outport = \"\"; "
+                              "flags = 0; flags.loopback = 1; ");
+        for (int ii = 0; ii < MFF_N_LOG_REGS; ii++) {
+                    ds_put_format(&actions, "reg%d = 0; ", ii);
+        }
+        ds_put_format(&actions, "next(pipeline=ingress, table=%d); };",
+                              ovn_stage_get_table(S_SWITCH_IN_CHAIN) + 1);
+        ovn_lflow_add(lflows, od, S_SWITCH_IN_CHAIN,
+                        egress_inner_priority, lcc_match, ds_cstr(&actions));
+        ds_destroy(&actions);
+        free(lcc_match);
+
+        for (int j = 1; j< lpc->n_port_pair_groups; j++) {
+
+            /* Completed first catch all rule for this port-chain. */
+
+            /* Apply inner rules flows */
+            if (chain_path == 1) { /* Path from source port. */
+                lcc_match = xasprintf(
+                    "eth.dst == "ETH_ADDR_FMT" && inport == %s",
+                    ETH_ADDR_ARGS(traffic_logical_port_ea),
+                    input_port_array[lpc->n_port_pair_groups-j]->json_key);
+                 lcc_action = xasprintf("outport = %s; output;",
+                  output_port_array[lpc->n_port_pair_groups-(j+1)]->json_key);
+
+            } else { /* Path to source port. */
+                lcc_match = xasprintf(
+                    "eth.src == "ETH_ADDR_FMT" && inport == %s",
+                        ETH_ADDR_ARGS(traffic_logical_port_ea),
+                        output_port_array[j-1]->json_key);
+                 lcc_action = xasprintf("outport = %s; output;",
+                        input_port_array[j]->json_key);
+            }
+            ovn_lflow_add(lflows, od, S_SWITCH_IN_CHAIN,
+                          egress_inner_priority, lcc_match, lcc_action);
+            free(lcc_match);
+            free(lcc_action);
+        }
+    }
+    free(input_port_array);
+    free(output_port_array);
+    free(port_pair_groups);
+    }
+}
 static void
 build_qos(struct ovn_datapath *od, struct hmap *lflows) {
     ovn_lflow_add(lflows, od, S_SWITCH_IN_QOS_MARK, 0, "1", "next;");
@@ -3142,7 +3484,7 @@  build_lswitch_flows(struct hmap *datapaths, struct hmap *ports,
     struct ds actions = DS_EMPTY_INITIALIZER;
 
     /* Build pre-ACL and ACL tables for both ingress and egress.
-     * Ingress tables 3 through 9.  Egress tables 0 through 6. */
+     * Ingress tables 3 through 10.  Egress tables 0 through 6. */
     struct ovn_datapath *od;
     HMAP_FOR_EACH (od, key_node, datapaths) {
         if (!od->nbs) {
@@ -3153,6 +3495,7 @@  build_lswitch_flows(struct hmap *datapaths, struct hmap *ports,
         build_pre_lb(od, lflows);
         build_pre_stateful(od, lflows);
         build_acls(od, lflows);
+        build_chain(od, lflows, ports);
         build_qos(od, lflows);
         build_lb(od, lflows);
         build_stateful(od, lflows);
@@ -3225,9 +3568,9 @@  build_lswitch_flows(struct hmap *datapaths, struct hmap *ports,
         ovn_lflow_add(lflows, od, S_SWITCH_IN_PORT_SEC_IP, 0, "1", "next;");
     }
 
-    /* Ingress table 10: ARP/ND responder, skip requests coming from localnet
-     * and vtep ports. (priority 100); see ovn-northd.8.xml for the
-     * rationale. */
+    /* Ingress table 11: ARP/ND responder, skip requests coming from localnet
+     * ports. (priority 100). */
+
     HMAP_FOR_EACH (op, key_node, ports) {
         if (!op->nbsp) {
             continue;
@@ -3242,7 +3585,7 @@  build_lswitch_flows(struct hmap *datapaths, struct hmap *ports,
         }
     }
 
-    /* Ingress table 10: ARP/ND responder, reply for known IPs.
+    /* Ingress table 11: ARP/ND responder, reply for known IPs.
      * (priority 50). */
     HMAP_FOR_EACH (op, key_node, ports) {
         if (!op->nbsp) {
@@ -3335,7 +3678,7 @@  build_lswitch_flows(struct hmap *datapaths, struct hmap *ports,
         }
     }
 
-    /* Ingress table 10: ARP/ND responder, by default goto next.
+    /* Ingress table 11: ARP/ND responder, by default goto next.
      * (priority 0)*/
     HMAP_FOR_EACH (od, key_node, datapaths) {
         if (!od->nbs) {
@@ -3345,7 +3688,7 @@  build_lswitch_flows(struct hmap *datapaths, struct hmap *ports,
         ovn_lflow_add(lflows, od, S_SWITCH_IN_ARP_ND_RSP, 0, "1", "next;");
     }
 
-    /* Logical switch ingress table 11 and 12: DHCP options and response
+    /* Logical switch ingress table 12 and 13: DHCP options and response
          * priority 100 flows. */
     HMAP_FOR_EACH (op, key_node, ports) {
         if (!op->nbsp) {
@@ -3449,7 +3792,7 @@  build_lswitch_flows(struct hmap *datapaths, struct hmap *ports,
         }
     }
 
-    /* Ingress table 11 and 12: DHCP options and response, by default goto next.
+    /* Ingress table 12 and 13: DHCP options and response, by default goto next.
      * (priority 0). */
 
     HMAP_FOR_EACH (od, key_node, datapaths) {
@@ -3461,7 +3804,8 @@  build_lswitch_flows(struct hmap *datapaths, struct hmap *ports,
         ovn_lflow_add(lflows, od, S_SWITCH_IN_DHCP_RESPONSE, 0, "1", "next;");
     }
 
-    /* Ingress table 13: Destination lookup, broadcast and multicast handling
+
+    /* Ingress table 14: Destination lookup, broadcast and multicast handling
      * (priority 100). */
     HMAP_FOR_EACH (op, key_node, ports) {
         if (!op->nbsp) {
@@ -3481,7 +3825,7 @@  build_lswitch_flows(struct hmap *datapaths, struct hmap *ports,
                       "outport = \""MC_FLOOD"\"; output;");
     }
 
-    /* Ingress table 13: Destination lookup, unicast handling (priority 50), */
+    /* Ingress table 14: Destination lookup, unicast handling (priority 50), */
     HMAP_FOR_EACH (op, key_node, ports) {
         if (!op->nbsp) {
             continue;
@@ -3581,7 +3925,7 @@  build_lswitch_flows(struct hmap *datapaths, struct hmap *ports,
         }
     }
 
-    /* Ingress table 13: Destination lookup for unknown MACs (priority 0). */
+    /* Ingress table 14: Destination lookup for unknown MACs (priority 0). */
     HMAP_FOR_EACH (od, key_node, datapaths) {
         if (!od->nbs) {
             continue;
diff --git ovn/ovn-architecture.7.xml ovn/ovn-architecture.7.xml
index d8114f1..3d8d50c 100644
--- ovn/ovn-architecture.7.xml
+++ ovn/ovn-architecture.7.xml
@@ -383,6 +383,19 @@ 
     </li>
 
     <li>
+      <dfn>Logical port chains</dfn> are logical references to virtual network
+      functions (VNF). Adding a logical port chain requires adding one or more
+      VNFs to a port chain and then steering traffic into the port chain by
+      adding the port chain to a port chain classifier. The port chain
+      classifier defines the flows to direct into the port chain, the
+      direction to apply to the flows on ( to or from a logical port) and
+      whether to apply the port chain in one direction (uni-directional) or
+      both directions, (bi-directional). This separation enables the port
+      chain to be defined once and used in several classifiers.
+      See <code>Life Cycle of an inserted VNF</code>, below, for details.
+    </li>
+
+    <li>
       <p>
         <dfn>Logical ports</dfn> represent the points of connectivity in and
         out of logical switches and logical routers.  Some common types of
@@ -566,6 +579,84 @@ 
     </li>
   </ol>
 
+ <h2>Life Cycle of an Inserted Virtual Network Function (VNF)</h2>
+
+ <p>
+   OVN provides an abstraction to enable the insertion of an arbitrary virtual
+   network function (VNF) into the path of traffic to or from an application.
+   A VNF is different from an application VM in that it acts on traffic
+   between applications, and in most cases does not terminate a flow. Proxy
+   functions are an exception as they terminate the flow from the source and
+   create a new flow to the destination. Examples of VNFs are security
+   functions (e.g. intrusion detection systems, firewalls), load balancers,
+   and traffic enhancement services.
+ </p>
+ <p>
+   The requirements on the VNF to be inserted here are minimal: it must act as
+   a <code>bump in the wire (BITW)</code> and can have one or two virtual
+   network ports for traffic. If it has two network ports, it accepts traffic
+   on one port and transmits it out the other; if it has only one port, that
+   port serves both purposes. The requirement for the VNF to act as a BITW
+   removes the need for the VNF to participate in L2/L3 networking, which
+   provides improved agility and reduces the coupling between OVN and the VNF.
+   Not havign the VNF participate in L2/L3 networking makes scale up/down
+   scenarios easier to implement.
+ </p>
+ <p>
+   The steps in this example refer to the details of the OVN Northbound
+   database schema. There is a new table in the OVN Northbound database to
+   direct traffic to VNFs called <code>port_chain_classifiers</code>, which
+   contains the required information to direct traffic. This table takes as
+   input a port chain which defines an ordered list of VNFs, and
+   classification parameters to define the traffic to send to the port chain.
+   The same port chain can be used for multiple applications, as there is
+   typically an N:1 relationship between applications and VNFs. A single
+   VNF may be part of several service insertions, but each one is logically
+   separate.
+ </p>
+ <p>
+   The following steps are an overview to inserting a new VNF into the traffic
+   path. The sections below go into each step in more detail.
+ </p>
+ <ol>
+   <li>
+     The CMS administrator creates a new virtual network function
+     <code>(VNF)</code>, using the CMS user interface or API. The CMS
+     administrator creates the logical ports (ingress and egress) for the VNF.
+     If the CMS is OpenStack, this creates a reusable port-pair defining the
+     interface to the VNF. The administrator also typically creates a separate
+     management port for the VNF, but that is not relevant to the service
+     insertion workflow. A single VNF can participate with several
+     applications, either as a security VM, protecting multiple applications,
+     or as a load balancer VM, distributing load across multiple applications.
+   </li>
+
+   <li>
+    <p>
+     The CMS administrator attaches the VNF port pair to a port pair group.
+     The purpose o fthe port pair group is to enable load balancing across
+     multiple port-pairs. The port-pair-group is added to an ordered list of
+     port-pair-groups contained in a port chain. At this point the port-chain
+     is just a logicial set of operations with no flows attached.
+     </p>
+     <p> Creating a port-chain-classifier, defining the flow classification
+     parameters and the chain to operate on flows inserts a row into the
+     <code>port-chain-classifier</code> table in the OVN northbound
+     database. This directs traffic to go through the VNF chain, applying
+     the operations defined in the port chain.
+   </p>
+   </li>
+
+   <li>
+     <p>
+       Eventually, the application VM shuts down and the CMS removes the
+       <code>port-chain-classifier</code>.  (However, it is harmless if it
+       remains, since no traffic will be sent to the port-chain unless it is
+       included in a port-chain-classifier.
+     </p>
+  </li>
+</ol>
+
   <h2>Life Cycle of a Container Interface Inside a VM</h2>
 
   <p>
diff --git ovn/ovn-nb.ovsschema ovn/ovn-nb.ovsschema
index dd0ac3d..c1c5323 100644
--- ovn/ovn-nb.ovsschema
+++ ovn/ovn-nb.ovsschema
@@ -1,7 +1,7 @@ 
 {
     "name": "OVN_Northbound",
-    "version": "5.5.0",
-    "cksum": "2099428463 14236",
+    "version": "5.5.1",
+    "cksum": "1431855236 18774",
     "tables": {
         "NB_Global": {
             "columns": {
@@ -30,6 +30,24 @@ 
                                            "refType": "strong"},
                                    "min": 0,
                                    "max": "unlimited"}},
+                "port_chain_classifiers":
+                              {"type":
+                                 {"key": {"type": "uuid",
+                                  "refTable": "Logical_Port_Chain_Classifier",
+                                  "refType": "strong"},
+                                  "min": 0,
+                                  "max": "unlimited"}},
+                "port_chains":
+                              {"type": {"key": {"type": "uuid",
+                                        "refTable": "Logical_Port_Chain",
+                                        "refType": "strong"},
+                                        "min": 0,
+                                        "max": "unlimited"}},
+                "port_pairs":  {"type": {"key": {"type": "uuid",
+                                         "refTable": "Logical_Port_Pair",
+                                         "refType": "strong"},
+                                         "min": 0,
+                                         "max": "unlimited"}},
                 "acls": {"type": {"key": {"type": "uuid",
                                           "refTable": "ACL",
                                           "refType": "strong"},
@@ -98,6 +116,71 @@ 
                              "min": 0, "max": "unlimited"}}},
             "indexes": [["name"]],
             "isRoot": false},
+        "Logical_Port_Chain": {
+            "columns": {
+                "name": {"type": "string"},
+                "port_pair_groups":  {"type":
+                                        {"key": {"type": "uuid",
+                                         "refTable": "Logical_Port_Pair_Group",
+                                         "refType": "strong"},
+                                         "min": 0, "max": "unlimited"}},
+                "external_ids": {
+                    "type": {"key": "string", "value": "string",
+                             "min": 0, "max": "unlimited"}}},
+            "isRoot": false},
+        "Logical_Port_Pair_Group": {
+            "columns": {
+                "name": {"type": "string"},
+                "port_pairs":  {"type":
+                                  {"key": {"type": "uuid",
+                                           "refTable": "Logical_Port_Pair",
+                                           "refType": "strong"},
+                                           "min": 0, "max": "unlimited"}},
+                "sortkey": {"type": {"key": {"type": "integer",
+                                              "minInteger": 0,
+                                              "maxInteger": 127 }}},
+                "external_ids": {
+                    "type": {"key": "string", "value": "string",
+                             "min": 0, "max": "unlimited"}}},
+            "isRoot": false},
+        "Logical_Port_Chain_Classifier": {
+            "columns": {
+                "name": {"type": "string"},
+                "chain": {"type": {"key": {"type": "uuid",
+                                             "refTable": "Logical_Port_Chain",
+                                             "refType": "strong"},
+                                     "min": 0, "max": 1}},
+                "port": {"type": {"key": {"type": "uuid",
+                                             "refTable": "Logical_Switch_Port",
+                                             "refType": "strong"},
+                                     "min": 0, "max": 1}},
+                "direction": {"type":
+                           {"key": {"type": "string",
+                            "enum": ["set", ["entry-lport", "exit-lport"]]}}},
+                "path": {"type":
+                           {"key": {"type": "string",
+                            "enum": ["set",
+                                    ["uni-directional", "bi-directional"]]}}},
+                "match": {"type": "string"},
+                "external_ids": {
+                    "type": {"key": "string", "value": "string",
+                             "min": 0, "max": "unlimited"}}},
+            "isRoot": false},
+        "Logical_Port_Pair": {
+            "columns": {
+                "name": {"type": "string"},
+                "outport": {"type": {"key": {"type": "uuid",
+                                             "refTable": "Logical_Switch_Port",
+                                             "refType": "strong"},
+                                     "min": 0, "max": 1}},
+                "inport": {"type": {"key": {"type": "uuid",
+                                            "refTable": "Logical_Switch_Port",
+                                            "refType": "strong"},
+                                    "min": 0, "max": 1}},
+                "external_ids": {
+                    "type": {"key": "string", "value": "string",
+                             "min": 0, "max": "unlimited"}}},
+            "isRoot": false},
         "Address_Set": {
             "columns": {
                 "name": {"type": "string"},
diff --git ovn/ovn-nb.xml ovn/ovn-nb.xml
index 7a1c20e..ea16686 100644
--- ovn/ovn-nb.xml
+++ ovn/ovn-nb.xml
@@ -121,6 +121,35 @@ 
       </p>
     </column>
 
+    <column name="port_chain_classifiers">
+      <p>
+        The logical port chain classifiers defining the path of traffic
+        to take through attached port chains. The port chain classifier
+        defines the logical port traffic is to or from and whether the
+        chain is applied to flows in one or both directions.
+      </p>
+    </column>
+
+    <column name="port_chains">
+      <p>
+        The logical port-chains connected to the logical switch. Port chains
+        can be reused across multiple classifiers.
+      </p>
+      <p>
+        It is an error for port-pairs within a port chain to span multiple
+        logical switches.
+      </p>
+    </column>
+
+    <column name="port_pairs">
+      <p>
+        The logical chains that define the service path.
+      </p>
+      <p>
+        Logical port pairs cannot currently cross logical switch boundaries.
+      </p>
+    </column>
+
     <column name="load_balancer">
       Load balance a virtual ipv4 address to a set of logical port endpoint
       ipv4 addresses.
@@ -205,6 +234,157 @@ 
     </group>
   </table>
 
+  <table name="Logical_Port_Chain_Classifier" title="Logical port chain classifier">
+    <p>
+      Each row represents one logical port chain classifier
+    </p>
+    <column name="name">
+      <p>
+        A name for the logical chain classifer. This name has no special
+        meaning or purpose other than to provide convenience for human
+        interaction with the ovn-nb database.  There is no requirement for
+        the name to be unique.  The logical chain classifier's UUID should
+        be used as the unique identifier.
+      </p>
+    </column>
+    <column name="port">
+      <p>
+        The logical port that is either the src or dst of flows for the
+        port chain.
+      </p>
+      <p>
+        It is an error for this to be NULL.
+      </p>
+    </column>
+    <column name="chain">
+      <p>
+        The port chain for the flows to traverse. The port chain can be
+        reused in multiple classifiers.
+      </p>
+    </column>
+    <column name="direction">
+      <p>
+        The direction of the flows through the port chain, this can be either
+        "uni-directional" or "bi-directional".
+      </p>
+    </column>
+    <column name="path">
+      <p>
+       The path of the flow from or to the logical port. The valid values are
+       "entry-lport" or "exit-lport". If the path is "entry-lport" the rules
+       are applied to traffic leaving the entry-lport, if the path is
+       "exit-lport" the port-chain is applied to traffic going to the lport.
+      </p>
+    </column>
+    <column name="match">
+      <p>
+       The match is an optional match statement that will filter flows going
+       to the chain as defined by any valid openvswitch matches.
+      </p>
+      <p>
+        Care should be taken using match statements to ensure they do not
+        conflicit with in logical port the is the src or dst of flows. In
+        addition when operating on "bi-directional" flows care shoudl be taken
+        to only use match statements that work in both directions.
+      </p>
+    </column>
+    <group title="Common Columns">
+      <column name="external_ids">
+        See <em>External IDs</em> at the beginning of this document.
+      </column>
+    </group>
+  </table>
+    <table name="Logical_Port_Chain" title="Logical port chain">
+    <p>
+      Each row represents one logical port chain
+    </p>
+
+    <column name="name">
+      <p>
+        A name for the logical chain.  This name has no special meaning or
+        purpose other than to provide convenience for human interaction with
+        the ovn-nb database.  There is no requirement for the name to be
+        unique. The logical chains's UUID should be used as the unique
+        identifier.
+      </p>
+    </column>
+
+    <column name="port_pair_groups">
+      <p>
+        The logical list of port pairs that the flow goes through.
+      </p>
+
+      <p>
+        It is an error for a port pair group to be empty.
+      </p>
+    </column>
+
+    <group title="Common Columns">
+      <column name="external_ids">
+        See <em>External IDs</em> at the beginning of this document.
+      </column>
+    </group>
+  </table>
+  <table name="Logical_Port_Pair_Group" title="logical port pair groups">
+    <p>
+      An ordered port pair list
+    </p>
+
+    <column name="name">
+      <p>
+        Logical port pair group name
+      </p>
+    </column>
+
+    <column name="port_pairs">
+      <p>
+        port pair for this group
+      </p>
+    </column>
+
+    <column name="sortkey">
+      <p>
+        An integer used for ordering instances of
+        <ref table="Logical_Port_Pair_Group"/> in the
+        <ref column="port_pairs" table="Logical_Port_Chain"/> column
+        of <ref table="Logical_Port_Chain"/>.
+      </p>
+    </column>
+    <group title="Common Columns">
+      <column name="external_ids">
+        See <em>External IDs</em> at the beginning of this document.
+      </column>
+    </group>
+  </table>
+
+  <table name="Logical_Port_Pair" title="logical port pairs">
+    <p>
+      Ports pairs defining the service
+    </p>
+
+    <column name="name">
+      <p>
+        Logical port pair
+      </p>
+    </column>
+
+    <column name="outport">
+      <p>
+        Out logical port for this port pair. Can be the same value as inport.
+      </p>
+    </column>
+
+    <column name="inport">
+      <p>
+        In logical port for this port pair.
+      </p>
+    </column>
+    <group title="Common Columns">
+      <column name="external_ids">
+        See <em>External IDs</em> at the beginning of this document.
+      </column>
+    </group>
+  </table>
   <table name="Logical_Switch_Port" title="L2 logical switch port">
     <p>
       A port within an L2 logical switch.
@@ -239,7 +419,6 @@ 
           model other types of connectivity into an OVN logical switch.  The
           following types are defined:
         </p>
-
         <dl>
           <dt>(empty string)</dt>
           <dd>
@@ -979,6 +1158,13 @@ 
           ICMP unreachable message for other IP-based protocols.
           <code>Not implemented--currently treated as drop</code>
         </li>
+
+        <li>
+          <code>sfc</code>: Forward the packet into a logical port chain.
+          The chain to be used -- as well as any other attributes that
+          determine the behavior of the packet while in the chain -- are
+          provided via <ref column="options"/>.
+        </li>
       </ul>
     </column>
 
diff --git ovn/utilities/ovn-nbctl.8.xml ovn/utilities/ovn-nbctl.8.xml
index 70afc10..0167ff8 100644
--- ovn/utilities/ovn-nbctl.8.xml
+++ ovn/utilities/ovn-nbctl.8.xml
@@ -288,6 +288,237 @@ 
 
     </dl>
 
+    <h1>Logical Port Chain Classifier Commands</h1>
+
+    <dl>
+      <dt><code>lsp-chain-classifier-add</code></dt>
+      <dd>
+        <p>
+          Creates a new, logical port chain classifier.
+        </p>
+      </dd>
+
+      <dt>[<code>--may-exist</code> | <code>--add-duplicate</code>] <code>lsp-chain-classifier-add</code> <var>switch</var> <var>port-chain</var> <var>port</var> <var>direction</var> <var>path</var> <var>match</var> [<var>classifier</var>] </dt>
+      <dd>
+        <p>
+          Creates a new logical port-chain-classifier named
+          <var>classifier</var>. This operation inserts the chaining rules
+          into the ovn database.
+        </p>
+
+        <p>
+          The OVN northbound database schema does not require logical port
+          chain classifier names to be unique, but the whole point to the
+          names is to provide an easy way for humans to refer to the port
+          chain classifiers, making duplicate names unhelpful. Thus, without
+          any options, this command regards it as an error if
+          <var>clasisfier</var> is a duplicate name.  With
+          <code>--may-exist</code>, adding a duplicate name succeeds but does
+          not create a new logical port-chain. With
+          <code>--add-duplicate</code>, the command really creates a new
+          logical port-chain-classifier with a duplicate name.  It is an error
+          to specify both options.  If there are multiple logical
+          port-chainclassifiers with a duplicate name, configure the logical
+          port-chain-classifiers using the UUID instead of the
+          <var>classifier</var> name.
+        </p>
+
+        <p>
+          The number of chains that can be attached to a port is limited to one
+          currently. If more than one chain is attached to a port an error is
+          generated.
+        </p>
+      </dd>
+
+      <dt>[<code>--if-exists</code>] <code>lsp-chain-classififer-del</code> <var>port-chain-classifier</var></dt>
+      <dd>
+        Deletes <var>port-chain</var>. It is an error if
+        <var>port-chain-classifier</var> does not exist, unless
+        <code>--if-exists</code> is specified.
+      </dd>
+
+      <dt><code>lsp-chain-classifier-list</code> <var>switch</var> <var>port-chain-classifier</var></dt>
+      <dd>
+        Lists all existing port-chain-classifiers on standard output, one per
+        line.
+      </dd>
+       <dt><code>lsp-chain-classifier-show</code> [<var>port-chain-classifier</var>]</dt>
+      <dd>
+        Show all existing port-chain-classifiers on standard output, one per
+        line.
+      </dd>
+    </dl>
+
+    <h1>Logical Port Chain Commands</h1>
+
+    <dl>
+      <dt><code>lsp-chain-add</code></dt>
+      <dd>
+        <p>
+          Creates a new, logical port chain.
+        </p>
+      </dd>
+
+      <dt>[<code>--may-exist</code> | <code>--add-duplicate</code>] <code>lsp-chain-add</code> <var>port-chain</var></dt>
+      <dd>
+        <p>
+          Creates a new logical port-chain named <var>port-chain</var>, which
+          initially has no port pair groups. Port chains are made from an
+          ordered list of port-pair-groups. Each port-pair-group has one or
+          more port-pairs. Every port-pair within a port-pair-group should be
+          of the same type of VNF, but OVN cannot enforce this requirement.
+        </p>
+        <p>
+          The number of port pair groups that can be added to a port chain is
+          limited to 128.
+        </p>
+        <p>
+          The OVN northbound database schema does not require logical port
+          chain names to be unique, but the whole point to the names is to
+          provide an easy way for humans to refer to the port chains, making
+          duplicate names unhelpful.  Thus, without any options, this command
+          regards it as an error if <var>port-chain</var> is a duplicate name.
+          With <code>--may-exist</code>, adding a duplicate name succeeds but
+          does not create a new logical port-chain. With
+          <code>--add-duplicate</code>, the command really creates a new
+          logical port-chain with a duplicate name.  It is an error to specify
+          both options. If there are multiple logical port-chains with a
+          duplicate name, configure the logical port-chains using the UUID
+          instead of the <var>port-chain</var> name.
+        </p>
+      </dd>
+
+      <dt>[<code>--if-exists</code>] <code>lsp-chain-del</code> <var>port-chain</var></dt>
+      <dd>
+        Deletes <var>port-chain</var>.  It is an error if
+        <var>port-chain</var> does not exist, unless <code>--if-exists</code>
+        is specified.
+      </dd>
+
+      <dt><code>lsp-chain-list</code> <var>switch</var> <var>port-chain</var></dt>
+      <dd>
+        Lists all existing port-chains on standard output, one per line.
+      </dd>
+       <dt><code>lsp-chain-show</code> [<var>port-chain</var>]</dt>
+      <dd>
+        Show all existing port-chains on standard output, one per line.
+      </dd>
+    </dl>
+
+    <h1>Logical Port Pair Group Commands</h1>
+
+    <dl>
+      <dt><code>lsp-pair-group-add</code></dt>
+      <dd>
+        <p>
+          Creates a new, logical port pair group.
+        </p>
+      </dd>
+
+      <dt>[<code>--may-exist</code> | <code>--add-duplicate</code>] <code>lsp-pair-group-add</code> <var>port-pair-group</var> [<var>sortkey</var>var>]</dt>
+      <dd>
+        <p>
+          Creates a new logical port pair group named
+          <var>port-pair-group</var>, which initially has no port pairs. Port
+          pair groups are made from a collection of port-pairs. Each
+          port-pair-group has one or more port-pairs. Every port-pair within a
+          port-pair-group should be of the same type of VNF, but OVN cannot
+          enforce this requirement.
+        </p>
+        <p>
+          The sortkey parameter allows the defintion of the order of in which
+          the port pair groups are applied. The default is the order they are defined.
+          The list command shows the sortkey with the port pair group.
+        </p>
+        <p>
+          The OVN northbound database schema does not require logical port pair
+          group names to be unique, but the whole point to the names is to
+          provide an easy way for humans to refer to the port pair groups,
+          making duplicate names unhelpful.  Thus, without any options, this
+          command regards it as an error if <var>port-pair-group</var> is a
+          duplicate name. With <code>--may-exist</code>, adding a duplicate
+          name succeeds but does not create a new logical port-pair-group.
+          With <code>--add-duplicate</code>, the command really creates a new
+          logical port-pair-group with a duplicate name. It is an error to
+          specify both options.  If there are multiple logical port pair
+          groups with a duplicate name, configure the logical port pair groups
+          using the UUID instead of the <var>port-pair-group</var> name.
+        </p>
+      </dd>
+
+      <dt>[<code>--if-exists</code>] <code>lsp-pair-group-del</code> <var>port-pair-group</var></dt>
+      <dd>
+        Deletes <var>port-pair-group</var>.  It is an error if
+        <var>port-pair-group</var> does not exist, unless
+        <code>--if-exists</code> is specified.
+      </dd>
+
+      <dt><code>lsp-pair-group-list</code> <var>port-chain</var> </dt>
+      <dd>
+        <p>
+        Lists all existing port-pair-groups in a port-chain on standard
+        output, one per line.
+        </p>
+      </dd>
+      <dt><code>lsp-pair-group-add-port-pair</code> </dt>
+      <dd>
+        <p>
+        Adds a port-pair to an existing port-pair group.
+        </p>
+      </dd>
+      <dt><code>lsp-pair-group-del-port-pair</code> <var>port-pair-group</var> <var>port-pair</var> </dt>
+      <dd>
+        Deletes an existing port-pair from an existing port-pair group.
+      </dd>
+      </dl>
+
+      <h1>Logical Port Pair Commands</h1>
+
+      <dl>
+      <dt><code>lsp-pair-add</code></dt>
+      <dd>
+        <p>
+          Creates a new, logical port pair.
+        </p>
+      </dd>
+
+      <dt>[<code>--may-exist</code> | <code>--add-duplicate</code>] <code>lsp-pair-add</code> <var>port-pair</var></dt>
+      <dd>
+        <p>
+          Creates a new logical port pair named <var>port-pair</var>. Port
+          pairs are made from one or two logical switch ports. The ports
+          require that the attached VNF passes traffic unchanged between the
+          two ports of in the case of a single port in and out the same port.
+        </p>
+
+        <p>
+          The OVN northbound database schema does not require logical port
+          pair names to be unique, but the whole point to the names is to
+          provide an easy way for humans to refer to the port pairs, making
+          duplicate names unhelpful. Thus, without any options, this command
+          regards it as an error if <var>port-pair</var> is a duplicate name.
+          With <code>--may-exist</code>, adding a duplicate name succeeds but
+          does not create a new logical port-pair. With
+          <code>--add-duplicate</code>, the command really creates a new
+          logical port-pair with a duplicate name.  It is an error to specify
+          both options.  If there are multiple logical port pairs with a
+          duplicate name, configure the logical port pairs using the UUID
+          instead of the <var>port-pair</var> name.
+        </p>
+      </dd>
+
+      <dt>[<code>--if-exists</code>] <code>lsp-pair-del</code> <var>port-pair</var></dt>
+      <dd>
+        Deletes <var>port-pair</var>.  It is an error if <var>port-pair</var>
+        does not exist, unless <code>--if-exists</code> is specified.
+      </dd>
+
+      <dt><code>lsp-pair-list</code> <var>switch</var> <var>port-pair-group</var></dt>
+      <dd>
+        Lists all existing port pairs on standard output, one per line.
+      </dd>
+    </dl>
+
     <h1>Logical Router Commands</h1>
 
     <dl>
diff --git ovn/utilities/ovn-nbctl.c ovn/utilities/ovn-nbctl.c
index e5999a6..787dcd8 100644
--- ovn/utilities/ovn-nbctl.c
+++ ovn/utilities/ovn-nbctl.c
@@ -366,6 +366,46 @@  Logical switch port commands:\n\
                             set dhcpv4 options for PORT\n\
   lsp-get-dhcpv4-options PORT  get the dhcpv4 options for PORT\n\
 \n\
+Logical port chain classifier commands:\n\
+  lsp-chain-classifier-add SWITCH CHAIN PORT DIRECTION PATH [NAME] [MATCH]\n\
+                            add a CHAIN to a CLASSIFIER\n\
+  lsp-chain-classifier-del CLASSIFIER \n\
+                            remove classifier from switch\n\
+  lsp-chain-classifier-list [SWITCH]\n\
+                            print classifiers for SWITCH\n\
+  lsp-chain-classifier-show [SWITCH] [CLASSIFIER]\n\
+                            show structure of classifiers\n\
+                            for [SWITCH] [CCLASSIFIER]\n\
+\n\
+Logical port chain commands:\n\
+  lsp-chain-add SWITCH CHAIN         create a logical port-chain\n\
+                                     named CHAIN\n\
+  lsp-chain-del CHAIN                delete CHAIN\n\
+  lsp-chain-list [SWITCH]            print the names of all logical\n\
+                                     port-chains [on SWITCH]\n\
+  lsp-chain-show SWITCH [CHAIN]      print details on port-chains\n\
+                                     on SWITCH\n\
+\n\
+Logical port pair group commands:\n\
+  lsp-pair-group-add CHAIN [PAIR-GROUP [OFFSET]]\n\
+                    create a logical port-pair-group. Optionally,\n\
+                    indicate the order it should be in chain.\n\
+  lsp-pair-group-del PAIR-GROUP    delete a port-pair-group, does\n\
+                                   not delete port-pairs\n\
+  lsp-pair-group-list CHAIN        print port-pair-groups for a given chain\n\
+  lsp-pair-group-add-port-pair PAIR-GROUP LSP-PAIR add a port pair to a\n\
+                                                   port-pair-group\n\
+  lsp-pair-group-del-port-pair PAIR-GROUP LSP-PAIR del a port pair from a\n\
+                                                   port-pair-group\n\
+\n\
+Logical port pair commands:\n\
+  lsp-pair-add SWITCH PORT-IN PORT-OUT [LSP-PAIR]\n\
+                                     create a logical port-pair\n\
+  lsp-pair-del LSP-PAIR              delete a port-pair, does\n\
+                                     not delete ports\n\
+  lsp-pair-list [SWITCH [LSP-PAIR]]  print the names of all\n\
+                                     logical port-pairs\n\
+\n\
 Logical router commands:\n\
   lr-add [ROUTER]           create a logical router named ROUTER\n\
   lr-del ROUTER             delete ROUTER and all its ports\n\
@@ -778,6 +818,1105 @@  lsp_by_name_or_uuid(struct ctl_context *ctx, const char *id,
 
     return lsp;
 }
+
+/*
+ * Port chain CLI Functions
+ */
+static const struct nbrec_logical_port_chain *
+lsp_chain_by_name_or_uuid(struct ctl_context *ctx, const char *id,
+                           const bool must_exist)
+{
+    const struct nbrec_logical_port_chain *lsp_chain = NULL;
+    bool is_uuid = false;
+    struct uuid lsp_chain_uuid;
+
+    if (uuid_from_string(&lsp_chain_uuid, id)) {
+        is_uuid = true;
+        lsp_chain = nbrec_logical_port_chain_get_for_uuid(ctx->idl,
+                                                          &lsp_chain_uuid);
+    }
+
+    if (!lsp_chain) {
+        NBREC_LOGICAL_PORT_CHAIN_FOR_EACH(lsp_chain, ctx->idl) {
+            if (!strcmp(lsp_chain->name, id)) {
+                break;
+            }
+        }
+    }
+    if (!lsp_chain && must_exist) {
+        ctl_fatal("lsp_chain not found for %s: '%s'",
+                  is_uuid ? "UUID" : "name", id);
+    }
+
+    return lsp_chain;
+}
+static const struct nbrec_logical_port_pair_group *
+lsp_pair_group_by_name_or_uuid(struct ctl_context *ctx, const char *id,
+                                const bool must_exist)
+{
+    const struct nbrec_logical_port_pair_group *lsp_pair_group = NULL;
+    bool is_uuid = false;
+    struct uuid lsp_pair_group_uuid;
+
+    if (uuid_from_string(&lsp_pair_group_uuid, id)) {
+        is_uuid = true;
+        lsp_pair_group =
+          nbrec_logical_port_pair_group_get_for_uuid(ctx->idl,
+                                                      &lsp_pair_group_uuid);
+    }
+
+    if (!lsp_pair_group) {
+        NBREC_LOGICAL_PORT_PAIR_GROUP_FOR_EACH(lsp_pair_group, ctx->idl) {
+            if (!strcmp(lsp_pair_group->name, id)) {
+                break;
+            }
+        }
+    }
+    if (!lsp_pair_group && must_exist) {
+        ctl_fatal("lsp_pair_group not found for %s: '%s'",
+                  is_uuid ? "UUID" : "name", id);
+    }
+
+    return lsp_pair_group;
+}
+
+static const struct nbrec_logical_port_chain_classifier *
+lsp_chain_classifier_by_name_or_uuid(struct ctl_context *ctx, const char *id,
+                                const bool must_exist)
+{
+    const struct nbrec_logical_port_chain_classifier
+                                *lsp_chain_classifier = NULL;
+    bool is_uuid = false;
+    struct uuid lsp_chain_classifier_uuid;
+
+    if (uuid_from_string(&lsp_chain_classifier_uuid, id)) {
+        is_uuid = true;
+        lsp_chain_classifier =
+          nbrec_logical_port_chain_classifier_get_for_uuid(ctx->idl,
+                                              &lsp_chain_classifier_uuid);
+    }
+
+    if (!lsp_chain_classifier) {
+        NBREC_LOGICAL_PORT_CHAIN_CLASSIFIER_FOR_EACH(lsp_chain_classifier,
+                                                      ctx->idl) {
+            if (!strcmp(lsp_chain_classifier->name, id)) {
+                break;
+            }
+        }
+    }
+    if (!lsp_chain_classifier && must_exist) {
+        ctl_fatal("lsp_chain_classifier not found for %s: '%s'",
+                  is_uuid ? "UUID" : "name", id);
+    }
+
+    return lsp_chain_classifier;
+}
+
+static const struct nbrec_logical_port_pair *
+lsp_pair_by_name_or_uuid(struct ctl_context *ctx, const char *id,
+                          const bool must_exist)
+{
+    const struct nbrec_logical_port_pair *lsp_pair = NULL;
+    bool is_uuid = false;
+    struct uuid lsp_pair_uuid;
+
+    if (uuid_from_string(&lsp_pair_uuid, id)) {
+        is_uuid = true;
+        lsp_pair = nbrec_logical_port_pair_get_for_uuid(ctx->idl,
+                                                        &lsp_pair_uuid);
+    }
+
+    if (!lsp_pair) {
+        NBREC_LOGICAL_PORT_PAIR_FOR_EACH(lsp_pair, ctx->idl) {
+            if (!strcmp(lsp_pair->name, id)) {
+                break;
+            }
+        }
+    }
+    if (!lsp_pair && must_exist) {
+        ctl_fatal("lsp_pair not found for %s: '%s'",
+                  is_uuid ? "UUID" : "name", id);
+    }
+
+    return lsp_pair;
+}
+
+
+static void
+nbctl_lsp_chain_add(struct ctl_context *ctx)
+{
+    const struct nbrec_logical_switch *lswitch;
+
+    lswitch = ls_by_name_or_uuid(ctx, ctx->argv[1], true);
+    const char *lsp_chain_name = ctx->argc > 2 ? ctx->argv[2] : NULL;
+
+    const bool may_exist = shash_find(&ctx->options, "--may-exist") != NULL;
+    const bool add_duplicate = shash_find(&ctx->options,
+                                 "--add-duplicate") != NULL;
+    if (may_exist && add_duplicate) {
+        ctl_fatal("--may-exist and --add-duplicate may not be used together");
+    }
+
+    if (lsp_chain_name) {
+        if (!add_duplicate) {
+            const struct nbrec_logical_port_chain *lsp_chain;
+            NBREC_LOGICAL_PORT_CHAIN_FOR_EACH(lsp_chain, ctx->idl) {
+                if (!strcmp(lsp_chain->name, lsp_chain_name)) {
+                    if (may_exist) {
+                        return;
+                    }
+                    ctl_fatal("%s: a lsp_chain with this name already exists",
+                              lsp_chain_name);
+                }
+            }
+        }
+    } else if (may_exist) {
+        ctl_fatal("--may-exist requires specifying a name");
+    } else if (add_duplicate) {
+        ctl_fatal("--add-duplicate requires specifying a name");
+    }
+
+    struct nbrec_logical_port_chain *lsp_chain;
+    lsp_chain = nbrec_logical_port_chain_insert(ctx->txn);
+    if (lsp_chain_name) {
+        nbrec_logical_port_chain_set_name(lsp_chain, lsp_chain_name);
+    }
+
+    /* Insert the logical port-chain into the logical switch. */
+
+    nbrec_logical_switch_verify_port_chains(lswitch);
+    struct nbrec_logical_port_chain  **new_port_chain =
+              xmalloc(sizeof *new_port_chain * (lswitch->n_port_chains + 1));
+    memcpy(new_port_chain, lswitch->port_chains,
+              sizeof *new_port_chain * lswitch->n_port_chains);
+    new_port_chain[lswitch->n_port_chains] =
+              CONST_CAST(struct nbrec_logical_port_chain *, lsp_chain);
+    nbrec_logical_switch_set_port_chains(lswitch, new_port_chain,
+              lswitch->n_port_chains + 1);
+    free(new_port_chain);
+}
+
+/* Removes lswitch->pair_chain[idx]'. */
+static void
+remove_lsp_chain(const struct nbrec_logical_switch *lswitch, size_t idx)
+{
+    const struct nbrec_logical_port_chain *lsp_chain =
+                                                    lswitch->port_chains[idx];
+
+    /* First remove 'lsp-chain' from the array of port-chains.
+     * This is what will actually cause the logical port-chain to be deleted
+     * when the transaction is sent to the database server
+     * (due to garbage collection). */
+    struct nbrec_logical_port_chain **new_port_chain
+        = xmemdup(lswitch->port_chains,
+                   sizeof *new_port_chain * lswitch->n_port_chains);
+    new_port_chain[idx] = new_port_chain[lswitch->n_port_chains - 1];
+    nbrec_logical_switch_verify_port_chains(lswitch);
+    nbrec_logical_switch_set_port_chains(lswitch, new_port_chain,
+                                          lswitch->n_port_chains - 1);
+    free(new_port_chain);
+
+    /* Delete 'lsp-chain' from the IDL.  This won't have a real effect on the
+     * database server (the IDL will suppress it in fact) but it means that it
+     * won't show up when we iterate with
+     * NBREC_LOGICAL_PORT_CHAIN_FOR_EACH later. */
+    nbrec_logical_port_chain_delete(lsp_chain);
+}
+
+static void
+nbctl_lsp_chain_del(struct ctl_context *ctx)
+{
+    const struct nbrec_logical_port_chain *lsp_chain;
+    const bool must_exist = !shash_find(&ctx->options, "--if-exists");
+
+    lsp_chain = lsp_chain_by_name_or_uuid(ctx, ctx->argv[1], must_exist);
+    if (!lsp_chain) {
+        return;
+    }
+
+    /* Find the lswitch that contains 'port-chain', then delete it. */
+    const struct nbrec_logical_switch *lswitch;
+    NBREC_LOGICAL_SWITCH_FOR_EACH (lswitch, ctx->idl) {
+        for (size_t i = 0; i < lswitch->n_port_chains; i++) {
+            if (lswitch->port_chains[i] == lsp_chain) {
+                remove_lsp_chain(lswitch,i);
+                return;
+            }
+        }
+    }
+}
+
+static void
+print_lsp_chain_entry(struct ctl_context *ctx,
+                      const struct nbrec_logical_switch *lswitch,
+                      const char *chain_name_filter,
+                      const bool show_switch_name)
+{
+    struct smap lsp_chains;
+    size_t i;
+
+    smap_init(&lsp_chains);
+    for (i = 0; i < lswitch->n_port_chains; i++) {
+        const struct nbrec_logical_port_chain *lsp_chain =
+                                                    lswitch->port_chains[i];
+        if (chain_name_filter && strcmp(chain_name_filter, lsp_chain->name)) {
+            continue;
+        }
+        if (show_switch_name) {
+            smap_add_format(&lsp_chains, lsp_chain->name, UUID_FMT " (%s:%s)",
+                            UUID_ARGS(&lsp_chain->header_.uuid),
+                            lswitch->name, lsp_chain->name);
+        } else {
+            smap_add_format(&lsp_chains, lsp_chain->name, UUID_FMT " (%s)",
+                            UUID_ARGS(&lsp_chain->header_.uuid),
+                            lsp_chain->name);
+        }
+    }
+
+    const struct smap_node **nodes = smap_sort(&lsp_chains);
+    for (i = 0; i < smap_count(&lsp_chains); i++) {
+        const struct smap_node *node = nodes[i];
+        ds_put_format(&ctx->output, "%s\n", node->value);
+    }
+    smap_destroy(&lsp_chains);
+    free(nodes);
+}
+
+static void
+nbctl_lsp_chain_list(struct ctl_context *ctx)
+{
+    const char *id = ctx->argc > 1 ? ctx->argv[1] : NULL;
+    const char *chain_name_filter = ctx->argc > 2 ? ctx->argv[2] : NULL;
+    const struct nbrec_logical_switch *lswitch;
+
+    if (id) {
+        lswitch = ls_by_name_or_uuid(ctx, id, true);
+        print_lsp_chain_entry(ctx, lswitch, chain_name_filter, false);
+    } else {
+        NBREC_LOGICAL_SWITCH_FOR_EACH(lswitch, ctx->idl) {
+            if (lswitch->n_port_chains == 0) {
+                continue;
+            }
+            print_lsp_chain_entry(ctx, lswitch, chain_name_filter, true);
+        }
+    }
+}
+
+static void
+print_lsp_chain(const struct nbrec_logical_port_chain *lsp_chain,
+                  struct ctl_context *ctx)
+{
+    ds_put_format(&ctx->output, "lsp-chain "UUID_FMT" (%s)\n",
+                  UUID_ARGS(&lsp_chain->header_.uuid), lsp_chain->name);
+    for (size_t i = 0; i < lsp_chain->n_port_pair_groups; i++) {
+        const struct nbrec_logical_port_pair_group *lsp_pair_group
+            = lsp_chain->port_pair_groups[i];
+        ds_put_format(&ctx->output, "    lsp-pair-group %s\n",
+                       lsp_pair_group->name);
+        for (size_t j = 0; j < lsp_pair_group->n_port_pairs; j++) {
+            const struct nbrec_logical_port_pair *lsp_pair =
+                                                lsp_pair_group->port_pairs[j];
+            ds_put_format(&ctx->output,
+                           "        lsp-pair %s\n", lsp_pair->name);
+
+            const struct nbrec_logical_switch_port *linport = lsp_pair->inport;
+            if (linport) {
+                ds_put_format(&ctx->output,
+                           "            lsp-pair inport "UUID_FMT" (%s)\n",
+                            UUID_ARGS(&linport->header_.uuid), linport->name);
+            }
+
+            const struct nbrec_logical_switch_port *loutport =
+                                                            lsp_pair->outport;
+            if (loutport) {
+                ds_put_format(&ctx->output,
+                          "            lsp-pair outport "UUID_FMT" (%s)\n",
+                          UUID_ARGS(&loutport->header_.uuid), loutport->name);
+            }
+        }
+    }
+}
+
+static void
+nbctl_lsp_chain_show(struct ctl_context *ctx)
+{
+    const struct nbrec_logical_port_chain *lsp_chain;
+
+    if (ctx->argc == 2) {
+        lsp_chain = lsp_chain_by_name_or_uuid(ctx, ctx->argv[1], false);
+        if (lsp_chain) {
+            print_lsp_chain(lsp_chain, ctx);
+        }
+    } else {
+        NBREC_LOGICAL_PORT_CHAIN_FOR_EACH(lsp_chain, ctx->idl) {
+            print_lsp_chain(lsp_chain, ctx);
+        }
+    }
+}
+/* End of port-chain operations */
+static int
+parse_sortkey(const char *arg)
+{
+    /* Validate sortkey. */
+    int64_t sortkey;
+    if (!ovs_scan(arg, "%"SCNd64, &sortkey)
+        || sortkey < 0 || sortkey > 127) {
+        ctl_fatal("%s: sortkey must in range 0...127", arg);
+    }
+    return sortkey;
+}
+/*
+ * Port Pair Groups CLI Functions
+ */
+static void
+nbctl_lsp_pair_group_add(struct ctl_context *ctx)
+{
+    const struct nbrec_logical_port_pair_group *lsp_pair_group;
+    const char *ppg_name = ctx->argc >= 3 ? ctx->argv[2] : NULL;
+
+    const bool may_exist = shash_find(&ctx->options, "--may-exist") != NULL;
+    const bool add_duplicate = shash_find(&ctx->options,
+                                           "--add-duplicate") != NULL;
+    if (may_exist && add_duplicate) {
+        ctl_fatal("--may-exist and --add-duplicate may not be used together");
+    }
+
+    if (ppg_name) {
+        if (!add_duplicate) {
+            NBREC_LOGICAL_PORT_PAIR_GROUP_FOR_EACH(lsp_pair_group, ctx->idl) {
+                if (!strcmp(lsp_pair_group->name, ppg_name)) {
+                    if (may_exist) {
+                        return;
+                    }
+                    ctl_fatal("%s: an lsp_port_pair_group with this \
+                               name already exists", ppg_name);
+                }
+            }
+        }
+    } else if (may_exist) {
+        ctl_fatal("--may-exist requires specifying a name");
+    } else if (add_duplicate) {
+        ctl_fatal("--add-duplicate requires specifying a name");
+    }
+
+    /* check lsp_chain exists */
+    const struct nbrec_logical_port_chain *lsp_chain;
+    lsp_chain = lsp_chain_by_name_or_uuid(ctx, ctx->argv[1], true);
+    if (!lsp_chain) {
+        return;
+    }
+
+    /* create the logical port-pair-group. */
+    lsp_pair_group = nbrec_logical_port_pair_group_insert(ctx->txn);
+    if (ppg_name) {
+        nbrec_logical_port_pair_group_set_name(lsp_pair_group, ctx->argv[2]);
+    }
+    nbrec_logical_port_chain_verify_port_pair_groups(lsp_chain);
+    /*
+    * Create a sort key for the port pair groups
+    */
+    int64_t sortkey = (int64_t) lsp_chain->n_port_pair_groups + 1;
+    if (ctx->argc >= 4) {
+         sortkey = (int64_t) parse_sortkey(ctx->argv[3]);
+    }
+    nbrec_logical_port_pair_group_set_sortkey(lsp_pair_group, sortkey);
+    /*
+     * Insert the logical port-pair-group into the logical chain.
+     */
+    struct nbrec_logical_port_pair_group  **new_port_pair_group =
+                              xmalloc(sizeof *new_port_pair_group *
+                              (lsp_chain->n_port_pair_groups + 1));
+    memcpy(new_port_pair_group, lsp_chain->port_pair_groups,
+                  sizeof *new_port_pair_group * lsp_chain->n_port_pair_groups);
+    new_port_pair_group[lsp_chain->n_port_pair_groups] =
+        CONST_CAST(struct nbrec_logical_port_pair_group *,lsp_pair_group);
+    nbrec_logical_port_chain_set_port_pair_groups(lsp_chain,
+                     new_port_pair_group,
+                     lsp_chain->n_port_pair_groups + 1);
+    free(new_port_pair_group);
+}
+
+/* Removes lsp-pair-group 'lsp_chain->port_pair_group[idx]'. */
+static void
+remove_lsp_pair_group(const struct nbrec_logical_port_chain *lsp_chain,
+                      size_t idx)
+{
+    const struct nbrec_logical_port_pair_group *lsp_pair_group =
+                                              lsp_chain->port_pair_groups[idx];
+
+    /* First remove 'lsp-pair-group' from the array of port-pair-groups.
+     * This is what will actually cause the logical port-pair-group to be
+     * deleted when the transaction is sent to the database server
+     * (due to garbage collection). */
+    struct nbrec_logical_port_pair_group **new_port_pair_group
+        = xmemdup(lsp_chain->port_pair_groups,
+            sizeof *new_port_pair_group * lsp_chain->n_port_pair_groups);
+    new_port_pair_group[idx] =
+            new_port_pair_group[lsp_chain->n_port_pair_groups - 1];
+    nbrec_logical_port_chain_verify_port_pair_groups(lsp_chain);
+    nbrec_logical_port_chain_set_port_pair_groups(lsp_chain,
+            new_port_pair_group, lsp_chain->n_port_pair_groups - 1);
+    free(new_port_pair_group);
+
+    /* Delete 'lsp-pair-group' from the IDL.  This won't have a real
+     * effect on the database server (the IDL will suppress it in fact) but
+     * it means that it won't show up when we iterate with
+     * NBREC_LOGICAL_PORT_PAIR_GROUP_FOR_EACH later. */
+    nbrec_logical_port_pair_group_delete(lsp_pair_group);
+}
+
+static void
+nbctl_lsp_pair_group_del(struct ctl_context *ctx)
+{
+    const struct nbrec_logical_port_pair_group *lsp_pair_group;
+    const bool must_exist = !shash_find(&ctx->options, "--if-exists");
+
+    lsp_pair_group = lsp_pair_group_by_name_or_uuid(ctx, ctx->argv[1],
+                                                     must_exist);
+    if (!lsp_pair_group) {
+        return;
+    }
+
+    /* Find the port-chain that contains 'port-pair-group', then delete it. */
+    const struct nbrec_logical_port_chain *lsp_chain;
+    NBREC_LOGICAL_PORT_CHAIN_FOR_EACH (lsp_chain, ctx->idl) {
+        for (size_t i = 0; i < lsp_chain->n_port_pair_groups; i++) {
+            if (lsp_chain->port_pair_groups[i] == lsp_pair_group) {
+                remove_lsp_pair_group(lsp_chain,i);
+                return;
+            }
+        }
+    }
+    if (must_exist) {
+        ctl_fatal("logical port-pair-group %s is not part of any\
+                    logical port-chain", ctx->argv[1]);
+    }
+}
+
+static void
+nbctl_lsp_pair_group_list(struct ctl_context *ctx)
+{
+    const char *id = ctx->argv[1];
+    const struct nbrec_logical_port_chain *lsp_chain;
+    struct smap lsp_pair_groups;
+    size_t i;
+    lsp_chain = lsp_chain_by_name_or_uuid(ctx, id, true);
+    if (!lsp_chain) {
+        return;
+    }
+
+    smap_init(&lsp_pair_groups);
+    for (i = 0; i < lsp_chain->n_port_pair_groups; i++) {
+        const struct nbrec_logical_port_pair_group *lsp_pair_group =
+                       lsp_chain->port_pair_groups[i];
+        smap_add_format(&lsp_pair_groups, lsp_pair_group->name,
+                         UUID_FMT " (%s: %5"PRId64")",
+                         UUID_ARGS(&lsp_pair_group->header_.uuid),
+                         lsp_pair_group->name, lsp_pair_group->sortkey );
+    }
+    const struct smap_node **nodes = smap_sort(&lsp_pair_groups);
+    for (i = 0; i < smap_count(&lsp_pair_groups); i++) {
+        const struct smap_node *node = nodes[i];
+        ds_put_format(&ctx->output, "%s\n", node->value);
+    }
+    smap_destroy(&lsp_pair_groups);
+    free(nodes);
+}
+
+static void
+nbctl_lsp_pair_group_add_port_pair(struct ctl_context *ctx)
+{
+    const struct nbrec_logical_port_pair_group *lsp_pair_group;
+    const struct nbrec_logical_port_pair *lsp_pair;
+    const bool may_exist = shash_find(&ctx->options, "--may-exist") != NULL;
+
+    lsp_pair_group = lsp_pair_group_by_name_or_uuid(ctx, ctx->argv[1], true);
+    if (!lsp_pair_group) {
+        return;
+    }
+
+    /* Check that port-pair exists  */
+    lsp_pair = lsp_pair_by_name_or_uuid(ctx, ctx->argv[2], true);
+    if (!lsp_pair) {
+        return;
+    }
+
+    /* Do not add port pair more than once in a given port-pair-group */
+    for (size_t i = 0; i < lsp_pair_group->n_port_pairs; i++) {
+        if (lsp_pair_group->port_pairs[i] == lsp_pair) {
+            if (!may_exist) {
+                ctl_fatal("lsp_pair: %s is already added to\
+                           port-pair-group %s\n", ctx->argv[2], ctx->argv[1]);
+            }
+            return;
+        }
+    }
+
+    /* Insert the logical port-pair into the logical port-pair-group. */
+    nbrec_logical_port_pair_group_verify_port_pairs(lsp_pair_group);
+    struct nbrec_logical_port_pair  **new_port_pair =
+          xmalloc(sizeof *new_port_pair * (lsp_pair_group->n_port_pairs + 1));
+    memcpy(new_port_pair, lsp_pair_group->port_pairs,
+          sizeof *new_port_pair * lsp_pair_group->n_port_pairs);
+    new_port_pair[lsp_pair_group->n_port_pairs] =
+          CONST_CAST(struct nbrec_logical_port_pair *, lsp_pair);
+    nbrec_logical_port_pair_group_set_port_pairs(lsp_pair_group,
+          new_port_pair, lsp_pair_group->n_port_pairs + 1);
+    free(new_port_pair);
+}
+
+/* Removes port-pair from port-pair-groiup but does not delete it'. */
+static void
+remove_lsp_pair_from_port_pair_group(
+      const struct nbrec_logical_port_pair_group *lsp_pair_group, size_t idx)
+{
+
+    /* First remove 'lsp-pair' from the array of port-pairs.  This is
+     * what will actually cause the logical port-pair to be deleted when the
+     * transaction is sent to the database server
+     * (due to garbage collection). */
+    struct nbrec_logical_port_pair **new_port_pair
+        = xmemdup(lsp_pair_group->port_pairs, sizeof *new_port_pair *
+                  lsp_pair_group->n_port_pairs);
+    new_port_pair[idx] = new_port_pair[lsp_pair_group->n_port_pairs - 1];
+    nbrec_logical_port_pair_group_verify_port_pairs(lsp_pair_group);
+    nbrec_logical_port_pair_group_set_port_pairs(lsp_pair_group, new_port_pair,
+                  lsp_pair_group->n_port_pairs - 1);
+    free(new_port_pair);
+
+    /* Do not delete actual port-pair as they are owned by a
+     * lswitch and can be reused. */
+}
+
+static void
+nbctl_lsp_pair_group_del_port_pair(struct ctl_context *ctx)
+{
+    const struct nbrec_logical_port_pair *lsp_pair;
+    const struct nbrec_logical_port_pair_group *lsp_pair_group;
+    const bool must_exist = !shash_find(&ctx->options, "--if-exists");
+
+
+    lsp_pair_group = lsp_pair_group_by_name_or_uuid(ctx, ctx->argv[1],
+                                                     must_exist);
+    if (!lsp_pair_group) {
+        return;
+    }
+    lsp_pair = lsp_pair_by_name_or_uuid(ctx, ctx->argv[2], must_exist);
+    if (!lsp_pair) {
+        return;
+    }
+    /* Find the port-pair_group that contains 'port-pair', then delete it. */
+
+    for (size_t i = 0; i < lsp_pair_group->n_port_pairs; i++) {
+      if (lsp_pair_group->port_pairs[i] == lsp_pair) {
+        remove_lsp_pair_from_port_pair_group(lsp_pair_group,i);
+        return;
+      }
+    }
+    if (must_exist) {
+        ctl_fatal("logical port-pair %s is not part of any logical switch",
+                  ctx->argv[1]);
+    }
+}
+/* End of port-pair-group operations */
+
+/*
+ * Port Chain Classifier CLI Functions
+ */
+static bool
+parse_chain_direction(const char *direction)
+{
+    if (strcasecmp(direction, "uni-directional")) {
+        return true;
+    } else if (strcasecmp(direction, "bi-directional")) {
+        return true;
+    } else {
+        ctl_fatal("%s: direction must be \"uni-directional\" \
+                     or \"bi-directional\"", direction);
+    }
+}
+static bool
+parse_chain_path(const char *path){
+    if (strcasecmp(path, "entry-lport")) {
+        return true;
+    } else if (strcasecmp(path, "exit-lport")) {
+        return true;
+    } else {
+        ctl_fatal("%s: path must be \"entry-lport\" \
+                     or \"exit-lport\"", path);
+    }
+}
+static void
+nbctl_lsp_chain_classifier_add(struct ctl_context *ctx)
+{
+    const struct nbrec_logical_switch *lswitch;
+    const struct nbrec_logical_port_chain *lsp_chain;
+    const struct nbrec_logical_switch_port *lsp_input, *lsp_exist;
+    struct nbrec_logical_port_chain_classifier *lsp_chain_classifier;
+
+    lswitch = ls_by_name_or_uuid(ctx, ctx->argv[1], true);
+    lsp_chain = lsp_chain_by_name_or_uuid(ctx, ctx->argv[2], true);
+    lsp_input = lsp_by_name_or_uuid(ctx, ctx->argv[3], true);
+     /*
+     * Check that this port is not already in use be an existing classifier
+     * The current implementation is limited to attaching a single chain
+     * to a port.
+     */
+    NBREC_LOGICAL_SWITCH_FOR_EACH(lswitch, ctx->idl) {
+      for (int k=0; k < lswitch->n_port_chain_classifiers; k++) {
+      lsp_chain_classifier =  lswitch->port_chain_classifiers[k];
+      lsp_exist = lsp_chain_classifier->port;
+      if (uuid_equals(&lsp_exist->header_.uuid, &lsp_input->header_.uuid)) {
+                    ctl_fatal("%s: lsp is already assigned a chain",
+                               lsp_input->name);
+                }
+              }
+    }
+    lswitch = ls_by_name_or_uuid(ctx, ctx->argv[1], true);
+    const char *lsp_chain_classifier_name =
+                                       ctx->argc > 6 ? ctx->argv[6] : NULL;
+    const char *lsp_chain_classifier_match =
+                                       ctx->argc > 7 ? ctx->argv[7] : NULL;
+
+    const bool may_exist = shash_find(&ctx->options, "--may-exist") != NULL;
+    const bool add_duplicate = shash_find(&ctx->options,
+                                 "--add-duplicate") != NULL;
+    if (may_exist && add_duplicate) {
+        ctl_fatal("--may-exist and --add-duplicate may not be used together");
+    }
+    if (lsp_chain_classifier_name) {
+        if (!add_duplicate) {
+            const struct nbrec_logical_port_chain_classifier
+                                                      *lsp_chain_classifier;
+            NBREC_LOGICAL_PORT_CHAIN_CLASSIFIER_FOR_EACH(lsp_chain_classifier,
+                                                  ctx->idl) {
+                if (!strcmp(lsp_chain_classifier->name,
+                                lsp_chain_classifier_name)) {
+                    if (may_exist) {
+                        return;
+                    }
+                    ctl_fatal("%s: an lsp_chain_classifier \
+                               with this name already exists",
+                               lsp_chain_classifier_name);
+                }
+            }
+        }
+    } else if (may_exist) {
+        ctl_fatal("--may-exist requires specifying a name");
+    } else if (add_duplicate) {
+        ctl_fatal("--add-duplicate requires specifying a name");
+    }
+    lsp_chain_classifier =
+                       nbrec_logical_port_chain_classifier_insert(ctx->txn);
+
+    nbrec_logical_port_chain_classifier_set_chain(
+                                        lsp_chain_classifier, lsp_chain);
+    nbrec_logical_port_chain_classifier_set_port(
+                                        lsp_chain_classifier, lsp_input);
+    if (parse_chain_direction(ctx->argv[4])) {
+    nbrec_logical_port_chain_classifier_set_direction(
+                                        lsp_chain_classifier, ctx->argv[4]);
+    }
+    if (parse_chain_path(ctx->argv[5])) {
+          nbrec_logical_port_chain_classifier_set_path(
+                                        lsp_chain_classifier, ctx->argv[5]);
+    }
+    if (lsp_chain_classifier_match != NULL) {
+           nbrec_logical_port_chain_classifier_set_match(
+                                        lsp_chain_classifier,
+                                        lsp_chain_classifier_match);
+         }
+    if (lsp_chain_classifier_name != NULL) {
+           nbrec_logical_port_chain_classifier_set_name(lsp_chain_classifier,
+                                        lsp_chain_classifier_name);
+    }
+    /* Insert the logical port-chain into the logical switch. */
+    nbrec_logical_switch_verify_port_chain_classifiers(lswitch);
+
+    struct nbrec_logical_port_chain_classifier  **new_port_chain_classifier =
+               xmalloc( sizeof *new_port_chain_classifier *
+              (lswitch->n_port_chain_classifiers + 1));
+    memcpy(new_port_chain_classifier, lswitch->port_chain_classifiers,
+              sizeof *new_port_chain_classifier *
+              lswitch->n_port_chain_classifiers);
+    new_port_chain_classifier[lswitch->n_port_chain_classifiers] =
+              CONST_CAST(struct nbrec_logical_port_chain_classifier *,
+                lsp_chain_classifier);
+    nbrec_logical_switch_set_port_chain_classifiers(lswitch,
+              new_port_chain_classifier,
+              lswitch->n_port_chain_classifiers + 1);
+    free(new_port_chain_classifier);
+}
+
+/* Removes lsp-chain-classifier from logical switch. */
+static void
+remove_lsp_chain_classifier(const struct nbrec_logical_switch *lswitch,
+                                size_t idx)
+{
+    const struct nbrec_logical_port_chain_classifier *lsp_chain_classifier =
+                                      lswitch->port_chain_classifiers[idx];
+
+    /* First remove 'lsp-chain' from the array of port-chains.
+     * This is what will actually cause the logical port-chain to be deleted
+     * when the transaction is sent to the database server
+     * (due to garbage collection). */
+    struct nbrec_logical_port_chain_classifier **new_port_chain_classifier
+        = xmemdup(lswitch->port_chain_classifiers,
+                   sizeof *new_port_chain_classifier *
+                    lswitch->n_port_chain_classifiers);
+    new_port_chain_classifier[idx] =
+             new_port_chain_classifier[lswitch->n_port_chain_classifiers - 1];
+    nbrec_logical_switch_verify_port_chain_classifiers(lswitch);
+    nbrec_logical_switch_set_port_chain_classifiers(lswitch,
+                      new_port_chain_classifier,
+                      lswitch->n_port_chain_classifiers - 1);
+    free(new_port_chain_classifier);
+
+    /* Delete 'lsp-chain' from the IDL.  This won't have a real effect on the
+     * database server (the IDL will suppress it in fact) but it means that it
+     * won't show up when we iterate with
+     * NBREC_LOGICAL_PORT_CHAIN_FOR_EACH later. */
+    nbrec_logical_port_chain_classifier_delete(lsp_chain_classifier);
+}
+
+static void
+nbctl_lsp_chain_classifier_del(struct ctl_context *ctx)
+{
+    const struct nbrec_logical_port_chain_classifier *lsp_chain_classifier;
+    const bool must_exist = !shash_find(&ctx->options, "--if-exists");
+
+    lsp_chain_classifier =
+         lsp_chain_classifier_by_name_or_uuid(ctx, ctx->argv[1], must_exist);
+    if (!lsp_chain_classifier) {
+        return;
+    }
+
+    /* Find the lswitch that contains 'port-chain', then delete it. */
+    const struct nbrec_logical_switch *lswitch;
+    NBREC_LOGICAL_SWITCH_FOR_EACH (lswitch, ctx->idl) {
+        for (size_t i = 0; i < lswitch->n_port_chain_classifiers; i++) {
+            if (lswitch->port_chain_classifiers[i] == lsp_chain_classifier) {
+                remove_lsp_chain_classifier(lswitch,i);
+                return;
+            }
+        }
+    }
+}
+static void
+print_lsp_chain_classifier(struct ctl_context *ctx,
+                      const struct nbrec_logical_switch *lswitch,
+                      const bool show_switch_name)
+{
+    struct smap lsp_chain_classifiers;
+    size_t i;
+    smap_init(&lsp_chain_classifiers);
+    /*
+    * Loop over all chain classifiers
+    */
+    for (i = 0; i < lswitch->n_port_chain_classifiers; i++) {
+        const struct nbrec_logical_port_chain_classifier
+                 *lsp_chain_classifier =  lswitch->port_chain_classifiers[i];
+
+        if (show_switch_name) {
+            smap_add_format(&lsp_chain_classifiers,
+                            lsp_chain_classifier->name, UUID_FMT " (%s:%s)",
+                            UUID_ARGS(&lsp_chain_classifier->header_.uuid),
+                            lswitch->name, lsp_chain_classifier->name);
+        } else {
+            smap_add_format(&lsp_chain_classifiers,
+                            lsp_chain_classifier->name, UUID_FMT " (%s)",
+                            UUID_ARGS(&lsp_chain_classifier->header_.uuid),
+                            lsp_chain_classifier->name);
+        }
+    }
+
+    const struct smap_node **nodes = smap_sort(&lsp_chain_classifiers);
+    for (i = 0; i < smap_count(&lsp_chain_classifiers); i++) {
+        const struct smap_node *node = nodes[i];
+        ds_put_format(&ctx->output, "%s\n", node->value);
+    }
+    smap_destroy(&lsp_chain_classifiers);
+    free(nodes);
+}
+
+static void
+nbctl_lsp_chain_classifier_list(struct ctl_context *ctx)
+{
+    const char *id = ctx->argc > 1 ? ctx->argv[1] : NULL;
+    const struct nbrec_logical_switch *lswitch;
+    if (id) {
+        lswitch = ls_by_name_or_uuid(ctx, id, true);
+        print_lsp_chain_classifier(ctx, lswitch, false);
+    } else {
+        NBREC_LOGICAL_SWITCH_FOR_EACH(lswitch, ctx->idl) {
+            if (lswitch->n_port_chain_classifiers > 0) {
+                 print_lsp_chain_classifier(ctx, lswitch, true);
+          }
+        }
+    }
+}
+
+static void
+print_lsp_chain_classifier_entry(struct ctl_context *ctx,
+                      const struct nbrec_logical_switch *lswitch,
+                      const char *chain_classifier_name_filter,
+                      const bool show_switch_name)
+{
+    size_t i;
+    /*
+    * Loop over all chain classifiers
+    */
+    for (i = 0; i < lswitch->n_port_chain_classifiers; i++) {
+        const struct nbrec_logical_port_chain_classifier
+                 *lsp_chain_classifier =  lswitch->port_chain_classifiers[i];
+        const struct nbrec_logical_port_chain *lsp_chain;
+        const struct nbrec_logical_switch_port *lsp;
+
+
+        lsp_chain = lsp_chain_classifier->chain;
+        lsp = lsp_chain_classifier->port;
+
+        if (chain_classifier_name_filter != NULL &&
+                 strcmp(chain_classifier_name_filter,
+                 lsp_chain_classifier->name) !=0) {
+            continue;
+        }
+        if (show_switch_name) {
+          ds_put_format(&ctx->output,
+                            "\nls-chain-classifier: " UUID_FMT " (%s:%s)\n",
+                            UUID_ARGS(&lsp_chain_classifier->header_.uuid),
+                            lswitch->name, lsp_chain_classifier->name);
+        } else {
+          ds_put_format(&ctx->output,"ls-chain-classifier: "UUID_FMT" (%s)\n",
+                            UUID_ARGS(&lsp_chain_classifier->header_.uuid),
+                            lsp_chain_classifier->name);
+        }
+        ds_put_format(&ctx->output,"     lsp-chain: "UUID_FMT " (%s)\n",
+                            UUID_ARGS(&lsp_chain->header_.uuid),
+                            lsp_chain->name);
+        ds_put_format(&ctx->output, "     lsp: "UUID_FMT " (%s)\n",
+                            UUID_ARGS(&lsp->header_.uuid),
+                            lsp->name);
+        ds_put_format(&ctx->output, "     Flow Direction: %s\n",
+                            lsp_chain_classifier->direction);
+        ds_put_format(&ctx->output, "     Flow Type: %s\n",
+                            lsp_chain_classifier->path);
+        ds_put_format(&ctx->output, "     Match Statement: %s\n",
+                            lsp_chain_classifier->match);
+    }
+}
+
+static void
+nbctl_lsp_chain_classifier_show(struct ctl_context *ctx)
+{
+    const char *id = ctx->argc > 1 ? ctx->argv[1] : NULL;
+    const char *chain_classifier_name_filter =
+                                        ctx->argc > 2 ? ctx->argv[2] : NULL;
+    const struct nbrec_logical_switch *lswitch;
+
+    if (id) {
+        lswitch = ls_by_name_or_uuid(ctx, id, true);
+        print_lsp_chain_classifier_entry(ctx, lswitch,
+                                 chain_classifier_name_filter, false);
+    } else {
+        NBREC_LOGICAL_SWITCH_FOR_EACH(lswitch, ctx->idl) {
+            if (lswitch->n_port_chain_classifiers > 0) {
+                 print_lsp_chain_classifier_entry(ctx, lswitch,
+                                  chain_classifier_name_filter, true);
+          }
+        }
+    }
+}
+/* End of port-chain-classifier operations  */
+/*
+ * port-pair operations
+ */
+static void
+nbctl_lsp_pair_add(struct ctl_context *ctx)
+{
+    const struct nbrec_logical_switch *lswitch;
+    const struct nbrec_logical_switch_port *lsp_in,*lsp_out;
+    const struct nbrec_logical_port_pair *lsp_pair;
+
+    const bool may_exist = shash_find(&ctx->options, "--may-exist") != NULL;
+    const bool add_duplicate = shash_find(&ctx->options,
+                                           "--add-duplicate") != NULL;
+
+    lswitch = ls_by_name_or_uuid(ctx, ctx->argv[1], true);
+    lsp_in = lsp_by_name_or_uuid(ctx, ctx->argv[2], true);
+    lsp_out = lsp_by_name_or_uuid(ctx, ctx->argv[3], true);
+
+    const char *lsp_pair_name = ctx->argc >= 5 ? ctx->argv[4] : NULL;
+    if (may_exist && add_duplicate) {
+        ctl_fatal("--may-exist and --add-duplicate may not be used together");
+    }
+
+    if (lsp_pair_name) {
+        if (!add_duplicate) {
+            NBREC_LOGICAL_PORT_PAIR_FOR_EACH(lsp_pair, ctx->idl) {
+                if (!strcmp(lsp_pair->name, lsp_pair_name)) {
+                    if (may_exist) {
+                        return;
+                    }
+                    ctl_fatal("%s: an lsp_pair with this name already exists",
+                              lsp_pair_name);
+                }
+            }
+        }
+    } else if (may_exist) {
+        ctl_fatal("--may-exist requires specifying a name");
+    } else if (add_duplicate) {
+        ctl_fatal("--add-duplicate requires specifying a name");
+    }
+
+    /* create the logical port-pair. */
+    lsp_pair = nbrec_logical_port_pair_insert(ctx->txn);
+    nbrec_logical_port_pair_set_inport(lsp_pair, lsp_in);
+    nbrec_logical_port_pair_set_outport(lsp_pair, lsp_out);
+    if (lsp_pair_name) {
+        nbrec_logical_port_pair_set_name(lsp_pair, lsp_pair_name);
+    }
+
+    /* Insert the logical port-pair into the logical port-pair-group. */
+    nbrec_logical_switch_verify_port_pairs(lswitch);
+    struct nbrec_logical_port_pair  **new_port_pair =
+                 xmalloc(sizeof *new_port_pair * (lswitch->n_port_pairs + 1));
+    memcpy(new_port_pair, lswitch->port_pairs,
+                 sizeof *new_port_pair * lswitch->n_port_pairs);
+    new_port_pair[lswitch->n_port_pairs] =
+                 CONST_CAST(struct nbrec_logical_port_pair *, lsp_pair);
+    nbrec_logical_switch_set_port_pairs(lswitch, new_port_pair,
+                 lswitch->n_port_pairs + 1);
+    free(new_port_pair);
+}
+/* Removes lswitch->pair_pair[idx]'. */
+static void
+remove_lsp_pair(const struct nbrec_logical_switch *lswitch, size_t idx)
+{
+    const struct nbrec_logical_port_pair *lsp_pair = lswitch->port_pairs[idx];
+
+    /* First remove 'lsp-pair' from the array of port-pairs.
+     * This is what will actually cause the logical port-pair to be deleted
+     * when the transaction is sent to the database server
+     * (due to garbage collection). */
+    struct nbrec_logical_port_pair **new_port_pair
+        = xmemdup(lswitch->port_pairs,
+                   sizeof *new_port_pair * lswitch->n_port_pairs);
+    new_port_pair[idx] = new_port_pair[lswitch->n_port_pairs - 1];
+    nbrec_logical_switch_verify_port_pairs(lswitch);
+    nbrec_logical_switch_set_port_pairs(lswitch, new_port_pair,
+                   lswitch->n_port_pairs - 1);
+    free(new_port_pair);
+
+    /* Delete 'lsp-pair' from the IDL.  This won't have a real effect on the
+     * database server (the IDL will suppress it in fact) but it means that it
+     * won't show up when we iterate with
+     * NBREC_LOGICAL_PORT_PAIR_FOR_EACH later. */
+    nbrec_logical_port_pair_delete(lsp_pair);
+}
+
+static void
+nbctl_lsp_pair_del(struct ctl_context *ctx)
+{
+    const struct nbrec_logical_port_pair *lsp_pair;
+    const bool must_exist = !shash_find(&ctx->options, "--if-exists");
+
+    lsp_pair = lsp_pair_by_name_or_uuid(ctx, ctx->argv[1], must_exist);
+    if (!lsp_pair) {
+        if (must_exist) {
+            ctl_fatal("Cannot find lsp_pair: %s\n", ctx->argv[1]);
+        }
+    }
+
+    /* Find the port-pair_group that contains 'port-pair', then delete it. */
+    const struct nbrec_logical_switch *lswitch;
+    NBREC_LOGICAL_SWITCH_FOR_EACH(lswitch, ctx->idl) {
+        for (size_t i = 0; i < lswitch->n_port_pairs; i++) {
+            if (lswitch->port_pairs[i] == lsp_pair) {
+                remove_lsp_pair(lswitch,i);
+                return;
+            }
+        }
+    }
+    if (must_exist) {
+        ctl_fatal("logical port-pair %s is not part of any logical switch",
+                  ctx->argv[1]);
+    }
+}
+
+static void
+print_lsp_pairs_for_switch(struct ctl_context *ctx,
+                           const struct nbrec_logical_switch *lswitch,
+                           const char *ppair_name_filter,
+                           const bool show_switch_name)
+{
+    struct smap lsp_pairs;
+    size_t i;
+
+    smap_init(&lsp_pairs);
+    for (i = 0; i < lswitch->n_port_pairs; i++) {
+        const struct nbrec_logical_port_pair *lsp_pair =
+                    lswitch->port_pairs[i];
+        if (ppair_name_filter!= NULL && strcmp(ppair_name_filter,
+                   lsp_pair->name)!= 0) {
+            continue;
+        } else {
+          const struct nbrec_logical_switch_port *linport  = lsp_pair->inport;
+          const struct nbrec_logical_switch_port *loutport = lsp_pair->outport;
+          const char *linport_name = linport ? linport->name : "<not_set>";
+          const char *loutport_name = loutport ? loutport->name : "<not_set>";
+
+          if (show_switch_name) {
+            smap_add_format(&lsp_pairs, lsp_pair->name,
+                             UUID_FMT " (%s:%s) in:%s out:%s",
+                             UUID_ARGS(&lsp_pair->header_.uuid), lswitch->name,
+                             lsp_pair->name, linport_name, loutport_name);
+          } else {
+            smap_add_format(&lsp_pairs, lsp_pair->name,
+                             UUID_FMT " (%s) in:%s out:%s",
+                             UUID_ARGS(&lsp_pair->header_.uuid),
+                             lsp_pair->name, linport_name, loutport_name);
+        }
+      }
+    }
+    const struct smap_node **nodes = smap_sort(&lsp_pairs);
+    for (i = 0; i < smap_count(&lsp_pairs); i++) {
+        const struct smap_node *node = nodes[i];
+        ds_put_format(&ctx->output, "%s\n", node->value);
+    }
+    smap_destroy(&lsp_pairs);
+    free(nodes);
+}
+
+static void
+nbctl_lsp_pair_list(struct ctl_context *ctx)
+{
+    const char *id = ctx->argc > 1 ? ctx->argv[1] : NULL;
+    const char *pair_name_filter = ctx->argc > 2 ? ctx->argv[2] : NULL;
+    const struct nbrec_logical_switch *lswitch;
+    const struct nbrec_logical_port_pair *lsp_pair;
+
+    if (pair_name_filter!= NULL) {
+      lsp_pair = lsp_pair_by_name_or_uuid(ctx, ctx->argv[2], true);
+      if (!lsp_pair) {
+           ctl_fatal("%s: an lsp_pair with this name does not exist",
+                              ctx->argv[2]);
+           return;
+      }
+    }
+    if (id) {
+        lswitch = ls_by_name_or_uuid(ctx, id, true);
+        print_lsp_pairs_for_switch(ctx, lswitch, pair_name_filter, false);
+    } else {
+        NBREC_LOGICAL_SWITCH_FOR_EACH(lswitch, ctx->idl) {
+            if (lswitch->n_port_pairs == 0) {
+                continue;
+            }
+            print_lsp_pairs_for_switch(ctx, lswitch, pair_name_filter, true);
+        }
+    }
+}
+/* End of port-pair operations */
 
 /* Returns the logical switch that contains 'lsp'. */
 static const struct nbrec_logical_switch *
@@ -1324,6 +2463,26 @@  nbctl_acl_add(struct ctl_context *ctx)
         return;
     }
 
+    /* Validate ACL Options, if there were any provided. */
+    struct smap acl_options = SMAP_INITIALIZER(&acl_options);
+    if (ctx->argc >= 7) {
+        struct sset acl_options_set;
+        sset_from_delimited_string(&acl_options_set, ctx->argv[6], " ");
+
+        const char *acl_option_tuple;
+        SSET_FOR_EACH (acl_option_tuple, &acl_options_set) {
+            char *key, *value;
+            value = xstrdup(acl_option_tuple);
+            key = strsep(&value, "=");
+            if (value) {
+                smap_add(&acl_options, key, value);
+            }
+            free(key);
+        }
+
+        sset_destroy(&acl_options_set);
+    }
+
     /* Create the acl. */
     struct nbrec_acl *acl = nbrec_acl_insert(ctx->txn);
     nbrec_acl_set_priority(acl, priority);
@@ -1353,6 +2512,8 @@  nbctl_acl_add(struct ctl_context *ctx)
     new_acls[ls->n_acls] = acl;
     nbrec_logical_switch_set_acls(ls, new_acls, ls->n_acls + 1);
     free(new_acls);
+
+    smap_destroy(&acl_options);
 }
 
 static void
@@ -3308,6 +4469,8 @@  static const struct ctl_command_syntax nbctl_commands[] = {
     { "sync", 0, 0, "", nbctl_pre_sync, nbctl_sync, NULL, "", RO },
     { "show", 0, 1, "[SWITCH]", NULL, nbctl_show, NULL, "", RO },
 
+
+
     /* logical switch commands. */
     { "ls-add", 0, 1, "[SWITCH]", NULL, nbctl_ls_add, NULL,
       "--may-exist,--add-duplicate", RW },
@@ -3354,6 +4517,51 @@  static const struct ctl_command_syntax nbctl_commands[] = {
     { "lsp-get-dhcpv4-options", 1, 1, "PORT", NULL,
       nbctl_lsp_get_dhcpv4_options, NULL, "", RO },
 
+          /* lsp-chain-classifier commands. */
+    { "lsp-chain-classifier-add", 5, 7,
+                       "SWITCH, CHAIN, PORT, DIRECTION, PATH, [NAME], [MATCH]",
+      NULL, nbctl_lsp_chain_classifier_add, NULL,
+                       "--may-exist,--add-duplicate", RW },
+    { "lsp-chain-classifier-del", 1, 1, "CLASSIFIER", NULL,
+      nbctl_lsp_chain_classifier_del, NULL, "--if-exists", RW },
+    { "lsp-chain-classifier-list", 0, 1, "[SWITCH]", NULL,
+      nbctl_lsp_chain_classifier_list, NULL, "", RO },
+    { "lsp-chain-classifier-show", 0, 2, "[SWITCH], [CLASSIFIER]", NULL,
+      nbctl_lsp_chain_classifier_show, NULL, "", RO },
+
+    /* lsp-chain commands. */
+    { "lsp-chain-add", 1, 2, "SWITCH [CHAIN]", NULL,
+      nbctl_lsp_chain_add,
+      NULL, "--may-exist,--add-duplicate", RW },
+    { "lsp-chain-del", 1, 1, "CHAIN", NULL, nbctl_lsp_chain_del,
+      NULL, "--if-exists", RW },
+    { "lsp-chain-list", 0, 2, "[SWITCH [CHAIN]]", NULL,
+      nbctl_lsp_chain_list, NULL, "", RO },
+    { "lsp-chain-show", 0, 1, "[CHAIN]", NULL,
+      nbctl_lsp_chain_show, NULL, "", RO },
+
+    /* lsp-pair-group commands. */
+    { "lsp-pair-group-add", 1, 3, "CHAIN [PAIR-GROUP [OFFSET]]",
+      NULL, nbctl_lsp_pair_group_add, NULL, "--may-exist,--add-duplicate",
+      RW },
+    { "lsp-pair-group-del", 1, 1, "PAIR-GROUP", NULL, nbctl_lsp_pair_group_del,
+      NULL, "--if-exists", RW },
+    { "lsp-pair-group-list", 1, 1, "CHAIN", NULL, nbctl_lsp_pair_group_list,
+      NULL, "", RO },
+    { "lsp-pair-group-add-port-pair", 2, 2, "PAIR-GROUP LSP-PAIR",
+      NULL, nbctl_lsp_pair_group_add_port_pair, NULL, "--may-exist", RW },
+    { "lsp-pair-group-del-port-pair", 2, 2, "PAIR-GROUP LSP-PAIR",
+      NULL, nbctl_lsp_pair_group_del_port_pair, NULL, "--if-exists", RW },
+
+    /* lsp-pair commands. */
+    { "lsp-pair-add", 3, 4, "SWITCH, PORT-IN, PORT-OUT [LSP-PAIR]", NULL,
+      nbctl_lsp_pair_add,
+      NULL, "--may-exist,--add-duplicate", RW },
+    { "lsp-pair-del", 1, 1, "LSP-PAIR", NULL, nbctl_lsp_pair_del,
+      NULL, "--if-exists", RW },
+    { "lsp-pair-list", 0, 2, "[SWITCH [LSP-PAIR]]", NULL,
+      nbctl_lsp_pair_list, NULL, "", RO },
+
     /* logical router commands. */
     { "lr-add", 0, 1, "[ROUTER]", NULL, nbctl_lr_add, NULL,
       "--may-exist,--add-duplicate", RW },