[ovs-dev,ovn,10/19] ovn-ic: Interconnection controller with AZ registeration.
diff mbox series

Message ID 1571619079-75503-11-git-send-email-hzhou@ovn.org
State New
Headers show
Series
  • OVN Interconnection
Related show

Commit Message

Han Zhou Oct. 21, 2019, 12:51 a.m. UTC
This patch introduces interconnection controller, ovn-ic, and
implements the basic AZ registration feature: taking the AZ
name from NB DB and create an Availability_Zone entry in
IC-SB DB.

Signed-off-by: Han Zhou <hzhou@ovn.org>
---
 Makefile.am          |   1 +
 ic/.gitignore        |   2 +
 ic/automake.mk       |  10 ++
 ic/ovn-ic.8.xml      | 111 ++++++++++++++
 ic/ovn-ic.c          | 413 +++++++++++++++++++++++++++++++++++++++++++++++++++
 ovn-nb.ovsschema     |   5 +-
 ovn-nb.xml           |   7 +
 tests/automake.mk    |   4 +-
 tests/ovn-ic.at      |  29 ++++
 tests/ovn-macros.at  | 161 ++++++++++++++++----
 tests/testsuite.at   |   1 +
 tutorial/ovs-sandbox |  78 +++++++++-
 12 files changed, 792 insertions(+), 30 deletions(-)
 create mode 100644 ic/.gitignore
 create mode 100644 ic/automake.mk
 create mode 100644 ic/ovn-ic.8.xml
 create mode 100644 ic/ovn-ic.c
 create mode 100644 tests/ovn-ic.at

Comments

aginwala Oct. 30, 2019, 12:49 a.m. UTC | #1
Thanks Han for the patches.

On Sun, Oct 20, 2019 at 5:54 PM Han Zhou <hzhou@ovn.org> wrote:

> This patch introduces interconnection controller, ovn-ic, and
> implements the basic AZ registration feature: taking the AZ
> name from NB DB and create an Availability_Zone entry in
> IC-SB DB.
>
> Signed-off-by: Han Zhou <hzhou@ovn.org>
> ---
>  Makefile.am          |   1 +
>  ic/.gitignore        |   2 +
>  ic/automake.mk       |  10 ++
>  ic/ovn-ic.8.xml      | 111 ++++++++++++++
>  ic/ovn-ic.c          | 413
> +++++++++++++++++++++++++++++++++++++++++++++++++++
>  ovn-nb.ovsschema     |   5 +-
>  ovn-nb.xml           |   7 +
>  tests/automake.mk    |   4 +-
>  tests/ovn-ic.at      |  29 ++++
>  tests/ovn-macros.at  | 161 ++++++++++++++++----
>  tests/testsuite.at   |   1 +
>  tutorial/ovs-sandbox |  78 +++++++++-
>  12 files changed, 792 insertions(+), 30 deletions(-)
>  create mode 100644 ic/.gitignore
>  create mode 100644 ic/automake.mk
>  create mode 100644 ic/ovn-ic.8.xml
>  create mode 100644 ic/ovn-ic.c
>  create mode 100644 tests/ovn-ic.at
>
> diff --git a/Makefile.am b/Makefile.am
> index 33c18c5..d22a220 100644
> --- a/Makefile.am
> +++ b/Makefile.am
> @@ -500,4 +500,5 @@ include selinux/automake.mk
>  include controller/automake.mk
>  include controller-vtep/automake.mk
>  include northd/automake.mk
> +include ic/automake.mk
>  include build-aux/automake.mk
> diff --git a/ic/.gitignore b/ic/.gitignore
> new file mode 100644
> index 0000000..1b73eb4
> --- /dev/null
> +++ b/ic/.gitignore
> @@ -0,0 +1,2 @@
> +/ovn-ic
> +/ovn-ic.8
> diff --git a/ic/automake.mk b/ic/automake.mk
> new file mode 100644
> index 0000000..8e71bc3
> --- /dev/null
> +++ b/ic/automake.mk
> @@ -0,0 +1,10 @@
> +# ovn-ic
> +bin_PROGRAMS += ic/ovn-ic
> +ic_ovn_ic_SOURCES = ic/ovn-ic.c
> +ic_ovn_ic_LDADD = \
> +       lib/libovn.la \
> +       $(OVSDB_LIBDIR)/libovsdb.la \
> +       $(OVS_LIBDIR)/libopenvswitch.la
> +man_MANS += ic/ovn-ic.8
> +EXTRA_DIST += ic/ovn-ic.8.xml
> +CLEANFILES += ic/ovn-ic.8
> diff --git a/ic/ovn-ic.8.xml b/ic/ovn-ic.8.xml
> new file mode 100644
> index 0000000..00f33aa
> --- /dev/null
> +++ b/ic/ovn-ic.8.xml
> @@ -0,0 +1,111 @@
> +<?xml version="1.0" encoding="utf-8"?>
> +<manpage program="ovn-ic" section="8" title="ovn-ic">
> +    <h1>Name</h1>
> +    <p>ovn-ic -- Open Virtual Network interconnection controller</p>
> +
> +    <h1>Synopsis</h1>
> +    <p><code>ovn-ic</code> [<var>options</var>]</p>
> +
> +    <h1>Description</h1>
> +    <p>
> +      <code>ovn-ic</code>, OVN interconnection controller, is a
> centralized
> +      daemon which communicates with global interconnection databases
> INB/ISB
> +      to configure and exchange data with local NB/SB for interconnecting
> +      with other OVN deployments.
> +    </p>
> +
> +    <h1>Options</h1>
> +    <dl>
> +      <dt><code>--ovnnb-db=<var>database</var></code></dt>
> +      <dd>
> +        The OVSDB database containing the OVN Northbound Database.  If the
> +        <env>OVN_NB_DB</env> environment variable is set, its value is
> used
> +        as the default.  Otherwise, the default is
> +        <code>unix:@RUNDIR@/ovnnb_db.sock</code>.
> +      </dd>
> +      <dt><code>--ovnsb-db=<var>database</var></code></dt>
> +      <dd>
> +        The OVSDB database containing the OVN Southbound Database.  If the
> +        <env>OVN_SB_DB</env> environment variable is set, its value is
> used
> +        as the default.  Otherwise, the default is
> +        <code>unix:@RUNDIR@/ovnsb_db.sock</code>.
> +      </dd>
> +      <dt><code>--ovninb-db=<var>database</var></code></dt>
> +      <dd>
> +        The OVSDB database containing the OVN Interconnection Northbound
> +        Database.  If the <env>OVN_INB_DB</env> environment variable is
> set,
> +        its value is used as the default.  Otherwise, the default is
> +        <code>unix:@RUNDIR@/ovninb_db.sock</code>.
> +      </dd>
> +      <dt><code>--ovnisb-db=<var>database</var></code></dt>
> +      <dd>
> +        The OVSDB database containing the OVN Interconnection Southbound
> +        Database.  If the <env>OVN_ISB_DB</env> environment variable is
> set,
> +        its value is used as the default.  Otherwise, the default is
> +        <code>unix:@RUNDIR@/ovnisb_db.sock</code>.
> +      </dd>
> +    </dl>
> +    <p>
> +      <var>database</var> in the above options must be an OVSDB active or
> +      passive connection method, as described in <code>ovsdb</code>(7).
> +    </p>
> +
> +    <h2>Daemon Options</h2>
> +    <xi:include href="lib/daemon.xml" xmlns:xi="
> http://www.w3.org/2003/XInclude"/>
> +
> +    <h2>Logging Options</h2>
> +    <xi:include href="lib/vlog.xml" xmlns:xi="
> http://www.w3.org/2003/XInclude"/>
> +
> +    <h2>PKI Options</h2>
> +    <p>
> +      PKI configuration is required in order to use SSL for the
> connections to
> +      the Northbound and Southbound databases.
> +    </p>
> +    <xi:include href="lib/ssl.xml" xmlns:xi="
> http://www.w3.org/2003/XInclude"/>
> +
> +    <h2>Other Options</h2>
> +    <xi:include href="lib/unixctl.xml"
> +     xmlns:xi="http://www.w3.org/2003/XInclude"/>
> +    <h3></h3>
> +    <xi:include href="lib/common.xml"
> +     xmlns:xi="http://www.w3.org/2003/XInclude"/>
> +
> +    <h1>Runtime Management Commands</h1>
> +    <p>
> +      <code>ovs-appctl</code> can send commands to a running
> +      <code>ovn-ic</code> process.  The currently supported commands
> +      are described below.
> +      <dl>
> +      <dt><code>exit</code></dt>
> +      <dd>
> +        Causes <code>ovn-ic</code> to gracefully terminate.
> +      </dd>
> +
> +      <dt><code>pause</code></dt>
> +      <dd>
> +        Pauses the ovn-ic operation from processing any Northbound and
> +        Southbound database changes.
> +      </dd>
> +
> +      <dt><code>resume</code></dt>
> +      <dd>
> +        Resumes the ovn-ic operation to process Northbound and
> +        Southbound database contents and generate logical flows.
> +      </dd>
> +
> +      <dt><code>is-paused</code></dt>
> +      <dd>
> +        Returns "true" if ovn-ic is currently paused, "false" otherwise.
> +      </dd>
> +      </dl>
> +    </p>
> +
> +    <h1>Active-Standby for High Availability</h1>
> +    <p>
> +      You may run <code>ovn-ic</code> more than once in an OVN deployment.
> +      OVN will automatically ensure that only one of them is active at a
> time.
> +      If multiple instances of <code>ovn-ic</code> are running and the
> +      active <code>ovn-ic</code> fails, one of the hot standby instances
> +      of <code>ovn-ic</code> will automatically take over.
> +    </p>
> +</manpage>
> diff --git a/ic/ovn-ic.c b/ic/ovn-ic.c
> new file mode 100644
> index 0000000..7cc80bc
> --- /dev/null
> +++ b/ic/ovn-ic.c
> @@ -0,0 +1,413 @@
> +/*
> + * Licensed under the Apache License, Version 2.0 (the "License");
> + * you may not use this file except in compliance with the License.
> + * You may obtain a copy of the License at:
> + *
> + *     http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> + */
> +
> +#include <config.h>
> +
> +#include <getopt.h>
> +#include <stdlib.h>
> +#include <stdio.h>
> +
> +#include "bitmap.h"
> +#include "command-line.h"
> +#include "daemon.h"
> +#include "dirs.h"
> +#include "openvswitch/dynamic-string.h"
> +#include "fatal-signal.h"
> +#include "hash.h"
> +#include "openvswitch/hmap.h"
> +#include "lib/ovn-inb-idl.h"
> +#include "lib/ovn-isb-idl.h"
> +#include "lib/ovn-nb-idl.h"
> +#include "lib/ovn-sb-idl.h"
> +#include "lib/ovn-util.h"
> +#include "openvswitch/poll-loop.h"
> +#include "smap.h"
> +#include "sset.h"
> +#include "stream.h"
> +#include "stream-ssl.h"
> +#include "unixctl.h"
> +#include "util.h"
> +#include "uuid.h"
> +#include "openvswitch/vlog.h"
> +
> +VLOG_DEFINE_THIS_MODULE(ovn_ic);
> +
> +static unixctl_cb_func ovn_ic_exit;
> +static unixctl_cb_func ovn_ic_pause;
> +static unixctl_cb_func ovn_ic_resume;
> +static unixctl_cb_func ovn_ic_is_paused;
> +
> +struct ic_context {
> +    struct ovsdb_idl *ovnnb_idl;
> +    struct ovsdb_idl *ovnsb_idl;
> +    struct ovsdb_idl *ovninb_idl;
> +    struct ovsdb_idl *ovnisb_idl;
> +    struct ovsdb_idl_txn *ovnnb_txn;
> +    struct ovsdb_idl_txn *ovnsb_txn;
> +    struct ovsdb_idl_txn *ovninb_txn;
> +    struct ovsdb_idl_txn *ovnisb_txn;
> +};
> +
> +static const char *ovnnb_db;
> +static const char *ovnsb_db;
> +static const char *ovninb_db;
> +static const char *ovnisb_db;
> +static const char *unixctl_path;
> +
> +
> +static void
> +usage(void)
> +{
> +    printf("\
> +%s: OVN interconnection management daemon\n\
> +usage: %s [OPTIONS]\n\
> +\n\
> +Options:\n\
> +  --ovnnb-db=DATABASE       connect to ovn-nb database at DATABASE\n\
> +                            (default: %s)\n\
> +  --ovnsb-db=DATABASE       connect to ovn-sb database at DATABASE\n\
> +                            (default: %s)\n\
> +  --unixctl=SOCKET          override default control socket name\n\
> +  -h, --help                display this help message\n\
> +  -o, --options             list available options\n\
> +  -V, --version             display version information\n\
> +", program_name, program_name, default_nb_db(), default_sb_db());
> +    daemon_usage();
> +    vlog_usage();
> +    stream_usage("database", true, true, false);
> +}
> +
> +static const struct isbrec_availability_zone *
> +az_run(struct ic_context *ctx)
> +{
> +    const struct nbrec_nb_global *nb_global =
> +        nbrec_nb_global_first(ctx->ovnnb_idl);
> +
> +    if (!nb_global) {
> +        VLOG_INFO("NB Global not exist.");
> +        return NULL;
> +    }
> +
> +    /* Delete old AZ if name changes.  Note: if name changed when ovn-ic
> +     * is not running, one has to manually delete the old AZ with:
> +     * "ovn-isbctl destroy avail <az>". */
> +    static char *az_name;
> +    const struct isbrec_availability_zone *az;
> +    if (az_name && strcmp(az_name, nb_global->name)) {
> +        ISBREC_AVAILABILITY_ZONE_FOR_EACH (az, ctx->ovnisb_idl) {
> +            if (!strcmp(az->name, az_name)) {
> +                isbrec_availability_zone_delete(az);
> +                break;
> +            }
> +        }
> +        free(az_name);
> +        az_name = NULL;
> +    }
> +
> +    if (!nb_global->name[0]) {
> +        return NULL;
> +    }
> +
> +    if (!az_name) {
> +        az_name = xstrdup(nb_global->name);
> +    }
> +
> +    ISBREC_AVAILABILITY_ZONE_FOR_EACH (az, ctx->ovnisb_idl) {
> +        if (!strcmp(az->name, az_name)) {
> +            return az;
> +        }
> +    }
> +
> +    /* Create AZ in ISB */
> +    if (ctx->ovnisb_txn) {
> +        VLOG_INFO("Register AZ %s to interconnection DB.", az_name);
> +        az = isbrec_availability_zone_insert(ctx->ovnisb_txn);
> +        isbrec_availability_zone_set_name(az, az_name);
> +        return az;
> +    }
> +    return NULL;
> +}
> +
> +static void
> +ovn_db_run(struct ic_context *ctx)
> +{
> +    const struct isbrec_availability_zone *az = az_run(ctx);
> +    VLOG_DBG("Availability zone: %s", az ? az->name : "not created yet.");
> +}
> +
> +static void
> +parse_options(int argc OVS_UNUSED, char *argv[] OVS_UNUSED)
> +{
> +    enum {
> +        DAEMON_OPTION_ENUMS,
> +        VLOG_OPTION_ENUMS,
> +        SSL_OPTION_ENUMS,
> +    };
> +    static const struct option long_options[] = {
> +        {"ovnsb-db", required_argument, NULL, 'd'},
> +        {"ovnnb-db", required_argument, NULL, 'D'},
> +        {"ovnisb-db", required_argument, NULL, 'i'},
> +        {"ovninb-db", required_argument, NULL, 'I'},
> +        {"unixctl", required_argument, NULL, 'u'},
> +        {"help", no_argument, NULL, 'h'},
> +        {"options", no_argument, NULL, 'o'},
> +        {"version", no_argument, NULL, 'V'},
> +        DAEMON_LONG_OPTIONS,
> +        VLOG_LONG_OPTIONS,
> +        STREAM_SSL_LONG_OPTIONS,
> +        {NULL, 0, NULL, 0},
> +    };
> +    char *short_options =
> ovs_cmdl_long_options_to_short_options(long_options);
> +
> +    for (;;) {
> +        int c;
> +
> +        c = getopt_long(argc, argv, short_options, long_options, NULL);
> +        if (c == -1) {
> +            break;
> +        }
> +
> +        switch (c) {
> +        DAEMON_OPTION_HANDLERS;
> +        VLOG_OPTION_HANDLERS;
> +        STREAM_SSL_OPTION_HANDLERS;
> +
> +        case 'd':
> +            ovnsb_db = optarg;
> +            break;
> +
> +        case 'D':
> +            ovnnb_db = optarg;
> +            break;
> +
> +        case 'u':
> +            unixctl_path = optarg;
> +            break;
> +
> +        case 'h':
> +            usage();
> +            exit(EXIT_SUCCESS);
> +
> +        case 'o':
> +            ovs_cmdl_print_options(long_options);
> +            exit(EXIT_SUCCESS);
> +
> +        case 'V':
> +            ovs_print_version(0, 0);
> +            exit(EXIT_SUCCESS);
> +
> +        default:
> +            break;
> +        }
> +    }
> +
> +    if (!ovnsb_db) {
> +        ovnsb_db = default_sb_db();
> +    }
> +
> +    if (!ovnnb_db) {
> +        ovnnb_db = default_nb_db();
> +    }
> +
> +    if (!ovnisb_db) {
> +        ovnisb_db = default_isb_db();
>
This still did not work if we are starting ic controller on different azs
with remote="tcp:10.xxx:6645/46" after patching ovn-ctl. As per code it
checks env vars  OVN_INB/SB_DB without passing values in command line. So
rectification is needed here.

> +    }
> +
> +    if (!ovninb_db) {
> +        ovninb_db = default_inb_db();
> +    }
> +
> +    free(short_options);
> +}
> +
> +static void OVS_UNUSED
> +add_column_noalert(struct ovsdb_idl *idl,
> +                   const struct ovsdb_idl_column *column)
> +{
> +    ovsdb_idl_add_column(idl, column);
> +    ovsdb_idl_omit_alert(idl, column);
> +}
> +
> +int
> +main(int argc, char *argv[])
> +{
> +    int res = EXIT_SUCCESS;
> +    struct unixctl_server *unixctl;
> +    int retval;
> +    bool exiting;
> +    bool paused;
> +
> +    fatal_ignore_sigpipe();
> +    ovs_cmdl_proctitle_init(argc, argv);
> +    set_program_name(argv[0]);
> +    service_start(&argc, &argv);
> +    parse_options(argc, argv);
> +
> +    daemonize_start(false);
> +
> +    if (!unixctl_path) {
> +        char *abs_unixctl_path = get_abs_unix_ctl_path();
> +        retval = unixctl_server_create(abs_unixctl_path, &unixctl);
> +        free(abs_unixctl_path);
> +    } else {
> +        retval = unixctl_server_create(unixctl_path, &unixctl);
> +    }
> +
> +    if (retval) {
> +        exit(EXIT_FAILURE);
> +    }
> +    unixctl_command_register("exit", "", 0, 0, ovn_ic_exit, &exiting);
> +    unixctl_command_register("pause", "", 0, 0, ovn_ic_pause, &paused);
> +    unixctl_command_register("resume", "", 0, 0, ovn_ic_resume, &paused);
> +    unixctl_command_register("is-paused", "", 0, 0, ovn_ic_is_paused,
> +                             &paused);
> +
> +    daemonize_complete();
> +
> +    /* ovn-inb db. */
> +    struct ovsdb_idl_loop ovninb_idl_loop = OVSDB_IDL_LOOP_INITIALIZER(
> +        ovsdb_idl_create(ovninb_db, &inbrec_idl_class, true, true));
> +
> +    /* ovn-isb db. */
> +    struct ovsdb_idl_loop ovnisb_idl_loop = OVSDB_IDL_LOOP_INITIALIZER(
> +        ovsdb_idl_create(ovnisb_db, &isbrec_idl_class, true, true));
> +
> +    /* ovn-nb db. XXX: add only needed tables and columns */
> +    struct ovsdb_idl_loop ovnnb_idl_loop = OVSDB_IDL_LOOP_INITIALIZER(
> +        ovsdb_idl_create(ovnnb_db, &nbrec_idl_class, true, true));
> +
> +    /* ovn-sb db. XXX: add only needed tables and columns */
> +    struct ovsdb_idl_loop ovnsb_idl_loop = OVSDB_IDL_LOOP_INITIALIZER(
> +        ovsdb_idl_create(ovnsb_db, &sbrec_idl_class, true, true));
> +
> +    /* Ensure that only a single ovn-ic is active in the deployment by
> +     * acquiring a lock called "ovn_ic" on the southbound database
> +     * and then only performing DB transactions if the lock is held. */
> +    ovsdb_idl_set_lock(ovnsb_idl_loop.idl, "ovn_ic");
> +    bool had_lock = false;
> +
> +    /* Main loop. */
> +    exiting = false;
> +    paused = false;
> +    while (!exiting) {
> +        if (!paused) {
> +            struct ic_context ctx = {
> +                .ovnnb_idl = ovnnb_idl_loop.idl,
> +                .ovnnb_txn = ovsdb_idl_loop_run(&ovnnb_idl_loop),
> +                .ovnsb_idl = ovnsb_idl_loop.idl,
> +                .ovnsb_txn = ovsdb_idl_loop_run(&ovnsb_idl_loop),
> +                .ovninb_idl = ovninb_idl_loop.idl,
> +                .ovninb_txn = ovsdb_idl_loop_run(&ovninb_idl_loop),
> +                .ovnisb_idl = ovnisb_idl_loop.idl,
> +                .ovnisb_txn = ovsdb_idl_loop_run(&ovnisb_idl_loop),
> +            };
> +
> +            if (!had_lock && ovsdb_idl_has_lock(ovnsb_idl_loop.idl)) {
> +                VLOG_INFO("ovn-ic lock acquired. "
> +                        "This ovn-ic instance is now active.");
> +                had_lock = true;
> +            } else if (had_lock &&
> !ovsdb_idl_has_lock(ovnsb_idl_loop.idl)) {
> +                VLOG_INFO("ovn-ic lock lost. "
> +                        "This ovn-ic instance is now on standby.");
> +                had_lock = false;
> +            }
> +
> +            if (ovsdb_idl_has_lock(ovnsb_idl_loop.idl)) {
> +                ovn_db_run(&ctx);
> +            }
> +
> +            ovsdb_idl_loop_commit_and_wait(&ovnnb_idl_loop);
> +            ovsdb_idl_loop_commit_and_wait(&ovnsb_idl_loop);
> +            ovsdb_idl_loop_commit_and_wait(&ovninb_idl_loop);
> +            ovsdb_idl_loop_commit_and_wait(&ovnisb_idl_loop);
> +        } else {
> +            /* ovn-ic is paused
> +             *    - we still want to handle any db updates and update the
> +             *      local IDL. Otherwise, when it is resumed, the local
> IDL
> +             *      copy will be out of sync.
> +             *    - but we don't want to create any txns.
> +             * */
> +            ovsdb_idl_run(ovnnb_idl_loop.idl);
> +            ovsdb_idl_run(ovnsb_idl_loop.idl);
> +            ovsdb_idl_run(ovninb_idl_loop.idl);
> +            ovsdb_idl_run(ovnisb_idl_loop.idl);
> +            ovsdb_idl_wait(ovnnb_idl_loop.idl);
> +            ovsdb_idl_wait(ovnsb_idl_loop.idl);
> +            ovsdb_idl_wait(ovninb_idl_loop.idl);
> +            ovsdb_idl_wait(ovnisb_idl_loop.idl);
> +        }
> +
> +        unixctl_server_run(unixctl);
> +        unixctl_server_wait(unixctl);
> +        if (exiting) {
> +            poll_immediate_wake();
> +        }
> +
> +        poll_block();
> +        if (should_service_stop()) {
> +            exiting = true;
> +        }
> +    }
> +
> +    unixctl_server_destroy(unixctl);
> +    ovsdb_idl_loop_destroy(&ovnnb_idl_loop);
> +    ovsdb_idl_loop_destroy(&ovnsb_idl_loop);
> +    ovsdb_idl_loop_destroy(&ovninb_idl_loop);
> +    ovsdb_idl_loop_destroy(&ovnisb_idl_loop);
> +    service_stop();
> +
> +    exit(res);
> +}
> +
> +static void
> +ovn_ic_exit(struct unixctl_conn *conn, int argc OVS_UNUSED,
> +                const char *argv[] OVS_UNUSED, void *exiting_)
> +{
> +    bool *exiting = exiting_;
> +    *exiting = true;
> +
> +    unixctl_command_reply(conn, NULL);
> +}
> +
> +static void
> +ovn_ic_pause(struct unixctl_conn *conn, int argc OVS_UNUSED,
> +                const char *argv[] OVS_UNUSED, void *pause_)
> +{
> +    bool *pause = pause_;
> +    *pause = true;
> +
> +    unixctl_command_reply(conn, NULL);
> +}
> +
> +static void
> +ovn_ic_resume(struct unixctl_conn *conn, int argc OVS_UNUSED,
> +                  const char *argv[] OVS_UNUSED, void *pause_)
> +{
> +    bool *pause = pause_;
> +    *pause = false;
> +
> +    unixctl_command_reply(conn, NULL);
> +}
> +
> +static void
> +ovn_ic_is_paused(struct unixctl_conn *conn, int argc OVS_UNUSED,
> +                     const char *argv[] OVS_UNUSED, void *paused_)
> +{
> +    bool *paused = paused_;
> +    if (*paused) {
> +        unixctl_command_reply(conn, "true");
> +    } else {
> +        unixctl_command_reply(conn, "false");
> +    }
> +}
> diff --git a/ovn-nb.ovsschema b/ovn-nb.ovsschema
> index 2c87cbb..1a9acdb 100644
> --- a/ovn-nb.ovsschema
> +++ b/ovn-nb.ovsschema
> @@ -1,10 +1,11 @@
>  {
>      "name": "OVN_Northbound",
> -    "version": "5.16.0",
> -    "cksum": "923459061 23095",
> +    "version": "5.16.1",
> +    "cksum": "2818115555 23139",
>      "tables": {
>          "NB_Global": {
>              "columns": {
> +                "name": {"type": "string"},
>                  "nb_cfg": {"type": {"key": "integer"}},
>                  "sb_cfg": {"type": {"key": "integer"}},
>                  "hv_cfg": {"type": {"key": "integer"}},
> diff --git a/ovn-nb.xml b/ovn-nb.xml
> index 1504f8f..05091fb 100644
> --- a/ovn-nb.xml
> +++ b/ovn-nb.xml
> @@ -36,6 +36,13 @@
>        one row.
>      </p>
>
> +    <group title="Identity">
> +      <column name="name">
> +        The name of the OVN cluster, which uniquely identifies the OVN
> cluster
> +        throughout all OVN clusters supposed to interconnect with each
> other.
> +      </column>
> +    </group>
> +
>      <group title="Status">
>        These columns allow a client to track the overall configuration
> state of
>        the system.
> diff --git a/tests/automake.mk b/tests/automake.mk
> index c941ec2..8e12afd 100644
> --- a/tests/automake.mk
> +++ b/tests/automake.mk
> @@ -28,6 +28,7 @@ TESTSUITE_AT = \
>         tests/ovn-isbctl.at \
>         tests/ovn-controller.at \
>         tests/ovn-controller-vtep.at \
> +       tests/ovn-ic.at \
>         tests/ovn-macros.at \
>         tests/ovn-performance.at
>
> @@ -54,7 +55,7 @@ SYSTEM_KMOD_TESTSUITE =
> $(srcdir)/tests/system-kmod-testsuite
>  SYSTEM_USERSPACE_TESTSUITE = $(srcdir)/tests/system-userspace-testsuite
>  DISTCLEANFILES += tests/atconfig tests/atlocal
>
> -AUTOTEST_PATH =
> $(ovs_builddir)/utilities:$(ovs_builddir)/vswitchd:$(ovs_builddir)/ovsdb:$(ovs_builddir)/vtep:tests:$(PTHREAD_WIN32_DIR_DLL):$(SSL_DIR):controller-vtep:northd:utilities:controller
> +AUTOTEST_PATH =
> $(ovs_builddir)/utilities:$(ovs_builddir)/vswitchd:$(ovs_builddir)/ovsdb:$(ovs_builddir)/vtep:tests:$(PTHREAD_WIN32_DIR_DLL):$(SSL_DIR):controller-vtep:northd:utilities:controller:ic
>
>  check-local:
>         set $(SHELL) '$(TESTSUITE)' -C tests
> AUTOTEST_PATH=$(AUTOTEST_PATH) ovs_srcdir=$(ovs_srcdir); \
> @@ -101,6 +102,7 @@ valgrind_wrappers = \
>         tests/valgrind/ovn-sbctl \
>         tests/valgrind/ovn-inbctl \
>         tests/valgrind/ovn-isbctl \
> +       tests/valgrind/ovn-ic \
>         tests/valgrind/ovs-appctl \
>         tests/valgrind/ovs-ofctl \
>         tests/valgrind/ovs-vsctl \
> diff --git a/tests/ovn-ic.at b/tests/ovn-ic.at
> new file mode 100644
> index 0000000..1e69820
> --- /dev/null
> +++ b/tests/ovn-ic.at
> @@ -0,0 +1,29 @@
> +AT_BANNER([OVN Interconnection Controller])
> +AT_SETUP([ovn-ic -- AZ register])
> +AT_SKIP_IF([test $HAVE_PYTHON = no])
> +
> +ovn_init_ic_db
> +ovn_start az1
> +ovn_start az2
> +
> +AT_CHECK([ovn-isbctl show], [0], [dnl
> +availability-zone az1
> +availability-zone az2
> +])
> +
> +ovn_as az1
> +ovn-nbctl set NB_Global . name=az3
> +AT_CHECK([ovn-isbctl show], [0], [dnl
> +availability-zone az2
> +availability-zone az3
> +])
> +
> +ovn_as az2
> +ovn-nbctl set NB_Global . name=\"\"
> +AT_CHECK([ovn-isbctl show], [0], [dnl
> +availability-zone az3
> +])
> +
> +OVN_CLEANUP_IC([az1], [az2])
> +
> +AT_CLEANUP
> diff --git a/tests/ovn-macros.at b/tests/ovn-macros.at
> index 4045ce0..ea080dc 100644
> --- a/tests/ovn-macros.at
> +++ b/tests/ovn-macros.at
> @@ -48,9 +48,49 @@ m4_define([OVN_CLEANUP],[
>      OVN_CLEANUP_VSWITCH([main])
>  ])
>
> +# OVN_CLEANUP_AZ(az)
> +#
> +# Gracefully terminate all OVN daemons, including those in the
> +# specified sandbox instances.
> +m4_define([OVN_CLEANUP_AZ],[
> +    as $1/ovn-sb
> +    OVS_APP_EXIT_AND_WAIT([ovsdb-server])
> +
> +    as $1/ovn-nb
> +    OVS_APP_EXIT_AND_WAIT([ovsdb-server])
> +
> +    as $1/northd
> +    OVS_APP_EXIT_AND_WAIT([ovn-northd])
> +
> +    as $1/northd-backup
> +    OVS_APP_EXIT_AND_WAIT([ovn-northd])
> +
> +    as $1/ic
> +    OVS_APP_EXIT_AND_WAIT([ovn-ic])
> +])
> +
> +# OVN_CLEANUP_IC([az ...])
> +#
> +# Gracefully terminate all interconnection DBs, and daemons in the
> +# specified AZs, if any.
> +m4_define([OVN_CLEANUP_IC],[
> +    m4_foreach([az], [$@], [
> +        OVN_CLEANUP_AZ([az])
> +    ])
> +    as ovn-isb
> +    OVS_APP_EXIT_AND_WAIT([ovsdb-server])
> +
> +    as ovn-inb
> +    OVS_APP_EXIT_AND_WAIT([ovsdb-server])
> +
> +    if test -d "$ovs_base"/main; then
> +        OVN_CLEANUP_VSWITCH([main])
> +    fi
> +])
> +
>  m4_divert_push([PREPARE_TESTS])
>
> -# ovn_init_db DATABASE
> +# ovn_init_db DATABASE [AZ]
>  #
>  # Creates and initializes the given DATABASE (one of "ovn-sb" or
> "ovn-nb"),
>  # starts its ovsdb-server instance, and sets the appropriate environment
> @@ -60,36 +100,77 @@ m4_divert_push([PREPARE_TESTS])
>  # Usually invoked from ovn_start.
>  ovn_init_db () {
>      echo "creating $1 database"
> -    local d=$ovs_base/$1
> +    local as_d=$1
> +    if test -n "$2"; then
> +        as_d=$2/$as_d
> +    fi
> +    local d=$ovs_base/$as_d
>      mkdir "$d" || return 1
>      : > "$d"/.$1.db.~lock~
> -    as $1 ovsdb-tool create "$d"/$1.db "$abs_top_srcdir"/$1.ovsschema
> -    as $1 start_daemon ovsdb-server --remote=punix:"$d"/$1.sock "$d"/$1.db
> +    as $as_d ovsdb-tool create "$d"/$1.db "$abs_top_srcdir"/$1.ovsschema
> +    as $as_d start_daemon ovsdb-server --remote=punix:"$d"/$1.sock
> "$d"/$1.db
>      local var=`echo $1_db | tr a-z- A-Z_`
> -    AS_VAR_SET([$var], [unix:$ovs_base/$1/$1.sock]); export $var
> +    AS_VAR_SET([$var], [unix:"$d"/$1.sock]); export $var
>  }
>
> -# ovn_start
> +# ovn_init_ic_db
> +#
> +# Creates and initializes ovn-inb and ovn-isb databases and starts their
> +# ovsdb-server instances, for OVN interconnection.
> +ovn_init_ic_db () {
> +    ovn_init_db ovn-inb
> +    ovn_init_db ovn-isb
> +}
> +
> +# ovn_start [AZ]
>  #
>  # Creates and initializes ovn-sb and ovn-nb databases and starts their
>  # ovsdb-server instance, sets appropriate environment variables so that
>  # ovn-sbctl and ovn-nbctl use them by default, and starts ovn-northd
> running
>  # against them.
>  ovn_start () {
> -    ovn_init_db ovn-sb; ovn-sbctl init
> -    ovn_init_db ovn-nb; ovn-nbctl init
> +    if test -n "$1"; then
> +        mkdir "$ovs_base"/$1
> +    fi
>
> +    ovn_init_db ovn-sb $1; ovn-sbctl init
> +    ovn_init_db ovn-nb $1; ovn-nbctl init
> +    if test -n "$1"; then
> +        ovn-nbctl set NB_Global . name=$1
> +    fi
> +    local ovn_sb_db=$OVN_SB_DB
> +    local ovn_nb_db=$OVN_NB_DB
> +
> +    local as_d=northd
> +    if test -n "$1"; then
> +        as_d=$1/$as_d
> +    fi
>      echo "starting ovn-northd"
> -    mkdir "$ovs_base"/northd
> -    as northd start_daemon ovn-northd -v \
> -               --ovnnb-db=unix:"$ovs_base"/ovn-nb/ovn-nb.sock \
> -               --ovnsb-db=unix:"$ovs_base"/ovn-sb/ovn-sb.sock
> +    mkdir "$ovs_base"/$as_d
> +    as $as_d start_daemon ovn-northd -v \
> +               --ovnnb-db=$ovn_nb_db \
> +               --ovnsb-db=$ovn_sb_db
>
> +    as_d=northd-backup
> +    if test -n "$1"; then
> +        as_d=$1/$as_d
> +    fi
>      echo "starting backup ovn-northd"
> -    mkdir "$ovs_base"/northd-backup
> -    as northd-backup start_daemon ovn-northd -v \
> -               --ovnnb-db=unix:"$ovs_base"/ovn-nb/ovn-nb.sock \
> -               --ovnsb-db=unix:"$ovs_base"/ovn-sb/ovn-sb.sock
> +    mkdir "$ovs_base"/$as_d
> +    as $as_d start_daemon ovn-northd -v \
> +               --ovnnb-db=$ovn_nb_db \
> +               --ovnsb-db=$ovn_sb_db
> +
> +    if test -n "$1"; then
> +        as_d=$1/ic
> +        echo "starting ovn-ic"
> +        mkdir "$ovs_base"/$as_d
> +        as $as_d start_daemon ovn-ic -v \
> +               --ovnnb-db=$ovn_nb_db \
> +               --ovnsb-db=$ovn_sb_db \
> +               --ovninb-db=unix:"$ovs_base"/ovn-inb/ovn-inb.sock \
> +               --ovnisb-db=unix:"$ovs_base"/ovn-isb/ovn-isb.sock
> +    fi
>  }
>
>  # Interconnection networks.
> @@ -132,24 +213,25 @@ net_attach () {
>          || return 1
>  }
>
> -# ovn_attach NETWORK BRIDGE IP [MASKLEN]
> -#
> -# First, this command attaches BRIDGE to interconnection network NETWORK,
> just
> -# like "net_attach NETWORK BRIDGE".  Second, it configures (simulated) IP
> -# address IP (with network mask length MASKLEN, which defaults to 24) on
> -# BRIDGE.  Finally, it configures the Open vSwitch database to work with
> OVN
> -# and starts ovn-controller.
> -ovn_attach() {
> -    local net=$1 bridge=$2 ip=$3 masklen=${4-24}
> +# ovn_az_attach AZ NETWORK BRIDGE IP [MASKLEN]
> +ovn_az_attach() {
> +    local az=$1 net=$2 bridge=$3 ip=$4 masklen=${5-24}
>      net_attach $net $bridge || return 1
>
>      mac=`ovs-vsctl get Interface $bridge mac_in_use | sed s/\"//g`
>      arp_table="$arp_table $sandbox,$bridge,$ip,$mac"
>      ovs-appctl netdev-dummy/ip4addr $bridge $ip/$masklen >/dev/null ||
> return 1
>      ovs-appctl ovs/route/add $ip/$masklen $bridge >/dev/null || return 1
> +
> +    local ovn_remote
> +    if test X"$az" = XNONE; then
> +        ovn_remote=unix:$ovs_base/ovn-sb/ovn-sb.sock
> +    else
> +        ovn_remote=unix:$ovs_base/$az/ovn-sb/ovn-sb.sock
> +    fi
>      ovs-vsctl \
>          -- set Open_vSwitch . external-ids:system-id=$sandbox \
> -        -- set Open_vSwitch .
> external-ids:ovn-remote=unix:$ovs_base/ovn-sb/ovn-sb.sock \
> +        -- set Open_vSwitch . external-ids:ovn-remote=$ovn_remote \
>          -- set Open_vSwitch . external-ids:ovn-encap-type=geneve,vxlan \
>          -- set Open_vSwitch . external-ids:ovn-encap-ip=$ip \
>          -- add-br br-int \
> @@ -158,6 +240,33 @@ ovn_attach() {
>      start_daemon ovn-controller || return 1
>  }
>
> +# ovn_attach NETWORK BRIDGE IP [MASKLEN]
> +#
> +# First, this command attaches BRIDGE to interconnection network NETWORK,
> just
> +# like "net_attach NETWORK BRIDGE".  Second, it configures (simulated) IP
> +# address IP (with network mask length MASKLEN, which defaults to 24) on
> +# BRIDGE.  Finally, it configures the Open vSwitch database to work with
> OVN
> +# and starts ovn-controller.
> +ovn_attach() {
> +    ovn_az_attach NONE $@
> +}
> +
> +# ovn_setenv AZ
> +ovn_setenv () {
> +    local d=$ovs_base/$1
> +    AS_VAR_SET([OVN_NB_DB], [unix:"$d"/ovn-nb/ovn-nb.sock]); export $var
> +    AS_VAR_SET([OVN_SB_DB], [unix:"$d"/ovn-sb/ovn-sb.sock]); export $var
> +}
> +
> +# ovs_as AZ
> +ovn_as() {
> +    if test "X$2" != X; then
> +        (ovn_setenv $1; shift; "$@")
> +    else
> +        ovn_setenv $1
> +    fi
> +}
> +
>  # OVN_POPULATE_ARP
>  #
>  # This pre-populates the ARP tables of all of the OVN instances that have
> been
> diff --git a/tests/testsuite.at b/tests/testsuite.at
> index ea6b04a..4ba7cdc 100644
> --- a/tests/testsuite.at
> +++ b/tests/testsuite.at
> @@ -29,4 +29,5 @@ m4_include([tests/ovn-inbctl.at])
>  m4_include([tests/ovn-isbctl.at])
>  m4_include([tests/ovn-controller.at])
>  m4_include([tests/ovn-controller-vtep.at])
> +m4_include([tests/ovn-ic.at])
>  m4_include([tests/checkpatch.at])
> diff --git a/tutorial/ovs-sandbox b/tutorial/ovs-sandbox
> index 510651a..2f17924 100755
> --- a/tutorial/ovs-sandbox
> +++ b/tutorial/ovs-sandbox
> @@ -58,6 +58,8 @@ gdb_vswitchd_ex=false
>  gdb_ovsdb_ex=false
>  gdb_ovn_northd=false
>  gdb_ovn_northd_ex=false
> +gdb_ovn_ic=false
> +gdb_ovn_ic_ex=false
>  gdb_ovn_controller=false
>  gdb_ovn_controller_ex=false
>  gdb_ovn_controller_vtep=false
> @@ -72,13 +74,20 @@ built=false
>  ovn=true
>  ovnsb_schema=
>  ovnnb_schema=
> +ovnisb_schema=
> +ovninb_schema=
>  ovn_rbac=true
>  n_northds=1
> +n_ics=1
>  n_controllers=1
>  nbdb_model=standalone
>  nbdb_servers=3
>  sbdb_model=backup
>  sbdb_servers=3
> +inbdb_model=clustered
> +inbdb_servers=3
> +isbdb_model=clustered
> +isbdb_servers=3
>  dummy=override
>
>  for option; do
> @@ -125,6 +134,7 @@ General options:
>    -g, --gdb-vswitchd   run ovs-vswitchd under gdb
>    -d, --gdb-ovsdb      run ovsdb-server under gdb
>    --gdb-ovn-northd     run ovn-northd under gdb
> +  --gdb-ovn-ic         run ovn-ic under gdb
>    --gdb-ovn-controller run ovn-controller under gdb
>    --gdb-ovn-controller-vtep run ovn-controller-vtep under gdb
>    --dummy=ARG          pass --enable-dummy=ARG to vswitchd (default:
> override)
> @@ -135,10 +145,15 @@ General options:
>  OVN options:
>    --no-ovn-rbac        disable role-based access control for OVN
>    --n-northds=NUMBER   run NUMBER copies of northd (default: 1)
> +  --n-ics=NUMBER       run NUMBER copies of ic (default: 1)
>    --nbdb-model=standalone|backup|clustered    northbound database model
>    --nbdb-servers=N     number of servers in nbdb cluster (default: 3)
>    --sbdb-model=standalone|backup|clustered    southbound database model
>    --sbdb-servers=N     number of servers in sbdb cluster (default: 3)
> +  --inbdb-model=standalone|backup|clustered   ic-northbound database model
> +  --inbdb-servers=N     number of servers in inbdb cluster (default: 3)
> +  --isbdb-model=standalone|backup|clustered   ic-southbound database model
> +  --isbdb-servers=N     number of servers in isbdb cluster (default: 3)
>
>  Other options:
>    -h, --help           Print this usage message.
> @@ -210,6 +225,9 @@ EOF
>          --gdb-ovn-northd)
>              gdb_ovn_northd=true
>              ;;
> +        --gdb-ovn-ic)
> +            gdb_ovn_ic=true
> +            ;;
>          --gdb-ovn-controller)
>              gdb_ovn_controller=true
>              ;;
> @@ -225,6 +243,12 @@ EOF
>          --n-northd*)
>              prev=n_northds
>              ;;
> +        --n-ic*=*)
> +            n_ics=$optarg
> +            ;;
> +        --n-ic*)
> +            prev=n_ics_
> +            ;;
>          --n-controller*=*)
>              n_controllers=$optarg
>              ;;
> @@ -259,10 +283,39 @@ EOF
>          --sbdb-m*)
>              prev=sbdb_model
>              ;;
> +        --inbdb-s*=*)
> +            inbdb_servers=$optarg
> +            inbdb_model=clustered
> +            ;;
> +        --inbdb-s*)
> +            prev=inbdb_servers
> +            inbdb_model=clustered
> +            ;;
> +        --inbdb-m*=*)
> +            inbdb_model=$optarg
> +            ;;
> +        --inbdb-m*)
> +            prev=inbdb_model
> +            ;;
> +        --isbdb-s*=*)
> +            isbdb_servers=$optarg
> +            isbdb_model=clustered
> +            ;;
> +        --isbdb-s*)
> +            prev=isbdb_servers
> +            isbdb_model=clustered
> +            ;;
> +        --isbdb-m*=*)
> +            isbdb_model=$optarg
> +            ;;
> +        --isbdb-m*)
> +            prev=isbdb_model
> +            ;;
>          -R|--gdb-run)
>              gdb_vswitchd_ex=true
>              gdb_ovsdb_ex=true
>              gdb_ovn_northd_ex=true
> +            gdb_ovn_ic_ex=true
>              gdb_ovn_controller_ex=true
>              gdb_ovn_controller_vtep_ex=true
>              ;;
> @@ -327,6 +380,16 @@ if $built; then
>              echo >&2 'source directory not found, please use --srcdir'
>              exit 1
>          fi
> +        ovnisb_schema=$srcdir/ovn-isb.ovsschema
> +        if test ! -e "$ovnisb_schema"; then
> +            echo >&2 'source directory not found, please use --srcdir'
> +            exit 1
> +        fi
> +        ovninb_schema=$srcdir/ovn-inb.ovsschema
> +        if test ! -e "$ovninb_schema"; then
> +            echo >&2 'source directory not found, please use --srcdir'
> +            exit 1
> +        fi
>          vtep_schema=$ovssrcdir/vtep/vtep.ovsschema
>          if test ! -e "$vtep_schema"; then
>              echo >&2 'source directory not found, please use --srcdir'
> @@ -340,7 +403,7 @@ if $built; then
>          exit 1
>      fi
>
>  PATH=$ovsbuilddir/ovsdb:$ovsbuilddir/vswitchd:$ovsbuilddir/utilities:$ovsbuilddir/vtep:$PATH
> -
> PATH=$builddir/controller:$builddir/controller-vtep:$builddir/northd:$builddir/utilities:$PATH
> +
> PATH=$builddir/controller:$builddir/controller-vtep:$builddir/northd:$builddir/ic:$builddir/utilities:$PATH
>      export PATH
>  else
>      case $schema in
> @@ -489,6 +552,8 @@ The backup database file is sandbox/${db}2.db
>  backup_note=
>  ovn_start_db nb "$nbdb_model" "$nbdb_servers" "$ovnnb_schema"
>  ovn_start_db sb "$sbdb_model" "$sbdb_servers" "$ovnsb_schema"
> +ovn_start_db inb "$inbdb_model" "$inbdb_servers" "$ovninb_schema"
> +ovn_start_db isb "$isbdb_model" "$isbdb_servers" "$ovnisb_schema"
>
>  #Add a small delay to allow ovsdb-server to launch.
>  sleep 0.1
> @@ -511,6 +576,9 @@ rungdb $gdb_vswitchd $gdb_vswitchd_ex ovs-vswitchd
> --detach --no-chdir --pidfile
>
>  ovn-nbctl init
>  ovn-sbctl init
> +ovn-inbctl init
> +ovn-isbctl init
> +ovn-nbctl set NB_Global . name=az-1
>
>  ovs-vsctl set open . external-ids:system-id=chassis-1
>  ovs-vsctl set open . external-ids:hostname=sandbox
> @@ -532,6 +600,14 @@ else
>      ovs-vsctl set open . external-ids:ovn-remote=$OVN_SB_DB
>      OVN_CTRLR_PKI=""
>  fi
> +for i in $(seq $n_ics); do
> +    if [ $i -eq 1 ]; then inst=""; else inst=$i; fi
> +    rungdb $gdb_ovn_ic $gdb_ovn_ic_ex ovn-ic --detach \
> +            --no-chdir --pidfile=ovn-ic${inst}.pid -vconsole:off \
> +            --log-file=ovn-ic${inst}.log -vsyslog:off \
> +            --ovnsb-db="$OVN_SB_DB" --ovnnb-db="$OVN_NB_DB" \
> +            --ovnisb-db="$OVN_ISB_DB" --ovninb-db="$OVN_INB_DB"
> +done
>  for i in $(seq $n_northds); do
>      if [ $i -eq 1 ]; then inst=""; else inst=$i; fi
>      rungdb $gdb_ovn_northd $gdb_ovn_northd_ex ovn-northd --detach \
> --
> 2.1.0
>
> _______________________________________________
> dev mailing list
> dev@openvswitch.org
> https://mail.openvswitch.org/mailman/listinfo/ovs-dev
>
Han Zhou Oct. 30, 2019, 9:17 p.m. UTC | #2
On Tue, Oct 29, 2019 at 5:50 PM aginwala <aginwala@asu.edu> wrote:
>
> Thanks Han for the patches.
>
> On Sun, Oct 20, 2019 at 5:54 PM Han Zhou <hzhou@ovn.org> wrote:
>>
>> This patch introduces interconnection controller, ovn-ic, and
>> implements the basic AZ registration feature: taking the AZ
>> name from NB DB and create an Availability_Zone entry in
>> IC-SB DB.
>>
>> Signed-off-by: Han Zhou <hzhou@ovn.org>
>> ---
>>  Makefile.am          |   1 +
>>  ic/.gitignore        |   2 +
>>  ic/automake.mk       |  10 ++
>>  ic/ovn-ic.8.xml      | 111 ++++++++++++++
>>  ic/ovn-ic.c          | 413
+++++++++++++++++++++++++++++++++++++++++++++++++++
>>  ovn-nb.ovsschema     |   5 +-
>>  ovn-nb.xml           |   7 +
>>  tests/automake.mk    |   4 +-
>>  tests/ovn-ic.at      |  29 ++++
>>  tests/ovn-macros.at  | 161 ++++++++++++++++----
>>  tests/testsuite.at   |   1 +
>>  tutorial/ovs-sandbox |  78 +++++++++-
>>  12 files changed, 792 insertions(+), 30 deletions(-)
>>  create mode 100644 ic/.gitignore
>>  create mode 100644 ic/automake.mk
>>  create mode 100644 ic/ovn-ic.8.xml
>>  create mode 100644 ic/ovn-ic.c
>>  create mode 100644 tests/ovn-ic.at
>>
>> diff --git a/Makefile.am b/Makefile.am
>> index 33c18c5..d22a220 100644
>> --- a/Makefile.am
>> +++ b/Makefile.am
>> @@ -500,4 +500,5 @@ include selinux/automake.mk
>>  include controller/automake.mk
>>  include controller-vtep/automake.mk
>>  include northd/automake.mk
>> +include ic/automake.mk
>>  include build-aux/automake.mk
>> diff --git a/ic/.gitignore b/ic/.gitignore
>> new file mode 100644
>> index 0000000..1b73eb4
>> --- /dev/null
>> +++ b/ic/.gitignore
>> @@ -0,0 +1,2 @@
>> +/ovn-ic
>> +/ovn-ic.8
>> diff --git a/ic/automake.mk b/ic/automake.mk
>> new file mode 100644
>> index 0000000..8e71bc3
>> --- /dev/null
>> +++ b/ic/automake.mk
>> @@ -0,0 +1,10 @@
>> +# ovn-ic
>> +bin_PROGRAMS += ic/ovn-ic
>> +ic_ovn_ic_SOURCES = ic/ovn-ic.c
>> +ic_ovn_ic_LDADD = \
>> +       lib/libovn.la \
>> +       $(OVSDB_LIBDIR)/libovsdb.la \
>> +       $(OVS_LIBDIR)/libopenvswitch.la
>> +man_MANS += ic/ovn-ic.8
>> +EXTRA_DIST += ic/ovn-ic.8.xml
>> +CLEANFILES += ic/ovn-ic.8
>> diff --git a/ic/ovn-ic.8.xml b/ic/ovn-ic.8.xml
>> new file mode 100644
>> index 0000000..00f33aa
>> --- /dev/null
>> +++ b/ic/ovn-ic.8.xml
>> @@ -0,0 +1,111 @@
>> +<?xml version="1.0" encoding="utf-8"?>
>> +<manpage program="ovn-ic" section="8" title="ovn-ic">
>> +    <h1>Name</h1>
>> +    <p>ovn-ic -- Open Virtual Network interconnection controller</p>
>> +
>> +    <h1>Synopsis</h1>
>> +    <p><code>ovn-ic</code> [<var>options</var>]</p>
>> +
>> +    <h1>Description</h1>
>> +    <p>
>> +      <code>ovn-ic</code>, OVN interconnection controller, is a
centralized
>> +      daemon which communicates with global interconnection databases
INB/ISB
>> +      to configure and exchange data with local NB/SB for
interconnecting
>> +      with other OVN deployments.
>> +    </p>
>> +
>> +    <h1>Options</h1>
>> +    <dl>
>> +      <dt><code>--ovnnb-db=<var>database</var></code></dt>
>> +      <dd>
>> +        The OVSDB database containing the OVN Northbound Database.  If
the
>> +        <env>OVN_NB_DB</env> environment variable is set, its value is
used
>> +        as the default.  Otherwise, the default is
>> +        <code>unix:@RUNDIR@/ovnnb_db.sock</code>.
>> +      </dd>
>> +      <dt><code>--ovnsb-db=<var>database</var></code></dt>
>> +      <dd>
>> +        The OVSDB database containing the OVN Southbound Database.  If
the
>> +        <env>OVN_SB_DB</env> environment variable is set, its value is
used
>> +        as the default.  Otherwise, the default is
>> +        <code>unix:@RUNDIR@/ovnsb_db.sock</code>.
>> +      </dd>
>> +      <dt><code>--ovninb-db=<var>database</var></code></dt>
>> +      <dd>
>> +        The OVSDB database containing the OVN Interconnection Northbound
>> +        Database.  If the <env>OVN_INB_DB</env> environment variable is
set,
>> +        its value is used as the default.  Otherwise, the default is
>> +        <code>unix:@RUNDIR@/ovninb_db.sock</code>.
>> +      </dd>
>> +      <dt><code>--ovnisb-db=<var>database</var></code></dt>
>> +      <dd>
>> +        The OVSDB database containing the OVN Interconnection Southbound
>> +        Database.  If the <env>OVN_ISB_DB</env> environment variable is
set,
>> +        its value is used as the default.  Otherwise, the default is
>> +        <code>unix:@RUNDIR@/ovnisb_db.sock</code>.
>> +      </dd>
>> +    </dl>
>> +    <p>
>> +      <var>database</var> in the above options must be an OVSDB active
or
>> +      passive connection method, as described in <code>ovsdb</code>(7).
>> +    </p>
>> +
>> +    <h2>Daemon Options</h2>
>> +    <xi:include href="lib/daemon.xml" xmlns:xi="
http://www.w3.org/2003/XInclude"/>
>> +
>> +    <h2>Logging Options</h2>
>> +    <xi:include href="lib/vlog.xml" xmlns:xi="
http://www.w3.org/2003/XInclude"/>
>> +
>> +    <h2>PKI Options</h2>
>> +    <p>
>> +      PKI configuration is required in order to use SSL for the
connections to
>> +      the Northbound and Southbound databases.
>> +    </p>
>> +    <xi:include href="lib/ssl.xml" xmlns:xi="
http://www.w3.org/2003/XInclude"/>
>> +
>> +    <h2>Other Options</h2>
>> +    <xi:include href="lib/unixctl.xml"
>> +     xmlns:xi="http://www.w3.org/2003/XInclude"/>
>> +    <h3></h3>
>> +    <xi:include href="lib/common.xml"
>> +     xmlns:xi="http://www.w3.org/2003/XInclude"/>
>> +
>> +    <h1>Runtime Management Commands</h1>
>> +    <p>
>> +      <code>ovs-appctl</code> can send commands to a running
>> +      <code>ovn-ic</code> process.  The currently supported commands
>> +      are described below.
>> +      <dl>
>> +      <dt><code>exit</code></dt>
>> +      <dd>
>> +        Causes <code>ovn-ic</code> to gracefully terminate.
>> +      </dd>
>> +
>> +      <dt><code>pause</code></dt>
>> +      <dd>
>> +        Pauses the ovn-ic operation from processing any Northbound and
>> +        Southbound database changes.
>> +      </dd>
>> +
>> +      <dt><code>resume</code></dt>
>> +      <dd>
>> +        Resumes the ovn-ic operation to process Northbound and
>> +        Southbound database contents and generate logical flows.
>> +      </dd>
>> +
>> +      <dt><code>is-paused</code></dt>
>> +      <dd>
>> +        Returns "true" if ovn-ic is currently paused, "false" otherwise.
>> +      </dd>
>> +      </dl>
>> +    </p>
>> +
>> +    <h1>Active-Standby for High Availability</h1>
>> +    <p>
>> +      You may run <code>ovn-ic</code> more than once in an OVN
deployment.
>> +      OVN will automatically ensure that only one of them is active at
a time.
>> +      If multiple instances of <code>ovn-ic</code> are running and the
>> +      active <code>ovn-ic</code> fails, one of the hot standby instances
>> +      of <code>ovn-ic</code> will automatically take over.
>> +    </p>
>> +</manpage>
>> diff --git a/ic/ovn-ic.c b/ic/ovn-ic.c
>> new file mode 100644
>> index 0000000..7cc80bc
>> --- /dev/null
>> +++ b/ic/ovn-ic.c
>> @@ -0,0 +1,413 @@
>> +/*
>> + * Licensed under the Apache License, Version 2.0 (the "License");
>> + * you may not use this file except in compliance with the License.
>> + * You may obtain a copy of the License at:
>> + *
>> + *     http://www.apache.org/licenses/LICENSE-2.0
>> + *
>> + * Unless required by applicable law or agreed to in writing, software
>> + * distributed under the License is distributed on an "AS IS" BASIS,
>> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied.
>> + * See the License for the specific language governing permissions and
>> + * limitations under the License.
>> + */
>> +
>> +#include <config.h>
>> +
>> +#include <getopt.h>
>> +#include <stdlib.h>
>> +#include <stdio.h>
>> +
>> +#include "bitmap.h"
>> +#include "command-line.h"
>> +#include "daemon.h"
>> +#include "dirs.h"
>> +#include "openvswitch/dynamic-string.h"
>> +#include "fatal-signal.h"
>> +#include "hash.h"
>> +#include "openvswitch/hmap.h"
>> +#include "lib/ovn-inb-idl.h"
>> +#include "lib/ovn-isb-idl.h"
>> +#include "lib/ovn-nb-idl.h"
>> +#include "lib/ovn-sb-idl.h"
>> +#include "lib/ovn-util.h"
>> +#include "openvswitch/poll-loop.h"
>> +#include "smap.h"
>> +#include "sset.h"
>> +#include "stream.h"
>> +#include "stream-ssl.h"
>> +#include "unixctl.h"
>> +#include "util.h"
>> +#include "uuid.h"
>> +#include "openvswitch/vlog.h"
>> +
>> +VLOG_DEFINE_THIS_MODULE(ovn_ic);
>> +
>> +static unixctl_cb_func ovn_ic_exit;
>> +static unixctl_cb_func ovn_ic_pause;
>> +static unixctl_cb_func ovn_ic_resume;
>> +static unixctl_cb_func ovn_ic_is_paused;
>> +
>> +struct ic_context {
>> +    struct ovsdb_idl *ovnnb_idl;
>> +    struct ovsdb_idl *ovnsb_idl;
>> +    struct ovsdb_idl *ovninb_idl;
>> +    struct ovsdb_idl *ovnisb_idl;
>> +    struct ovsdb_idl_txn *ovnnb_txn;
>> +    struct ovsdb_idl_txn *ovnsb_txn;
>> +    struct ovsdb_idl_txn *ovninb_txn;
>> +    struct ovsdb_idl_txn *ovnisb_txn;
>> +};
>> +
>> +static const char *ovnnb_db;
>> +static const char *ovnsb_db;
>> +static const char *ovninb_db;
>> +static const char *ovnisb_db;
>> +static const char *unixctl_path;
>> +
>> +
>> +static void
>> +usage(void)
>> +{
>> +    printf("\
>> +%s: OVN interconnection management daemon\n\
>> +usage: %s [OPTIONS]\n\
>> +\n\
>> +Options:\n\
>> +  --ovnnb-db=DATABASE       connect to ovn-nb database at DATABASE\n\
>> +                            (default: %s)\n\
>> +  --ovnsb-db=DATABASE       connect to ovn-sb database at DATABASE\n\
>> +                            (default: %s)\n\
>> +  --unixctl=SOCKET          override default control socket name\n\
>> +  -h, --help                display this help message\n\
>> +  -o, --options             list available options\n\
>> +  -V, --version             display version information\n\
>> +", program_name, program_name, default_nb_db(), default_sb_db());
>> +    daemon_usage();
>> +    vlog_usage();
>> +    stream_usage("database", true, true, false);
>> +}
>> +
>> +static const struct isbrec_availability_zone *
>> +az_run(struct ic_context *ctx)
>> +{
>> +    const struct nbrec_nb_global *nb_global =
>> +        nbrec_nb_global_first(ctx->ovnnb_idl);
>> +
>> +    if (!nb_global) {
>> +        VLOG_INFO("NB Global not exist.");
>> +        return NULL;
>> +    }
>> +
>> +    /* Delete old AZ if name changes.  Note: if name changed when ovn-ic
>> +     * is not running, one has to manually delete the old AZ with:
>> +     * "ovn-isbctl destroy avail <az>". */
>> +    static char *az_name;
>> +    const struct isbrec_availability_zone *az;
>> +    if (az_name && strcmp(az_name, nb_global->name)) {
>> +        ISBREC_AVAILABILITY_ZONE_FOR_EACH (az, ctx->ovnisb_idl) {
>> +            if (!strcmp(az->name, az_name)) {
>> +                isbrec_availability_zone_delete(az);
>> +                break;
>> +            }
>> +        }
>> +        free(az_name);
>> +        az_name = NULL;
>> +    }
>> +
>> +    if (!nb_global->name[0]) {
>> +        return NULL;
>> +    }
>> +
>> +    if (!az_name) {
>> +        az_name = xstrdup(nb_global->name);
>> +    }
>> +
>> +    ISBREC_AVAILABILITY_ZONE_FOR_EACH (az, ctx->ovnisb_idl) {
>> +        if (!strcmp(az->name, az_name)) {
>> +            return az;
>> +        }
>> +    }
>> +
>> +    /* Create AZ in ISB */
>> +    if (ctx->ovnisb_txn) {
>> +        VLOG_INFO("Register AZ %s to interconnection DB.", az_name);
>> +        az = isbrec_availability_zone_insert(ctx->ovnisb_txn);
>> +        isbrec_availability_zone_set_name(az, az_name);
>> +        return az;
>> +    }
>> +    return NULL;
>> +}
>> +
>> +static void
>> +ovn_db_run(struct ic_context *ctx)
>> +{
>> +    const struct isbrec_availability_zone *az = az_run(ctx);
>> +    VLOG_DBG("Availability zone: %s", az ? az->name : "not created
yet.");
>> +}
>> +
>> +static void
>> +parse_options(int argc OVS_UNUSED, char *argv[] OVS_UNUSED)
>> +{
>> +    enum {
>> +        DAEMON_OPTION_ENUMS,
>> +        VLOG_OPTION_ENUMS,
>> +        SSL_OPTION_ENUMS,
>> +    };
>> +    static const struct option long_options[] = {
>> +        {"ovnsb-db", required_argument, NULL, 'd'},
>> +        {"ovnnb-db", required_argument, NULL, 'D'},
>> +        {"ovnisb-db", required_argument, NULL, 'i'},
>> +        {"ovninb-db", required_argument, NULL, 'I'},
>> +        {"unixctl", required_argument, NULL, 'u'},
>> +        {"help", no_argument, NULL, 'h'},
>> +        {"options", no_argument, NULL, 'o'},
>> +        {"version", no_argument, NULL, 'V'},
>> +        DAEMON_LONG_OPTIONS,
>> +        VLOG_LONG_OPTIONS,
>> +        STREAM_SSL_LONG_OPTIONS,
>> +        {NULL, 0, NULL, 0},
>> +    };
>> +    char *short_options =
ovs_cmdl_long_options_to_short_options(long_options);
>> +
>> +    for (;;) {
>> +        int c;
>> +
>> +        c = getopt_long(argc, argv, short_options, long_options, NULL);
>> +        if (c == -1) {
>> +            break;
>> +        }
>> +
>> +        switch (c) {
>> +        DAEMON_OPTION_HANDLERS;
>> +        VLOG_OPTION_HANDLERS;
>> +        STREAM_SSL_OPTION_HANDLERS;
>> +
>> +        case 'd':
>> +            ovnsb_db = optarg;
>> +            break;
>> +
>> +        case 'D':
>> +            ovnnb_db = optarg;
>> +            break;
>> +
>> +        case 'u':
>> +            unixctl_path = optarg;
>> +            break;
>> +
>> +        case 'h':
>> +            usage();
>> +            exit(EXIT_SUCCESS);
>> +
>> +        case 'o':
>> +            ovs_cmdl_print_options(long_options);
>> +            exit(EXIT_SUCCESS);
>> +
>> +        case 'V':
>> +            ovs_print_version(0, 0);
>> +            exit(EXIT_SUCCESS);
>> +
>> +        default:
>> +            break;
>> +        }
>> +    }
>> +
>> +    if (!ovnsb_db) {
>> +        ovnsb_db = default_sb_db();
>> +    }
>> +
>> +    if (!ovnnb_db) {
>> +        ovnnb_db = default_nb_db();
>> +    }
>> +
>> +    if (!ovnisb_db) {
>> +        ovnisb_db = default_isb_db();
>
> This still did not work if we are starting ic controller on different azs
with remote="tcp:10.xxx:6645/46" after patching ovn-ctl. As per code it
checks env vars  OVN_INB/SB_DB without passing values in command line. So
rectification is needed here.

Thanks Ali for reporting the problem. I fixed it in v2:
https://patchwork.ozlabs.org/project/openvswitch/list/?series=139731

Patch
diff mbox series

diff --git a/Makefile.am b/Makefile.am
index 33c18c5..d22a220 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -500,4 +500,5 @@  include selinux/automake.mk
 include controller/automake.mk
 include controller-vtep/automake.mk
 include northd/automake.mk
+include ic/automake.mk
 include build-aux/automake.mk
diff --git a/ic/.gitignore b/ic/.gitignore
new file mode 100644
index 0000000..1b73eb4
--- /dev/null
+++ b/ic/.gitignore
@@ -0,0 +1,2 @@ 
+/ovn-ic
+/ovn-ic.8
diff --git a/ic/automake.mk b/ic/automake.mk
new file mode 100644
index 0000000..8e71bc3
--- /dev/null
+++ b/ic/automake.mk
@@ -0,0 +1,10 @@ 
+# ovn-ic
+bin_PROGRAMS += ic/ovn-ic
+ic_ovn_ic_SOURCES = ic/ovn-ic.c
+ic_ovn_ic_LDADD = \
+	lib/libovn.la \
+	$(OVSDB_LIBDIR)/libovsdb.la \
+	$(OVS_LIBDIR)/libopenvswitch.la
+man_MANS += ic/ovn-ic.8
+EXTRA_DIST += ic/ovn-ic.8.xml
+CLEANFILES += ic/ovn-ic.8
diff --git a/ic/ovn-ic.8.xml b/ic/ovn-ic.8.xml
new file mode 100644
index 0000000..00f33aa
--- /dev/null
+++ b/ic/ovn-ic.8.xml
@@ -0,0 +1,111 @@ 
+<?xml version="1.0" encoding="utf-8"?>
+<manpage program="ovn-ic" section="8" title="ovn-ic">
+    <h1>Name</h1>
+    <p>ovn-ic -- Open Virtual Network interconnection controller</p>
+
+    <h1>Synopsis</h1>
+    <p><code>ovn-ic</code> [<var>options</var>]</p>
+
+    <h1>Description</h1>
+    <p>
+      <code>ovn-ic</code>, OVN interconnection controller, is a centralized
+      daemon which communicates with global interconnection databases INB/ISB
+      to configure and exchange data with local NB/SB for interconnecting
+      with other OVN deployments.
+    </p>
+
+    <h1>Options</h1>
+    <dl>
+      <dt><code>--ovnnb-db=<var>database</var></code></dt>
+      <dd>
+        The OVSDB database containing the OVN Northbound Database.  If the
+        <env>OVN_NB_DB</env> environment variable is set, its value is used
+        as the default.  Otherwise, the default is
+        <code>unix:@RUNDIR@/ovnnb_db.sock</code>.
+      </dd>
+      <dt><code>--ovnsb-db=<var>database</var></code></dt>
+      <dd>
+        The OVSDB database containing the OVN Southbound Database.  If the
+        <env>OVN_SB_DB</env> environment variable is set, its value is used
+        as the default.  Otherwise, the default is
+        <code>unix:@RUNDIR@/ovnsb_db.sock</code>.
+      </dd>
+      <dt><code>--ovninb-db=<var>database</var></code></dt>
+      <dd>
+        The OVSDB database containing the OVN Interconnection Northbound
+        Database.  If the <env>OVN_INB_DB</env> environment variable is set,
+        its value is used as the default.  Otherwise, the default is
+        <code>unix:@RUNDIR@/ovninb_db.sock</code>.
+      </dd>
+      <dt><code>--ovnisb-db=<var>database</var></code></dt>
+      <dd>
+        The OVSDB database containing the OVN Interconnection Southbound
+        Database.  If the <env>OVN_ISB_DB</env> environment variable is set,
+        its value is used as the default.  Otherwise, the default is
+        <code>unix:@RUNDIR@/ovnisb_db.sock</code>.
+      </dd>
+    </dl>
+    <p>
+      <var>database</var> in the above options must be an OVSDB active or
+      passive connection method, as described in <code>ovsdb</code>(7).
+    </p>
+
+    <h2>Daemon Options</h2>
+    <xi:include href="lib/daemon.xml" xmlns:xi="http://www.w3.org/2003/XInclude"/>
+
+    <h2>Logging Options</h2>
+    <xi:include href="lib/vlog.xml" xmlns:xi="http://www.w3.org/2003/XInclude"/>
+
+    <h2>PKI Options</h2>
+    <p>
+      PKI configuration is required in order to use SSL for the connections to
+      the Northbound and Southbound databases.
+    </p>
+    <xi:include href="lib/ssl.xml" xmlns:xi="http://www.w3.org/2003/XInclude"/>
+
+    <h2>Other Options</h2>
+    <xi:include href="lib/unixctl.xml"
+     xmlns:xi="http://www.w3.org/2003/XInclude"/>
+    <h3></h3>
+    <xi:include href="lib/common.xml"
+     xmlns:xi="http://www.w3.org/2003/XInclude"/>
+
+    <h1>Runtime Management Commands</h1>
+    <p>
+      <code>ovs-appctl</code> can send commands to a running
+      <code>ovn-ic</code> process.  The currently supported commands
+      are described below.
+      <dl>
+      <dt><code>exit</code></dt>
+      <dd>
+        Causes <code>ovn-ic</code> to gracefully terminate.
+      </dd>
+
+      <dt><code>pause</code></dt>
+      <dd>
+        Pauses the ovn-ic operation from processing any Northbound and
+        Southbound database changes.
+      </dd>
+
+      <dt><code>resume</code></dt>
+      <dd>
+        Resumes the ovn-ic operation to process Northbound and
+        Southbound database contents and generate logical flows.
+      </dd>
+
+      <dt><code>is-paused</code></dt>
+      <dd>
+        Returns "true" if ovn-ic is currently paused, "false" otherwise.
+      </dd>
+      </dl>
+    </p>
+
+    <h1>Active-Standby for High Availability</h1>
+    <p>
+      You may run <code>ovn-ic</code> more than once in an OVN deployment.
+      OVN will automatically ensure that only one of them is active at a time.
+      If multiple instances of <code>ovn-ic</code> are running and the
+      active <code>ovn-ic</code> fails, one of the hot standby instances
+      of <code>ovn-ic</code> will automatically take over.
+    </p>
+</manpage>
diff --git a/ic/ovn-ic.c b/ic/ovn-ic.c
new file mode 100644
index 0000000..7cc80bc
--- /dev/null
+++ b/ic/ovn-ic.c
@@ -0,0 +1,413 @@ 
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <config.h>
+
+#include <getopt.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+#include "bitmap.h"
+#include "command-line.h"
+#include "daemon.h"
+#include "dirs.h"
+#include "openvswitch/dynamic-string.h"
+#include "fatal-signal.h"
+#include "hash.h"
+#include "openvswitch/hmap.h"
+#include "lib/ovn-inb-idl.h"
+#include "lib/ovn-isb-idl.h"
+#include "lib/ovn-nb-idl.h"
+#include "lib/ovn-sb-idl.h"
+#include "lib/ovn-util.h"
+#include "openvswitch/poll-loop.h"
+#include "smap.h"
+#include "sset.h"
+#include "stream.h"
+#include "stream-ssl.h"
+#include "unixctl.h"
+#include "util.h"
+#include "uuid.h"
+#include "openvswitch/vlog.h"
+
+VLOG_DEFINE_THIS_MODULE(ovn_ic);
+
+static unixctl_cb_func ovn_ic_exit;
+static unixctl_cb_func ovn_ic_pause;
+static unixctl_cb_func ovn_ic_resume;
+static unixctl_cb_func ovn_ic_is_paused;
+
+struct ic_context {
+    struct ovsdb_idl *ovnnb_idl;
+    struct ovsdb_idl *ovnsb_idl;
+    struct ovsdb_idl *ovninb_idl;
+    struct ovsdb_idl *ovnisb_idl;
+    struct ovsdb_idl_txn *ovnnb_txn;
+    struct ovsdb_idl_txn *ovnsb_txn;
+    struct ovsdb_idl_txn *ovninb_txn;
+    struct ovsdb_idl_txn *ovnisb_txn;
+};
+
+static const char *ovnnb_db;
+static const char *ovnsb_db;
+static const char *ovninb_db;
+static const char *ovnisb_db;
+static const char *unixctl_path;
+
+
+static void
+usage(void)
+{
+    printf("\
+%s: OVN interconnection management daemon\n\
+usage: %s [OPTIONS]\n\
+\n\
+Options:\n\
+  --ovnnb-db=DATABASE       connect to ovn-nb database at DATABASE\n\
+                            (default: %s)\n\
+  --ovnsb-db=DATABASE       connect to ovn-sb database at DATABASE\n\
+                            (default: %s)\n\
+  --unixctl=SOCKET          override default control socket name\n\
+  -h, --help                display this help message\n\
+  -o, --options             list available options\n\
+  -V, --version             display version information\n\
+", program_name, program_name, default_nb_db(), default_sb_db());
+    daemon_usage();
+    vlog_usage();
+    stream_usage("database", true, true, false);
+}
+
+static const struct isbrec_availability_zone *
+az_run(struct ic_context *ctx)
+{
+    const struct nbrec_nb_global *nb_global =
+        nbrec_nb_global_first(ctx->ovnnb_idl);
+
+    if (!nb_global) {
+        VLOG_INFO("NB Global not exist.");
+        return NULL;
+    }
+
+    /* Delete old AZ if name changes.  Note: if name changed when ovn-ic
+     * is not running, one has to manually delete the old AZ with:
+     * "ovn-isbctl destroy avail <az>". */
+    static char *az_name;
+    const struct isbrec_availability_zone *az;
+    if (az_name && strcmp(az_name, nb_global->name)) {
+        ISBREC_AVAILABILITY_ZONE_FOR_EACH (az, ctx->ovnisb_idl) {
+            if (!strcmp(az->name, az_name)) {
+                isbrec_availability_zone_delete(az);
+                break;
+            }
+        }
+        free(az_name);
+        az_name = NULL;
+    }
+
+    if (!nb_global->name[0]) {
+        return NULL;
+    }
+
+    if (!az_name) {
+        az_name = xstrdup(nb_global->name);
+    }
+
+    ISBREC_AVAILABILITY_ZONE_FOR_EACH (az, ctx->ovnisb_idl) {
+        if (!strcmp(az->name, az_name)) {
+            return az;
+        }
+    }
+
+    /* Create AZ in ISB */
+    if (ctx->ovnisb_txn) {
+        VLOG_INFO("Register AZ %s to interconnection DB.", az_name);
+        az = isbrec_availability_zone_insert(ctx->ovnisb_txn);
+        isbrec_availability_zone_set_name(az, az_name);
+        return az;
+    }
+    return NULL;
+}
+
+static void
+ovn_db_run(struct ic_context *ctx)
+{
+    const struct isbrec_availability_zone *az = az_run(ctx);
+    VLOG_DBG("Availability zone: %s", az ? az->name : "not created yet.");
+}
+
+static void
+parse_options(int argc OVS_UNUSED, char *argv[] OVS_UNUSED)
+{
+    enum {
+        DAEMON_OPTION_ENUMS,
+        VLOG_OPTION_ENUMS,
+        SSL_OPTION_ENUMS,
+    };
+    static const struct option long_options[] = {
+        {"ovnsb-db", required_argument, NULL, 'd'},
+        {"ovnnb-db", required_argument, NULL, 'D'},
+        {"ovnisb-db", required_argument, NULL, 'i'},
+        {"ovninb-db", required_argument, NULL, 'I'},
+        {"unixctl", required_argument, NULL, 'u'},
+        {"help", no_argument, NULL, 'h'},
+        {"options", no_argument, NULL, 'o'},
+        {"version", no_argument, NULL, 'V'},
+        DAEMON_LONG_OPTIONS,
+        VLOG_LONG_OPTIONS,
+        STREAM_SSL_LONG_OPTIONS,
+        {NULL, 0, NULL, 0},
+    };
+    char *short_options = ovs_cmdl_long_options_to_short_options(long_options);
+
+    for (;;) {
+        int c;
+
+        c = getopt_long(argc, argv, short_options, long_options, NULL);
+        if (c == -1) {
+            break;
+        }
+
+        switch (c) {
+        DAEMON_OPTION_HANDLERS;
+        VLOG_OPTION_HANDLERS;
+        STREAM_SSL_OPTION_HANDLERS;
+
+        case 'd':
+            ovnsb_db = optarg;
+            break;
+
+        case 'D':
+            ovnnb_db = optarg;
+            break;
+
+        case 'u':
+            unixctl_path = optarg;
+            break;
+
+        case 'h':
+            usage();
+            exit(EXIT_SUCCESS);
+
+        case 'o':
+            ovs_cmdl_print_options(long_options);
+            exit(EXIT_SUCCESS);
+
+        case 'V':
+            ovs_print_version(0, 0);
+            exit(EXIT_SUCCESS);
+
+        default:
+            break;
+        }
+    }
+
+    if (!ovnsb_db) {
+        ovnsb_db = default_sb_db();
+    }
+
+    if (!ovnnb_db) {
+        ovnnb_db = default_nb_db();
+    }
+
+    if (!ovnisb_db) {
+        ovnisb_db = default_isb_db();
+    }
+
+    if (!ovninb_db) {
+        ovninb_db = default_inb_db();
+    }
+
+    free(short_options);
+}
+
+static void OVS_UNUSED
+add_column_noalert(struct ovsdb_idl *idl,
+                   const struct ovsdb_idl_column *column)
+{
+    ovsdb_idl_add_column(idl, column);
+    ovsdb_idl_omit_alert(idl, column);
+}
+
+int
+main(int argc, char *argv[])
+{
+    int res = EXIT_SUCCESS;
+    struct unixctl_server *unixctl;
+    int retval;
+    bool exiting;
+    bool paused;
+
+    fatal_ignore_sigpipe();
+    ovs_cmdl_proctitle_init(argc, argv);
+    set_program_name(argv[0]);
+    service_start(&argc, &argv);
+    parse_options(argc, argv);
+
+    daemonize_start(false);
+
+    if (!unixctl_path) {
+        char *abs_unixctl_path = get_abs_unix_ctl_path();
+        retval = unixctl_server_create(abs_unixctl_path, &unixctl);
+        free(abs_unixctl_path);
+    } else {
+        retval = unixctl_server_create(unixctl_path, &unixctl);
+    }
+
+    if (retval) {
+        exit(EXIT_FAILURE);
+    }
+    unixctl_command_register("exit", "", 0, 0, ovn_ic_exit, &exiting);
+    unixctl_command_register("pause", "", 0, 0, ovn_ic_pause, &paused);
+    unixctl_command_register("resume", "", 0, 0, ovn_ic_resume, &paused);
+    unixctl_command_register("is-paused", "", 0, 0, ovn_ic_is_paused,
+                             &paused);
+
+    daemonize_complete();
+
+    /* ovn-inb db. */
+    struct ovsdb_idl_loop ovninb_idl_loop = OVSDB_IDL_LOOP_INITIALIZER(
+        ovsdb_idl_create(ovninb_db, &inbrec_idl_class, true, true));
+
+    /* ovn-isb db. */
+    struct ovsdb_idl_loop ovnisb_idl_loop = OVSDB_IDL_LOOP_INITIALIZER(
+        ovsdb_idl_create(ovnisb_db, &isbrec_idl_class, true, true));
+
+    /* ovn-nb db. XXX: add only needed tables and columns */
+    struct ovsdb_idl_loop ovnnb_idl_loop = OVSDB_IDL_LOOP_INITIALIZER(
+        ovsdb_idl_create(ovnnb_db, &nbrec_idl_class, true, true));
+
+    /* ovn-sb db. XXX: add only needed tables and columns */
+    struct ovsdb_idl_loop ovnsb_idl_loop = OVSDB_IDL_LOOP_INITIALIZER(
+        ovsdb_idl_create(ovnsb_db, &sbrec_idl_class, true, true));
+
+    /* Ensure that only a single ovn-ic is active in the deployment by
+     * acquiring a lock called "ovn_ic" on the southbound database
+     * and then only performing DB transactions if the lock is held. */
+    ovsdb_idl_set_lock(ovnsb_idl_loop.idl, "ovn_ic");
+    bool had_lock = false;
+
+    /* Main loop. */
+    exiting = false;
+    paused = false;
+    while (!exiting) {
+        if (!paused) {
+            struct ic_context ctx = {
+                .ovnnb_idl = ovnnb_idl_loop.idl,
+                .ovnnb_txn = ovsdb_idl_loop_run(&ovnnb_idl_loop),
+                .ovnsb_idl = ovnsb_idl_loop.idl,
+                .ovnsb_txn = ovsdb_idl_loop_run(&ovnsb_idl_loop),
+                .ovninb_idl = ovninb_idl_loop.idl,
+                .ovninb_txn = ovsdb_idl_loop_run(&ovninb_idl_loop),
+                .ovnisb_idl = ovnisb_idl_loop.idl,
+                .ovnisb_txn = ovsdb_idl_loop_run(&ovnisb_idl_loop),
+            };
+
+            if (!had_lock && ovsdb_idl_has_lock(ovnsb_idl_loop.idl)) {
+                VLOG_INFO("ovn-ic lock acquired. "
+                        "This ovn-ic instance is now active.");
+                had_lock = true;
+            } else if (had_lock && !ovsdb_idl_has_lock(ovnsb_idl_loop.idl)) {
+                VLOG_INFO("ovn-ic lock lost. "
+                        "This ovn-ic instance is now on standby.");
+                had_lock = false;
+            }
+
+            if (ovsdb_idl_has_lock(ovnsb_idl_loop.idl)) {
+                ovn_db_run(&ctx);
+            }
+
+            ovsdb_idl_loop_commit_and_wait(&ovnnb_idl_loop);
+            ovsdb_idl_loop_commit_and_wait(&ovnsb_idl_loop);
+            ovsdb_idl_loop_commit_and_wait(&ovninb_idl_loop);
+            ovsdb_idl_loop_commit_and_wait(&ovnisb_idl_loop);
+        } else {
+            /* ovn-ic is paused
+             *    - we still want to handle any db updates and update the
+             *      local IDL. Otherwise, when it is resumed, the local IDL
+             *      copy will be out of sync.
+             *    - but we don't want to create any txns.
+             * */
+            ovsdb_idl_run(ovnnb_idl_loop.idl);
+            ovsdb_idl_run(ovnsb_idl_loop.idl);
+            ovsdb_idl_run(ovninb_idl_loop.idl);
+            ovsdb_idl_run(ovnisb_idl_loop.idl);
+            ovsdb_idl_wait(ovnnb_idl_loop.idl);
+            ovsdb_idl_wait(ovnsb_idl_loop.idl);
+            ovsdb_idl_wait(ovninb_idl_loop.idl);
+            ovsdb_idl_wait(ovnisb_idl_loop.idl);
+        }
+
+        unixctl_server_run(unixctl);
+        unixctl_server_wait(unixctl);
+        if (exiting) {
+            poll_immediate_wake();
+        }
+
+        poll_block();
+        if (should_service_stop()) {
+            exiting = true;
+        }
+    }
+
+    unixctl_server_destroy(unixctl);
+    ovsdb_idl_loop_destroy(&ovnnb_idl_loop);
+    ovsdb_idl_loop_destroy(&ovnsb_idl_loop);
+    ovsdb_idl_loop_destroy(&ovninb_idl_loop);
+    ovsdb_idl_loop_destroy(&ovnisb_idl_loop);
+    service_stop();
+
+    exit(res);
+}
+
+static void
+ovn_ic_exit(struct unixctl_conn *conn, int argc OVS_UNUSED,
+                const char *argv[] OVS_UNUSED, void *exiting_)
+{
+    bool *exiting = exiting_;
+    *exiting = true;
+
+    unixctl_command_reply(conn, NULL);
+}
+
+static void
+ovn_ic_pause(struct unixctl_conn *conn, int argc OVS_UNUSED,
+                const char *argv[] OVS_UNUSED, void *pause_)
+{
+    bool *pause = pause_;
+    *pause = true;
+
+    unixctl_command_reply(conn, NULL);
+}
+
+static void
+ovn_ic_resume(struct unixctl_conn *conn, int argc OVS_UNUSED,
+                  const char *argv[] OVS_UNUSED, void *pause_)
+{
+    bool *pause = pause_;
+    *pause = false;
+
+    unixctl_command_reply(conn, NULL);
+}
+
+static void
+ovn_ic_is_paused(struct unixctl_conn *conn, int argc OVS_UNUSED,
+                     const char *argv[] OVS_UNUSED, void *paused_)
+{
+    bool *paused = paused_;
+    if (*paused) {
+        unixctl_command_reply(conn, "true");
+    } else {
+        unixctl_command_reply(conn, "false");
+    }
+}
diff --git a/ovn-nb.ovsschema b/ovn-nb.ovsschema
index 2c87cbb..1a9acdb 100644
--- a/ovn-nb.ovsschema
+++ b/ovn-nb.ovsschema
@@ -1,10 +1,11 @@ 
 {
     "name": "OVN_Northbound",
-    "version": "5.16.0",
-    "cksum": "923459061 23095",
+    "version": "5.16.1",
+    "cksum": "2818115555 23139",
     "tables": {
         "NB_Global": {
             "columns": {
+                "name": {"type": "string"},
                 "nb_cfg": {"type": {"key": "integer"}},
                 "sb_cfg": {"type": {"key": "integer"}},
                 "hv_cfg": {"type": {"key": "integer"}},
diff --git a/ovn-nb.xml b/ovn-nb.xml
index 1504f8f..05091fb 100644
--- a/ovn-nb.xml
+++ b/ovn-nb.xml
@@ -36,6 +36,13 @@ 
       one row.
     </p>
 
+    <group title="Identity">
+      <column name="name">
+        The name of the OVN cluster, which uniquely identifies the OVN cluster
+        throughout all OVN clusters supposed to interconnect with each other.
+      </column>
+    </group>
+
     <group title="Status">
       These columns allow a client to track the overall configuration state of
       the system.
diff --git a/tests/automake.mk b/tests/automake.mk
index c941ec2..8e12afd 100644
--- a/tests/automake.mk
+++ b/tests/automake.mk
@@ -28,6 +28,7 @@  TESTSUITE_AT = \
 	tests/ovn-isbctl.at \
 	tests/ovn-controller.at \
 	tests/ovn-controller-vtep.at \
+	tests/ovn-ic.at \
 	tests/ovn-macros.at \
 	tests/ovn-performance.at
 
@@ -54,7 +55,7 @@  SYSTEM_KMOD_TESTSUITE = $(srcdir)/tests/system-kmod-testsuite
 SYSTEM_USERSPACE_TESTSUITE = $(srcdir)/tests/system-userspace-testsuite
 DISTCLEANFILES += tests/atconfig tests/atlocal
 
-AUTOTEST_PATH = $(ovs_builddir)/utilities:$(ovs_builddir)/vswitchd:$(ovs_builddir)/ovsdb:$(ovs_builddir)/vtep:tests:$(PTHREAD_WIN32_DIR_DLL):$(SSL_DIR):controller-vtep:northd:utilities:controller
+AUTOTEST_PATH = $(ovs_builddir)/utilities:$(ovs_builddir)/vswitchd:$(ovs_builddir)/ovsdb:$(ovs_builddir)/vtep:tests:$(PTHREAD_WIN32_DIR_DLL):$(SSL_DIR):controller-vtep:northd:utilities:controller:ic
 
 check-local:
 	set $(SHELL) '$(TESTSUITE)' -C tests AUTOTEST_PATH=$(AUTOTEST_PATH) ovs_srcdir=$(ovs_srcdir); \
@@ -101,6 +102,7 @@  valgrind_wrappers = \
 	tests/valgrind/ovn-sbctl \
 	tests/valgrind/ovn-inbctl \
 	tests/valgrind/ovn-isbctl \
+	tests/valgrind/ovn-ic \
 	tests/valgrind/ovs-appctl \
 	tests/valgrind/ovs-ofctl \
 	tests/valgrind/ovs-vsctl \
diff --git a/tests/ovn-ic.at b/tests/ovn-ic.at
new file mode 100644
index 0000000..1e69820
--- /dev/null
+++ b/tests/ovn-ic.at
@@ -0,0 +1,29 @@ 
+AT_BANNER([OVN Interconnection Controller])
+AT_SETUP([ovn-ic -- AZ register])
+AT_SKIP_IF([test $HAVE_PYTHON = no])
+
+ovn_init_ic_db
+ovn_start az1
+ovn_start az2
+
+AT_CHECK([ovn-isbctl show], [0], [dnl
+availability-zone az1
+availability-zone az2
+])
+
+ovn_as az1
+ovn-nbctl set NB_Global . name=az3
+AT_CHECK([ovn-isbctl show], [0], [dnl
+availability-zone az2
+availability-zone az3
+])
+
+ovn_as az2
+ovn-nbctl set NB_Global . name=\"\"
+AT_CHECK([ovn-isbctl show], [0], [dnl
+availability-zone az3
+])
+
+OVN_CLEANUP_IC([az1], [az2])
+
+AT_CLEANUP
diff --git a/tests/ovn-macros.at b/tests/ovn-macros.at
index 4045ce0..ea080dc 100644
--- a/tests/ovn-macros.at
+++ b/tests/ovn-macros.at
@@ -48,9 +48,49 @@  m4_define([OVN_CLEANUP],[
     OVN_CLEANUP_VSWITCH([main])
 ])
 
+# OVN_CLEANUP_AZ(az)
+#
+# Gracefully terminate all OVN daemons, including those in the
+# specified sandbox instances.
+m4_define([OVN_CLEANUP_AZ],[
+    as $1/ovn-sb
+    OVS_APP_EXIT_AND_WAIT([ovsdb-server])
+
+    as $1/ovn-nb
+    OVS_APP_EXIT_AND_WAIT([ovsdb-server])
+
+    as $1/northd
+    OVS_APP_EXIT_AND_WAIT([ovn-northd])
+
+    as $1/northd-backup
+    OVS_APP_EXIT_AND_WAIT([ovn-northd])
+
+    as $1/ic
+    OVS_APP_EXIT_AND_WAIT([ovn-ic])
+])
+
+# OVN_CLEANUP_IC([az ...])
+#
+# Gracefully terminate all interconnection DBs, and daemons in the
+# specified AZs, if any.
+m4_define([OVN_CLEANUP_IC],[
+    m4_foreach([az], [$@], [
+        OVN_CLEANUP_AZ([az])
+    ])
+    as ovn-isb
+    OVS_APP_EXIT_AND_WAIT([ovsdb-server])
+
+    as ovn-inb
+    OVS_APP_EXIT_AND_WAIT([ovsdb-server])
+
+    if test -d "$ovs_base"/main; then
+        OVN_CLEANUP_VSWITCH([main])
+    fi
+])
+
 m4_divert_push([PREPARE_TESTS])
 
-# ovn_init_db DATABASE
+# ovn_init_db DATABASE [AZ]
 #
 # Creates and initializes the given DATABASE (one of "ovn-sb" or "ovn-nb"),
 # starts its ovsdb-server instance, and sets the appropriate environment
@@ -60,36 +100,77 @@  m4_divert_push([PREPARE_TESTS])
 # Usually invoked from ovn_start.
 ovn_init_db () {
     echo "creating $1 database"
-    local d=$ovs_base/$1
+    local as_d=$1
+    if test -n "$2"; then
+        as_d=$2/$as_d
+    fi
+    local d=$ovs_base/$as_d
     mkdir "$d" || return 1
     : > "$d"/.$1.db.~lock~
-    as $1 ovsdb-tool create "$d"/$1.db "$abs_top_srcdir"/$1.ovsschema
-    as $1 start_daemon ovsdb-server --remote=punix:"$d"/$1.sock "$d"/$1.db
+    as $as_d ovsdb-tool create "$d"/$1.db "$abs_top_srcdir"/$1.ovsschema
+    as $as_d start_daemon ovsdb-server --remote=punix:"$d"/$1.sock "$d"/$1.db
     local var=`echo $1_db | tr a-z- A-Z_`
-    AS_VAR_SET([$var], [unix:$ovs_base/$1/$1.sock]); export $var
+    AS_VAR_SET([$var], [unix:"$d"/$1.sock]); export $var
 }
 
-# ovn_start
+# ovn_init_ic_db
+#
+# Creates and initializes ovn-inb and ovn-isb databases and starts their
+# ovsdb-server instances, for OVN interconnection.
+ovn_init_ic_db () {
+    ovn_init_db ovn-inb
+    ovn_init_db ovn-isb
+}
+
+# ovn_start [AZ]
 #
 # Creates and initializes ovn-sb and ovn-nb databases and starts their
 # ovsdb-server instance, sets appropriate environment variables so that
 # ovn-sbctl and ovn-nbctl use them by default, and starts ovn-northd running
 # against them.
 ovn_start () {
-    ovn_init_db ovn-sb; ovn-sbctl init
-    ovn_init_db ovn-nb; ovn-nbctl init
+    if test -n "$1"; then
+        mkdir "$ovs_base"/$1
+    fi
 
+    ovn_init_db ovn-sb $1; ovn-sbctl init
+    ovn_init_db ovn-nb $1; ovn-nbctl init
+    if test -n "$1"; then
+        ovn-nbctl set NB_Global . name=$1
+    fi
+    local ovn_sb_db=$OVN_SB_DB
+    local ovn_nb_db=$OVN_NB_DB
+
+    local as_d=northd
+    if test -n "$1"; then
+        as_d=$1/$as_d
+    fi
     echo "starting ovn-northd"
-    mkdir "$ovs_base"/northd
-    as northd start_daemon ovn-northd -v \
-               --ovnnb-db=unix:"$ovs_base"/ovn-nb/ovn-nb.sock \
-               --ovnsb-db=unix:"$ovs_base"/ovn-sb/ovn-sb.sock
+    mkdir "$ovs_base"/$as_d
+    as $as_d start_daemon ovn-northd -v \
+               --ovnnb-db=$ovn_nb_db \
+               --ovnsb-db=$ovn_sb_db
 
+    as_d=northd-backup
+    if test -n "$1"; then
+        as_d=$1/$as_d
+    fi
     echo "starting backup ovn-northd"
-    mkdir "$ovs_base"/northd-backup
-    as northd-backup start_daemon ovn-northd -v \
-               --ovnnb-db=unix:"$ovs_base"/ovn-nb/ovn-nb.sock \
-               --ovnsb-db=unix:"$ovs_base"/ovn-sb/ovn-sb.sock
+    mkdir "$ovs_base"/$as_d
+    as $as_d start_daemon ovn-northd -v \
+               --ovnnb-db=$ovn_nb_db \
+               --ovnsb-db=$ovn_sb_db
+
+    if test -n "$1"; then
+        as_d=$1/ic
+        echo "starting ovn-ic"
+        mkdir "$ovs_base"/$as_d
+        as $as_d start_daemon ovn-ic -v \
+               --ovnnb-db=$ovn_nb_db \
+               --ovnsb-db=$ovn_sb_db \
+               --ovninb-db=unix:"$ovs_base"/ovn-inb/ovn-inb.sock \
+               --ovnisb-db=unix:"$ovs_base"/ovn-isb/ovn-isb.sock
+    fi
 }
 
 # Interconnection networks.
@@ -132,24 +213,25 @@  net_attach () {
         || return 1
 }
 
-# ovn_attach NETWORK BRIDGE IP [MASKLEN]
-#
-# First, this command attaches BRIDGE to interconnection network NETWORK, just
-# like "net_attach NETWORK BRIDGE".  Second, it configures (simulated) IP
-# address IP (with network mask length MASKLEN, which defaults to 24) on
-# BRIDGE.  Finally, it configures the Open vSwitch database to work with OVN
-# and starts ovn-controller.
-ovn_attach() {
-    local net=$1 bridge=$2 ip=$3 masklen=${4-24}
+# ovn_az_attach AZ NETWORK BRIDGE IP [MASKLEN]
+ovn_az_attach() {
+    local az=$1 net=$2 bridge=$3 ip=$4 masklen=${5-24}
     net_attach $net $bridge || return 1
 
     mac=`ovs-vsctl get Interface $bridge mac_in_use | sed s/\"//g`
     arp_table="$arp_table $sandbox,$bridge,$ip,$mac"
     ovs-appctl netdev-dummy/ip4addr $bridge $ip/$masklen >/dev/null || return 1
     ovs-appctl ovs/route/add $ip/$masklen $bridge >/dev/null || return 1
+
+    local ovn_remote
+    if test X"$az" = XNONE; then
+        ovn_remote=unix:$ovs_base/ovn-sb/ovn-sb.sock
+    else
+        ovn_remote=unix:$ovs_base/$az/ovn-sb/ovn-sb.sock
+    fi
     ovs-vsctl \
         -- set Open_vSwitch . external-ids:system-id=$sandbox \
-        -- set Open_vSwitch . external-ids:ovn-remote=unix:$ovs_base/ovn-sb/ovn-sb.sock \
+        -- set Open_vSwitch . external-ids:ovn-remote=$ovn_remote \
         -- set Open_vSwitch . external-ids:ovn-encap-type=geneve,vxlan \
         -- set Open_vSwitch . external-ids:ovn-encap-ip=$ip \
         -- add-br br-int \
@@ -158,6 +240,33 @@  ovn_attach() {
     start_daemon ovn-controller || return 1
 }
 
+# ovn_attach NETWORK BRIDGE IP [MASKLEN]
+#
+# First, this command attaches BRIDGE to interconnection network NETWORK, just
+# like "net_attach NETWORK BRIDGE".  Second, it configures (simulated) IP
+# address IP (with network mask length MASKLEN, which defaults to 24) on
+# BRIDGE.  Finally, it configures the Open vSwitch database to work with OVN
+# and starts ovn-controller.
+ovn_attach() {
+    ovn_az_attach NONE $@
+}
+
+# ovn_setenv AZ
+ovn_setenv () {
+    local d=$ovs_base/$1
+    AS_VAR_SET([OVN_NB_DB], [unix:"$d"/ovn-nb/ovn-nb.sock]); export $var
+    AS_VAR_SET([OVN_SB_DB], [unix:"$d"/ovn-sb/ovn-sb.sock]); export $var
+}
+
+# ovs_as AZ
+ovn_as() {
+    if test "X$2" != X; then
+        (ovn_setenv $1; shift; "$@")
+    else
+        ovn_setenv $1
+    fi
+}
+
 # OVN_POPULATE_ARP
 #
 # This pre-populates the ARP tables of all of the OVN instances that have been
diff --git a/tests/testsuite.at b/tests/testsuite.at
index ea6b04a..4ba7cdc 100644
--- a/tests/testsuite.at
+++ b/tests/testsuite.at
@@ -29,4 +29,5 @@  m4_include([tests/ovn-inbctl.at])
 m4_include([tests/ovn-isbctl.at])
 m4_include([tests/ovn-controller.at])
 m4_include([tests/ovn-controller-vtep.at])
+m4_include([tests/ovn-ic.at])
 m4_include([tests/checkpatch.at])
diff --git a/tutorial/ovs-sandbox b/tutorial/ovs-sandbox
index 510651a..2f17924 100755
--- a/tutorial/ovs-sandbox
+++ b/tutorial/ovs-sandbox
@@ -58,6 +58,8 @@  gdb_vswitchd_ex=false
 gdb_ovsdb_ex=false
 gdb_ovn_northd=false
 gdb_ovn_northd_ex=false
+gdb_ovn_ic=false
+gdb_ovn_ic_ex=false
 gdb_ovn_controller=false
 gdb_ovn_controller_ex=false
 gdb_ovn_controller_vtep=false
@@ -72,13 +74,20 @@  built=false
 ovn=true
 ovnsb_schema=
 ovnnb_schema=
+ovnisb_schema=
+ovninb_schema=
 ovn_rbac=true
 n_northds=1
+n_ics=1
 n_controllers=1
 nbdb_model=standalone
 nbdb_servers=3
 sbdb_model=backup
 sbdb_servers=3
+inbdb_model=clustered
+inbdb_servers=3
+isbdb_model=clustered
+isbdb_servers=3
 dummy=override
 
 for option; do
@@ -125,6 +134,7 @@  General options:
   -g, --gdb-vswitchd   run ovs-vswitchd under gdb
   -d, --gdb-ovsdb      run ovsdb-server under gdb
   --gdb-ovn-northd     run ovn-northd under gdb
+  --gdb-ovn-ic         run ovn-ic under gdb
   --gdb-ovn-controller run ovn-controller under gdb
   --gdb-ovn-controller-vtep run ovn-controller-vtep under gdb
   --dummy=ARG          pass --enable-dummy=ARG to vswitchd (default: override)
@@ -135,10 +145,15 @@  General options:
 OVN options:
   --no-ovn-rbac        disable role-based access control for OVN
   --n-northds=NUMBER   run NUMBER copies of northd (default: 1)
+  --n-ics=NUMBER       run NUMBER copies of ic (default: 1)
   --nbdb-model=standalone|backup|clustered    northbound database model
   --nbdb-servers=N     number of servers in nbdb cluster (default: 3)
   --sbdb-model=standalone|backup|clustered    southbound database model
   --sbdb-servers=N     number of servers in sbdb cluster (default: 3)
+  --inbdb-model=standalone|backup|clustered   ic-northbound database model
+  --inbdb-servers=N     number of servers in inbdb cluster (default: 3)
+  --isbdb-model=standalone|backup|clustered   ic-southbound database model
+  --isbdb-servers=N     number of servers in isbdb cluster (default: 3)
 
 Other options:
   -h, --help           Print this usage message.
@@ -210,6 +225,9 @@  EOF
         --gdb-ovn-northd)
             gdb_ovn_northd=true
             ;;
+        --gdb-ovn-ic)
+            gdb_ovn_ic=true
+            ;;
         --gdb-ovn-controller)
             gdb_ovn_controller=true
             ;;
@@ -225,6 +243,12 @@  EOF
         --n-northd*)
             prev=n_northds
             ;;
+        --n-ic*=*)
+            n_ics=$optarg
+            ;;
+        --n-ic*)
+            prev=n_ics_
+            ;;
         --n-controller*=*)
             n_controllers=$optarg
             ;;
@@ -259,10 +283,39 @@  EOF
         --sbdb-m*)
             prev=sbdb_model
             ;;
+        --inbdb-s*=*)
+            inbdb_servers=$optarg
+            inbdb_model=clustered
+            ;;
+        --inbdb-s*)
+            prev=inbdb_servers
+            inbdb_model=clustered
+            ;;
+        --inbdb-m*=*)
+            inbdb_model=$optarg
+            ;;
+        --inbdb-m*)
+            prev=inbdb_model
+            ;;
+        --isbdb-s*=*)
+            isbdb_servers=$optarg
+            isbdb_model=clustered
+            ;;
+        --isbdb-s*)
+            prev=isbdb_servers
+            isbdb_model=clustered
+            ;;
+        --isbdb-m*=*)
+            isbdb_model=$optarg
+            ;;
+        --isbdb-m*)
+            prev=isbdb_model
+            ;;
         -R|--gdb-run)
             gdb_vswitchd_ex=true
             gdb_ovsdb_ex=true
             gdb_ovn_northd_ex=true
+            gdb_ovn_ic_ex=true
             gdb_ovn_controller_ex=true
             gdb_ovn_controller_vtep_ex=true
             ;;
@@ -327,6 +380,16 @@  if $built; then
             echo >&2 'source directory not found, please use --srcdir'
             exit 1
         fi
+        ovnisb_schema=$srcdir/ovn-isb.ovsschema
+        if test ! -e "$ovnisb_schema"; then
+            echo >&2 'source directory not found, please use --srcdir'
+            exit 1
+        fi
+        ovninb_schema=$srcdir/ovn-inb.ovsschema
+        if test ! -e "$ovninb_schema"; then
+            echo >&2 'source directory not found, please use --srcdir'
+            exit 1
+        fi
         vtep_schema=$ovssrcdir/vtep/vtep.ovsschema
         if test ! -e "$vtep_schema"; then
             echo >&2 'source directory not found, please use --srcdir'
@@ -340,7 +403,7 @@  if $built; then
         exit 1
     fi
     PATH=$ovsbuilddir/ovsdb:$ovsbuilddir/vswitchd:$ovsbuilddir/utilities:$ovsbuilddir/vtep:$PATH
-    PATH=$builddir/controller:$builddir/controller-vtep:$builddir/northd:$builddir/utilities:$PATH
+    PATH=$builddir/controller:$builddir/controller-vtep:$builddir/northd:$builddir/ic:$builddir/utilities:$PATH
     export PATH
 else
     case $schema in
@@ -489,6 +552,8 @@  The backup database file is sandbox/${db}2.db
 backup_note=
 ovn_start_db nb "$nbdb_model" "$nbdb_servers" "$ovnnb_schema"
 ovn_start_db sb "$sbdb_model" "$sbdb_servers" "$ovnsb_schema"
+ovn_start_db inb "$inbdb_model" "$inbdb_servers" "$ovninb_schema"
+ovn_start_db isb "$isbdb_model" "$isbdb_servers" "$ovnisb_schema"
 
 #Add a small delay to allow ovsdb-server to launch.
 sleep 0.1
@@ -511,6 +576,9 @@  rungdb $gdb_vswitchd $gdb_vswitchd_ex ovs-vswitchd --detach --no-chdir --pidfile
 
 ovn-nbctl init
 ovn-sbctl init
+ovn-inbctl init
+ovn-isbctl init
+ovn-nbctl set NB_Global . name=az-1
 
 ovs-vsctl set open . external-ids:system-id=chassis-1
 ovs-vsctl set open . external-ids:hostname=sandbox
@@ -532,6 +600,14 @@  else
     ovs-vsctl set open . external-ids:ovn-remote=$OVN_SB_DB
     OVN_CTRLR_PKI=""
 fi
+for i in $(seq $n_ics); do
+    if [ $i -eq 1 ]; then inst=""; else inst=$i; fi
+    rungdb $gdb_ovn_ic $gdb_ovn_ic_ex ovn-ic --detach \
+            --no-chdir --pidfile=ovn-ic${inst}.pid -vconsole:off \
+            --log-file=ovn-ic${inst}.log -vsyslog:off \
+            --ovnsb-db="$OVN_SB_DB" --ovnnb-db="$OVN_NB_DB" \
+            --ovnisb-db="$OVN_ISB_DB" --ovninb-db="$OVN_INB_DB"
+done
 for i in $(seq $n_northds); do
     if [ $i -eq 1 ]; then inst=""; else inst=$i; fi
     rungdb $gdb_ovn_northd $gdb_ovn_northd_ex ovn-northd --detach \