@@ -147,11 +147,118 @@ az_run(struct ic_context *ctx)
return NULL;
}
+static uint32_t
+allocate_ts_dp_key(struct hmap *dp_tnlids)
+{
+ static uint32_t hint = OVN_MIN_DP_KEY_GLOBAL;
+ return ovn_allocate_tnlid(dp_tnlids, "transit switch datapath",
+ OVN_MIN_DP_KEY_GLOBAL, OVN_MAX_DP_KEY_GLOBAL,
+ &hint);
+}
+
+static void
+ts_run(struct ic_context *ctx)
+{
+ const struct icnbrec_transit_switch *ts;
+
+ struct hmap dp_tnlids = HMAP_INITIALIZER(&dp_tnlids);
+ struct shash isb_dps = SHASH_INITIALIZER(&isb_dps);
+ const struct icsbrec_datapath_binding *isb_dp;
+ ICSBREC_DATAPATH_BINDING_FOR_EACH (isb_dp, ctx->ovnisb_idl) {
+ shash_add(&isb_dps, isb_dp->transit_switch, isb_dp);
+ ovn_add_tnlid(&dp_tnlids, isb_dp->tunnel_key);
+ }
+
+ /* Sync INB TS to AZ NB */
+ if (ctx->ovnnb_txn) {
+ struct shash nb_tses = SHASH_INITIALIZER(&nb_tses);
+ const struct nbrec_logical_switch *ls;
+
+ /* Get current NB Logical_Switch with other_config:interconn-ts */
+ NBREC_LOGICAL_SWITCH_FOR_EACH (ls, ctx->ovnnb_idl) {
+ const char *ts_name = smap_get(&ls->other_config, "interconn-ts");
+ if (ts_name) {
+ shash_add(&nb_tses, ts_name, ls);
+ }
+ }
+
+ /* Create NB Logical_Switch for each TS */
+ ICNBREC_TRANSIT_SWITCH_FOR_EACH (ts, ctx->ovninb_idl) {
+ ls = shash_find_and_delete(&nb_tses, ts->name);
+ if (!ls) {
+ ls = nbrec_logical_switch_insert(ctx->ovnnb_txn);
+ nbrec_logical_switch_set_name(ls, ts->name);
+ nbrec_logical_switch_update_other_config_setkey(ls,
+ "interconn-ts",
+ ts->name);
+ }
+ isb_dp = shash_find_data(&isb_dps, ts->name);
+ if (isb_dp) {
+ int64_t nb_tnl_key = smap_get_int(&ls->other_config,
+ "requested-tnl-key",
+ 0);
+ if (nb_tnl_key != isb_dp->tunnel_key) {
+ VLOG_DBG("Set other_config:requested-tnl-key %"PRId64
+ " for transit switch %s in NB.",
+ isb_dp->tunnel_key, ts->name);
+ char *tnl_key_str = xasprintf("%"PRId64,
+ isb_dp->tunnel_key);
+ nbrec_logical_switch_update_other_config_setkey(
+ ls, "requested-tnl-key", tnl_key_str);
+ free(tnl_key_str);
+ }
+ }
+ }
+
+ /* Delete extra NB Logical_Switch with other_config:interconn-ts */
+ struct shash_node *node;
+ SHASH_FOR_EACH (node, &nb_tses) {
+ nbrec_logical_switch_delete(node->data);
+ }
+ shash_destroy(&nb_tses);
+ }
+
+ /* Sync TS between INB and ISB. This is performed after syncing with AZ
+ * SB, to avoid uncommitted ISB datapath tunnel key to be synced back to
+ * AZ. */
+ if (ctx->ovnisb_txn) {
+ /* Create ISB Datapath_Binding */
+ ICNBREC_TRANSIT_SWITCH_FOR_EACH (ts, ctx->ovninb_idl) {
+ isb_dp = shash_find_and_delete(&isb_dps, ts->name);
+ if (!isb_dp) {
+ /* Allocate tunnel key */
+ int64_t dp_key = allocate_ts_dp_key(&dp_tnlids);
+ if (!dp_key) {
+ continue;
+ }
+
+ isb_dp = icsbrec_datapath_binding_insert(ctx->ovnisb_txn);
+ icsbrec_datapath_binding_set_transit_switch(isb_dp, ts->name);
+ icsbrec_datapath_binding_set_tunnel_key(isb_dp, dp_key);
+ }
+ }
+
+ /* Delete extra ISB Datapath_Binding */
+ struct shash_node *node;
+ SHASH_FOR_EACH (node, &isb_dps) {
+ icsbrec_datapath_binding_delete(node->data);
+ }
+ }
+ ovn_destroy_tnlids(&dp_tnlids);
+ shash_destroy(&isb_dps);
+}
+
static void
ovn_db_run(struct ic_context *ctx)
{
const struct icsbrec_availability_zone *az = az_run(ctx);
VLOG_DBG("Availability zone: %s", az ? az->name : "not created yet.");
+
+ if (!az) {
+ return;
+ }
+
+ ts_run(ctx);
}
static void
@@ -473,8 +473,8 @@ ovn_add_tnlid(struct hmap *set, uint32_t tnlid)
node->tnlid = tnlid;
}
-static bool
-tnlid_in_use(const struct hmap *set, uint32_t tnlid)
+bool
+ovn_tnlid_in_use(const struct hmap *set, uint32_t tnlid)
{
const struct tnlid_node *node;
HMAP_FOR_EACH_IN_BUCKET (node, hmap_node, hash_int(tnlid, 0), set) {
@@ -497,7 +497,7 @@ ovn_allocate_tnlid(struct hmap *set, const char *name, uint32_t min,
{
for (uint32_t tnlid = next_tnlid(*hint, min, max); tnlid != *hint;
tnlid = next_tnlid(tnlid, min, max)) {
- if (!tnlid_in_use(set, tnlid)) {
+ if (!ovn_tnlid_in_use(set, tnlid)) {
ovn_add_tnlid(set, tnlid);
*hint = tnlid;
return tnlid;
@@ -98,6 +98,7 @@ uint32_t ovn_logical_flow_hash(const struct uuid *logical_datapath,
struct hmap;
void ovn_destroy_tnlids(struct hmap *tnlids);
void ovn_add_tnlid(struct hmap *set, uint32_t tnlid);
+bool ovn_tnlid_in_use(const struct hmap *set, uint32_t tnlid);
uint32_t ovn_allocate_tnlid(struct hmap *set, const char *name, uint32_t min,
uint32_t max, uint32_t *hint);
#endif
@@ -894,6 +894,14 @@ ovn_datapath_update_external_ids(struct ovn_datapath *od)
if (name2 && name2[0]) {
smap_add(&ids, "name2", name2);
}
+
+ /* Set interconn-ts. */
+ if (od->nbs) {
+ const char *ts = smap_get(&od->nbs->other_config, "interconn-ts");
+ if (ts) {
+ smap_add(&ids, "interconn-ts", ts);
+ }
+ }
sbrec_datapath_binding_set_external_ids(od->sb, &ids);
smap_destroy(&ids);
}
@@ -1009,30 +1017,68 @@ build_datapaths(struct northd_context *ctx, struct hmap *datapaths,
join_datapaths(ctx, datapaths, &sb_only, &nb_only, &both, lr_list);
- if (!ovs_list_is_empty(&nb_only)) {
- /* First index the in-use datapath tunnel IDs. */
- struct hmap dp_tnlids = HMAP_INITIALIZER(&dp_tnlids);
- struct ovn_datapath *od;
+ /* First index the in-use datapath tunnel IDs. */
+ struct hmap dp_tnlids = HMAP_INITIALIZER(&dp_tnlids);
+ struct ovn_datapath *od, *next;
+ if (!ovs_list_is_empty(&nb_only) || !ovs_list_is_empty(&both)) {
LIST_FOR_EACH (od, list, &both) {
ovn_add_tnlid(&dp_tnlids, od->sb->tunnel_key);
}
+ }
- /* Add southbound record for each unmatched northbound record. */
- LIST_FOR_EACH (od, list, &nb_only) {
- uint32_t tunnel_key = ovn_datapath_allocate_key(&dp_tnlids);
+ /* Add southbound record for each unmatched northbound record. */
+ LIST_FOR_EACH (od, list, &nb_only) {
+ int64_t tunnel_key = 0;
+ if (od->nbs) {
+ tunnel_key = smap_get_int(&od->nbs->other_config,
+ "requested-tnl-key",
+ 0);
+ if (tunnel_key && ovn_tnlid_in_use(&dp_tnlids, tunnel_key)) {
+ static struct vlog_rate_limit rl =
+ VLOG_RATE_LIMIT_INIT(1, 1);
+ VLOG_WARN_RL(&rl, "Cannot create datapath binding for "
+ "logical switch %s due to duplicate key set "
+ "in other_config:requested-tnl-key: %"PRId64,
+ od->nbs->name, tunnel_key);
+ continue;
+ }
+ }
+ if (!tunnel_key) {
+ tunnel_key = ovn_datapath_allocate_key(&dp_tnlids);
if (!tunnel_key) {
break;
}
+ }
- od->sb = sbrec_datapath_binding_insert(ctx->ovnsb_txn);
- ovn_datapath_update_external_ids(od);
- sbrec_datapath_binding_set_tunnel_key(od->sb, tunnel_key);
+ od->sb = sbrec_datapath_binding_insert(ctx->ovnsb_txn);
+ ovn_datapath_update_external_ids(od);
+ sbrec_datapath_binding_set_tunnel_key(od->sb, tunnel_key);
+ }
+
+ /* Sync from northbound to southbound record for od existed in both. */
+ LIST_FOR_EACH (od, list, &both) {
+ if (od->nbs) {
+ int64_t tunnel_key = smap_get_int(&od->nbs->other_config,
+ "requested-tnl-key",
+ 0);
+ if (tunnel_key && tunnel_key != od->sb->tunnel_key) {
+ if (ovn_tnlid_in_use(&dp_tnlids, tunnel_key)) {
+ static struct vlog_rate_limit rl =
+ VLOG_RATE_LIMIT_INIT(1, 1);
+ VLOG_WARN_RL(&rl, "Cannot update datapath binding key for "
+ "logical switch %s due to duplicate key set "
+ "in other_config:requested-tnl-key: %"PRId64,
+ od->nbs->name, tunnel_key);
+ continue;
+ }
+ sbrec_datapath_binding_set_tunnel_key(od->sb, tunnel_key);
+ }
}
- ovn_destroy_tnlids(&dp_tnlids);
}
+ ovn_destroy_tnlids(&dp_tnlids);
+
/* Delete southbound records without northbound matches. */
- struct ovn_datapath *od, *next;
LIST_FOR_EACH_SAFE (od, next, list, &sb_only) {
ovs_list_remove(&od->list);
sbrec_datapath_binding_delete(od->sb);
@@ -359,6 +359,30 @@
</column>
</group>
+ <group title="Interconnection">
+ <column name="other_config" key="interconn-ts"
+ type='{"type": "string"}'>
+ The <ref table="Transit_Switch" column="name" db="OVN_IC_Northbound"/>
+ of corresponding transit switch in <ref db="OVN_IC_Northbound"/>
+ database. This kind of logical switch is created and controlled
+ by <code>ovn-ic</code>.
+ </column>
+ </group>
+
+ <group title="Tunnel Key">
+ <column name="other_config" key="requested-tnl-key"
+ type='{"type": "integer", "minInteger": 1, "maxInteger": 16777215}'>
+ Configures the datapath tunnel key for the logical switch. Usually
+ this is not needed because <code>ovn-northd</code> will assign an
+ unique key for each datapath by itself. However, if it is configured,
+ <code>ovn-northd</code> honors the configured value. The typical use
+ case is for interconnection: the tunnel keys for transit switches need
+ to be unique globally, so they are maintained in the global
+ <ref db="OVN_IC_Southbound"/> database, and <code>ovn-ic</code> simply
+ syncs the value from <ref db="OVN_IC_Southbound"/> through this config.
+ </column>
+ </group>
+
<group title="Common Columns">
<column name="external_ids">
See <em>External IDs</em> at the beginning of this document.
@@ -2361,6 +2361,15 @@ tcp.flags = RST;
the <ref db="OVN_Northbound"/> database.
</column>
+ <column name="external_ids" key="interconn-ts" type='{"type": "string"}'>
+ For a logical datapath that represents a logical switch that represents
+ a transit switch for interconnection, <code>ovn-northd</code> stores in
+ this key the value of the same <code>interconn-ts</code> key of the <ref
+ column="external_ids" table="Logical_Switch" db="OVN_Northbound"/>
+ column of the corresponding <ref table="Logical_Switch"
+ db="OVN_Northbound"/> row in the <ref db="OVN_Northbound"/> database.
+ </column>
+
<group title="Naming">
<p>
<code>ovn-northd</code> copies these from the name fields in the <ref
@@ -26,3 +26,43 @@ availability-zone az3
OVN_CLEANUP_IC([az1], [az2])
AT_CLEANUP
+
+AT_SETUP([ovn-ic -- transit switch handling])
+
+ovn_init_ic_db
+ovn_start az1
+
+AT_CHECK([ovn-ic-nbctl ts-add ts1])
+AT_CHECK([ovn-ic-nbctl ts-add ts2])
+
+# Check ISB
+OVS_WAIT_UNTIL([ovn-ic-sbctl list datapath | grep ts2])
+AT_CHECK([ovn-ic-sbctl -f csv -d bare --no-headings --columns transit_switch list datapath | sort], [0], [dnl
+ts1
+ts2
+])
+
+# Check NB
+AT_CHECK([ovn-nbctl -f csv -d bare --no-headings --columns name list logical_switch | sort], [0], [dnl
+ts1
+ts2
+])
+
+# Check SB DP key
+ts1_key=$(ovn-ic-sbctl -f csv -d bare --no-headings --columns tunnel_key find datapath transit_switch=ts1)
+sb_ts1_key=$(ovn-sbctl -f csv -d bare --no-headings --columns tunnel_key find datapath_binding external_ids:interconn-ts=ts1)
+AT_CHECK([test $ts1_key = $sb_ts1_key])
+
+# Test delete
+AT_CHECK([ovn-ic-nbctl ts-del ts1])
+OVS_WAIT_WHILE([ovn-ic-sbctl list datapath | grep ts1])
+AT_CHECK([ovn-ic-sbctl -f csv -d bare --no-headings --columns transit_switch list datapath], [0], [dnl
+ts2
+])
+AT_CHECK([ovn-nbctl -f csv -d bare --no-headings --columns name list logical_switch | sort], [0], [dnl
+ts2
+])
+
+OVN_CLEANUP_IC([az1])
+
+AT_CLEANUP
Processing transit switches and sync between IC-NB, IC-SB and NB. Signed-off-by: Han Zhou <hzhou@ovn.org> --- ic/ovn-ic.c | 107 ++++++++++++++++++++++++++++++++++++++++++++++++++++ lib/ovn-util.c | 6 +-- lib/ovn-util.h | 1 + northd/ovn-northd.c | 70 ++++++++++++++++++++++++++++------ ovn-nb.xml | 24 ++++++++++++ ovn-sb.xml | 9 +++++ tests/ovn-ic.at | 40 ++++++++++++++++++++ 7 files changed, 242 insertions(+), 15 deletions(-)