[ovs-dev,2/7] ic: remove orphan ovn interconnection routes

Before this patch if one deletes transit switch through which there were
routes in ICSB:Route table, such routes were left forever in the DB.

Now we validate that each ICSB:Route has an appropriate transit switch.

Signed-off-by: Vladislav Odintsov <odivlad@gmail.com>
---
 ic/ovn-ic.c     | 40 +++++++++++++++++++++++++++
 tests/ovn-ic.at | 73 +++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 113 insertions(+)

On 12/2/22 18:31, Vladislav Odintsov wrote:
> Before this patch if one deletes transit switch through which there were
> routes in ICSB:Route table, such routes were left forever in the DB.
> 
> Now we validate that each ICSB:Route has an appropriate transit switch.
> 
> Signed-off-by: Vladislav Odintsov <odivlad@gmail.com>
> ---
>  ic/ovn-ic.c     | 40 +++++++++++++++++++++++++++
>  tests/ovn-ic.at | 73 +++++++++++++++++++++++++++++++++++++++++++++++++
>  2 files changed, 113 insertions(+)
> 
> diff --git a/ic/ovn-ic.c b/ic/ovn-ic.c
> index 50ff65a26..b3790e965 100644
> --- a/ic/ovn-ic.c
> +++ b/ic/ovn-ic.c
> @@ -71,6 +71,7 @@ struct ic_context {
>      struct ovsdb_idl_index *icsbrec_port_binding_by_az;
>      struct ovsdb_idl_index *icsbrec_port_binding_by_ts;
>      struct ovsdb_idl_index *icsbrec_port_binding_by_ts_az;
> +    struct ovsdb_idl_index *icsbrec_route_by_az;
>      struct ovsdb_idl_index *icsbrec_route_by_ts;
>      struct ovsdb_idl_index *icsbrec_route_by_ts_az;
>  };
> @@ -1621,6 +1622,38 @@ advertise_lr_routes(struct ic_context *ctx,
>      hmap_destroy(&routes_ad);
>  }
>  
> +static void
> +delete_orphan_ic_routes(struct ic_context *ctx,
> +                         const struct icsbrec_availability_zone *az)
> +{
> +    const struct icsbrec_route *isb_route, *isb_route_key =
> +        icsbrec_route_index_init_row(ctx->icsbrec_route_by_az);
> +    icsbrec_route_index_set_availability_zone(isb_route_key, az);
> +
> +    const struct icnbrec_transit_switch *t_sw, *t_sw_key;
> +
> +    ICSBREC_ROUTE_FOR_EACH_EQUAL (isb_route, isb_route_key,
> +                                  ctx->icsbrec_route_by_az)
> +    {
> +        t_sw_key = icnbrec_transit_switch_index_init_row(
> +            ctx->icnbrec_transit_switch_by_name);
> +        icnbrec_transit_switch_index_set_name(t_sw_key,
> +            isb_route->transit_switch);
> +        t_sw = icnbrec_transit_switch_index_find(
> +            ctx->icnbrec_transit_switch_by_name, t_sw_key);
> +        icnbrec_transit_switch_index_destroy_row(t_sw_key);
> +
> +        if (!t_sw) {
> +            VLOG_WARN("Deleting orphan ICDB:Route: %s->%s (%s, rtb:%s, "
> +                      "transit switch: %s)", isb_route->ip_prefix,
> +                      isb_route->nexthop, isb_route->origin,
> +                      isb_route->route_table, isb_route->transit_switch);

This seems like something that can happen under normal operation (e.g.,
a zone going away).  I don't think we should WARN.  Maybe VLOG_INFO_RL
is more appropriate?  What do you think?

Thanks,
Dumitru

> +            icsbrec_route_delete(isb_route);
> +        }
> +    }
> +    icsbrec_route_index_destroy_row(isb_route_key);
> +}
> +
>  static void
>  route_run(struct ic_context *ctx,
>            const struct icsbrec_availability_zone *az)
> @@ -1629,6 +1662,8 @@ route_run(struct ic_context *ctx,
>          return;
>      }
>  
> +    delete_orphan_ic_routes(ctx, az);
> +
>      struct hmap ic_lrs = HMAP_INITIALIZER(&ic_lrs);
>      const struct icsbrec_port_binding *isb_pb;
>      const struct icsbrec_port_binding *isb_pb_key =
> @@ -1917,6 +1952,10 @@ main(int argc, char *argv[])
>                                    &icsbrec_port_binding_col_transit_switch,
>                                    &icsbrec_port_binding_col_availability_zone);
>  
> +    struct ovsdb_idl_index *icsbrec_route_by_az
> +        = ovsdb_idl_index_create1(ovnisb_idl_loop.idl,
> +                                  &icsbrec_route_col_availability_zone);
> +
>      struct ovsdb_idl_index *icsbrec_route_by_ts
>          = ovsdb_idl_index_create1(ovnisb_idl_loop.idl,
>                                    &icsbrec_route_col_transit_switch);
> @@ -1971,6 +2010,7 @@ main(int argc, char *argv[])
>                  .icsbrec_port_binding_by_az = icsbrec_port_binding_by_az,
>                  .icsbrec_port_binding_by_ts = icsbrec_port_binding_by_ts,
>                  .icsbrec_port_binding_by_ts_az = icsbrec_port_binding_by_ts_az,
> +                .icsbrec_route_by_az = icsbrec_route_by_az,
>                  .icsbrec_route_by_ts = icsbrec_route_by_ts,
>                  .icsbrec_route_by_ts_az = icsbrec_route_by_ts_az,
>              };
> diff --git a/tests/ovn-ic.at b/tests/ovn-ic.at
> index 0bdfc55e6..e234b7fb9 100644
> --- a/tests/ovn-ic.at
> +++ b/tests/ovn-ic.at
> @@ -121,6 +121,79 @@ OVN_CLEANUP_IC
>  AT_CLEANUP
>  ])
>  
> +OVN_FOR_EACH_NORTHD([
> +AT_SETUP([ovn-ic -- route deletion upon TS deletion])
> +
> +ovn_init_ic_db
> +net_add n1
> +
> +# 1 GW per AZ
> +for i in 1 2; do
> +    az=az$i
> +    ovn_start $az
> +    sim_add gw-$az
> +    as gw-$az
> +    check ovs-vsctl add-br br-phys
> +    ovn_az_attach $az n1 br-phys 192.168.1.$i
> +    check ovs-vsctl set open . external-ids:ovn-is-interconn=true
> +    check ovn-nbctl set nb-global . \
> +        options:ic-route-adv=true \
> +        options:ic-route-adv-default=true \
> +        options:ic-route-learn=true \
> +        options:ic-route-learn-default=true
> +done
> +
> +create_ic_infra() {
> +    az_id=$1
> +    ts_id=$2
> +    az=az$i
> +
> +    lsp=lsp${az_id}-${ts_id}
> +    lrp=lrp${az_id}-${ts_id}
> +    ts=ts${az_id}-${ts_id}
> +    lr=lr${az_id}-${ts_id}
> +
> +    ovn_as $az
> +
> +    check ovn-ic-nbctl ts-add $ts
> +    check ovn-nbctl lr-add $lr
> +    check ovn-nbctl lrp-add $lr $lrp 00:00:00:00:00:0$az_id 10.0.$az_id.1/24
> +    check ovn-nbctl lrp-set-gateway-chassis $lrp gw-$az
> +
> +    check ovn-nbctl lsp-add $ts $lsp -- \
> +        lsp-set-addresses $lsp router -- \
> +        lsp-set-type $lsp router -- \
> +        lsp-set-options $lsp router-port=$lrp
> +
> +    check ovn-nbctl lr-route-add $lr 192.168.0.0/16 10.0.$az_id.10
> +}
> +
> +create_ic_infra 1 1
> +create_ic_infra 1 2
> +create_ic_infra 2 1
> +
> +ovn_as az1
> +
> +wait_row_count ic-sb:Route 3 ip_prefix=192.168.0.0/16
> +
> +# remove transit switch 1 (from az1) and check if its route is deleted
> +# same route from another AZ and ts should remain, as
> +check ovn-ic-nbctl ts-del ts1-1
> +sleep 2
> +ovn-ic-sbctl list route
> +ovn-ic-nbctl list transit_switch
> +wait_row_count ic-sb:route 2 ip_prefix=192.168.0.0/16
> +ovn-ic-sbctl list route
> +
> +for i in 1 2; do
> +    az=az$i
> +    OVN_CLEANUP_SBOX(gw-$az)
> +    OVN_CLEANUP_AZ([$az])
> +done
> +OVN_CLEANUP_IC
> +AT_CLEANUP
> +])
> +
>  OVN_FOR_EACH_NORTHD([
>  AT_SETUP([ovn-ic -- gateway sync])
>

Hi Dumitru,

please, see answer inline.

Regards,
Vladislav Odintsov

> On 5 Dec 2022, at 19:37, Dumitru Ceara <dceara@redhat.com> wrote:
> 
> On 12/2/22 18:31, Vladislav Odintsov wrote:
>> Before this patch if one deletes transit switch through which there were
>> routes in ICSB:Route table, such routes were left forever in the DB.
>> 
>> Now we validate that each ICSB:Route has an appropriate transit switch.
>> 
>> Signed-off-by: Vladislav Odintsov <odivlad@gmail.com>
>> ---
>> ic/ovn-ic.c     | 40 +++++++++++++++++++++++++++
>> tests/ovn-ic.at | 73 +++++++++++++++++++++++++++++++++++++++++++++++++
>> 2 files changed, 113 insertions(+)
>> 
>> diff --git a/ic/ovn-ic.c b/ic/ovn-ic.c
>> index 50ff65a26..b3790e965 100644
>> --- a/ic/ovn-ic.c
>> +++ b/ic/ovn-ic.c
>> @@ -71,6 +71,7 @@ struct ic_context {
>>     struct ovsdb_idl_index *icsbrec_port_binding_by_az;
>>     struct ovsdb_idl_index *icsbrec_port_binding_by_ts;
>>     struct ovsdb_idl_index *icsbrec_port_binding_by_ts_az;
>> +    struct ovsdb_idl_index *icsbrec_route_by_az;
>>     struct ovsdb_idl_index *icsbrec_route_by_ts;
>>     struct ovsdb_idl_index *icsbrec_route_by_ts_az;
>> };
>> @@ -1621,6 +1622,38 @@ advertise_lr_routes(struct ic_context *ctx,
>>     hmap_destroy(&routes_ad);
>> }
>> 
>> +static void
>> +delete_orphan_ic_routes(struct ic_context *ctx,
>> +                         const struct icsbrec_availability_zone *az)
>> +{
>> +    const struct icsbrec_route *isb_route, *isb_route_key =
>> +        icsbrec_route_index_init_row(ctx->icsbrec_route_by_az);
>> +    icsbrec_route_index_set_availability_zone(isb_route_key, az);
>> +
>> +    const struct icnbrec_transit_switch *t_sw, *t_sw_key;
>> +
>> +    ICSBREC_ROUTE_FOR_EACH_EQUAL (isb_route, isb_route_key,
>> +                                  ctx->icsbrec_route_by_az)
>> +    {
>> +        t_sw_key = icnbrec_transit_switch_index_init_row(
>> +            ctx->icnbrec_transit_switch_by_name);
>> +        icnbrec_transit_switch_index_set_name(t_sw_key,
>> +            isb_route->transit_switch);
>> +        t_sw = icnbrec_transit_switch_index_find(
>> +            ctx->icnbrec_transit_switch_by_name, t_sw_key);
>> +        icnbrec_transit_switch_index_destroy_row(t_sw_key);
>> +
>> +        if (!t_sw) {
>> +            VLOG_WARN("Deleting orphan ICDB:Route: %s->%s (%s, rtb:%s, "
>> +                      "transit switch: %s)", isb_route->ip_prefix,
>> +                      isb_route->nexthop, isb_route->origin,
>> +                      isb_route->route_table, isb_route->transit_switch);
> 
> This seems like something that can happen under normal operation (e.g.,
> a zone going away).  I don't think we should WARN.  Maybe VLOG_INFO_RL
> is more appropriate?  What do you think?

No, the zone going away is not covered here. There is a for..each loop here, which iterates over routes from local az (index is used).
I don’t think that Availability Zone is a common scenario, but if user de-register Availability Zone, its port bindings should be removed as well as routes, gateways and encaps.

Regarding loglevel I’m agree with you — it’s a good notice, thanks. I’ll address this in v2.

> 
> Thanks,
> Dumitru
> 
>> +            icsbrec_route_delete(isb_route);
>> +        }
>> +    }
>> +    icsbrec_route_index_destroy_row(isb_route_key);
>> +}
>> +
>> static void
>> route_run(struct ic_context *ctx,
>>           const struct icsbrec_availability_zone *az)
>> @@ -1629,6 +1662,8 @@ route_run(struct ic_context *ctx,
>>         return;
>>     }
>> 
>> +    delete_orphan_ic_routes(ctx, az);
>> +
>>     struct hmap ic_lrs = HMAP_INITIALIZER(&ic_lrs);
>>     const struct icsbrec_port_binding *isb_pb;
>>     const struct icsbrec_port_binding *isb_pb_key =
>> @@ -1917,6 +1952,10 @@ main(int argc, char *argv[])
>>                                   &icsbrec_port_binding_col_transit_switch,
>>                                   &icsbrec_port_binding_col_availability_zone);
>> 
>> +    struct ovsdb_idl_index *icsbrec_route_by_az
>> +        = ovsdb_idl_index_create1(ovnisb_idl_loop.idl,
>> +                                  &icsbrec_route_col_availability_zone);
>> +
>>     struct ovsdb_idl_index *icsbrec_route_by_ts
>>         = ovsdb_idl_index_create1(ovnisb_idl_loop.idl,
>>                                   &icsbrec_route_col_transit_switch);
>> @@ -1971,6 +2010,7 @@ main(int argc, char *argv[])
>>                 .icsbrec_port_binding_by_az = icsbrec_port_binding_by_az,
>>                 .icsbrec_port_binding_by_ts = icsbrec_port_binding_by_ts,
>>                 .icsbrec_port_binding_by_ts_az = icsbrec_port_binding_by_ts_az,
>> +                .icsbrec_route_by_az = icsbrec_route_by_az,
>>                 .icsbrec_route_by_ts = icsbrec_route_by_ts,
>>                 .icsbrec_route_by_ts_az = icsbrec_route_by_ts_az,
>>             };
>> diff --git a/tests/ovn-ic.at b/tests/ovn-ic.at
>> index 0bdfc55e6..e234b7fb9 100644
>> --- a/tests/ovn-ic.at
>> +++ b/tests/ovn-ic.at
>> @@ -121,6 +121,79 @@ OVN_CLEANUP_IC
>> AT_CLEANUP
>> ])
>> 
>> +OVN_FOR_EACH_NORTHD([
>> +AT_SETUP([ovn-ic -- route deletion upon TS deletion])
>> +
>> +ovn_init_ic_db
>> +net_add n1
>> +
>> +# 1 GW per AZ
>> +for i in 1 2; do
>> +    az=az$i
>> +    ovn_start $az
>> +    sim_add gw-$az
>> +    as gw-$az
>> +    check ovs-vsctl add-br br-phys
>> +    ovn_az_attach $az n1 br-phys 192.168.1.$i
>> +    check ovs-vsctl set open . external-ids:ovn-is-interconn=true
>> +    check ovn-nbctl set nb-global . \
>> +        options:ic-route-adv=true \
>> +        options:ic-route-adv-default=true \
>> +        options:ic-route-learn=true \
>> +        options:ic-route-learn-default=true
>> +done
>> +
>> +create_ic_infra() {
>> +    az_id=$1
>> +    ts_id=$2
>> +    az=az$i
>> +
>> +    lsp=lsp${az_id}-${ts_id}
>> +    lrp=lrp${az_id}-${ts_id}
>> +    ts=ts${az_id}-${ts_id}
>> +    lr=lr${az_id}-${ts_id}
>> +
>> +    ovn_as $az
>> +
>> +    check ovn-ic-nbctl ts-add $ts
>> +    check ovn-nbctl lr-add $lr
>> +    check ovn-nbctl lrp-add $lr $lrp 00:00:00:00:00:0$az_id 10.0.$az_id.1/24
>> +    check ovn-nbctl lrp-set-gateway-chassis $lrp gw-$az
>> +
>> +    check ovn-nbctl lsp-add $ts $lsp -- \
>> +        lsp-set-addresses $lsp router -- \
>> +        lsp-set-type $lsp router -- \
>> +        lsp-set-options $lsp router-port=$lrp
>> +
>> +    check ovn-nbctl lr-route-add $lr 192.168.0.0/16 10.0.$az_id.10
>> +}
>> +
>> +create_ic_infra 1 1
>> +create_ic_infra 1 2
>> +create_ic_infra 2 1
>> +
>> +ovn_as az1
>> +
>> +wait_row_count ic-sb:Route 3 ip_prefix=192.168.0.0/16
>> +
>> +# remove transit switch 1 (from az1) and check if its route is deleted
>> +# same route from another AZ and ts should remain, as
>> +check ovn-ic-nbctl ts-del ts1-1
>> +sleep 2
>> +ovn-ic-sbctl list route
>> +ovn-ic-nbctl list transit_switch
>> +wait_row_count ic-sb:route 2 ip_prefix=192.168.0.0/16
>> +ovn-ic-sbctl list route
>> +
>> +for i in 1 2; do
>> +    az=az$i
>> +    OVN_CLEANUP_SBOX(gw-$az)
>> +    OVN_CLEANUP_AZ([$az])
>> +done
>> +OVN_CLEANUP_IC
>> +AT_CLEANUP
>> +])
>> +
>> OVN_FOR_EACH_NORTHD([
>> AT_SETUP([ovn-ic -- gateway sync])
>> 
>