[ovs-dev,54/62] Add QoS function offload support by meter

From: Taoyunxiang <taoyunxiang@cmss.chinamobile.com>

Code Source From: Self Code
Description:
This is the init version to support QoS function offload,
which is handled by meter aciton. So this change offload meter
action from openflow.
Jira:  #[Optional]
市场项目编号（名称）：[Optional]

Add QoS function offload support by meter.
---
 include/openvswitch/ofp-meter.h |   1 +
 lib/dpif-netdev.c               | 162 +++++++++++++++++++++++---
 lib/dpif-netdev.h               |  19 +++
 lib/dpif-netlink.c              |   5 +-
 lib/dpif-provider.h             |   5 +
 lib/dpif.c                      |  66 +++++++++++
 lib/dpif.h                      |   6 +
 lib/netdev-dpdk.c               |   6 +
 lib/netdev-dpdk.h               |   1 +
 lib/netdev-dummy.c              |   8 +-
 lib/netdev-offload-dpdk.c       | 250 +++++++++++++++++++++++++++++++++++++---
 lib/netdev-offload-provider.h   |  17 ++-
 lib/netdev-offload-tc.c         |   7 +-
 lib/netdev-offload.c            |  11 +-
 lib/netdev-offload.h            |   8 +-
 lib/ofp-meter.c                 |  14 +++
 16 files changed, 535 insertions(+), 51 deletions(-)

On Tue, Dec 29, 2020 at 2:13 AM Tao YunXiang
<taoyunxiang@cmss.chinamobile.com> wrote:
>
> From: Taoyunxiang <taoyunxiang@cmss.chinamobile.com>
>
> Code Source From: Self Code
> Description:
> This is the init version to support QoS function offload,
> which is handled by meter aciton. So this change offload meter
> action from openflow.
> Jira:  #[Optional]
> 市场项目编号（名称）：[Optional]
>
> Add QoS function offload support by meter.
Hi
this version is same as what I posted
http://patchwork.ozlabs.org/project/openvswitch/list/?series=150532
> ---
>  include/openvswitch/ofp-meter.h |   1 +
>  lib/dpif-netdev.c               | 162 +++++++++++++++++++++++---
>  lib/dpif-netdev.h               |  19 +++
>  lib/dpif-netlink.c              |   5 +-
>  lib/dpif-provider.h             |   5 +
>  lib/dpif.c                      |  66 +++++++++++
>  lib/dpif.h                      |   6 +
>  lib/netdev-dpdk.c               |   6 +
>  lib/netdev-dpdk.h               |   1 +
>  lib/netdev-dummy.c              |   8 +-
>  lib/netdev-offload-dpdk.c       | 250 +++++++++++++++++++++++++++++++++++++---
>  lib/netdev-offload-provider.h   |  17 ++-
>  lib/netdev-offload-tc.c         |   7 +-
>  lib/netdev-offload.c            |  11 +-
>  lib/netdev-offload.h            |   8 +-
>  lib/ofp-meter.c                 |  14 +++
>  16 files changed, 535 insertions(+), 51 deletions(-)
>
> diff --git a/include/openvswitch/ofp-meter.h b/include/openvswitch/ofp-meter.h
> index 6776eae..c6678f5 100644
> --- a/include/openvswitch/ofp-meter.h
> +++ b/include/openvswitch/ofp-meter.h
> @@ -62,6 +62,7 @@ int ofputil_decode_meter_config(struct ofpbuf *,
>                                  struct ofpbuf *bands);
>  void ofputil_format_meter_config(struct ds *,
>                                   const struct ofputil_meter_config *);
> +uint32_t ofputil_meter_config_max_rate(struct ofputil_meter_config *conf);
>
>  struct ofputil_meter_mod {
>      uint16_t command;
> diff --git a/lib/dpif-netdev.c b/lib/dpif-netdev.c
> index a40441f..b03a2ab 100644
> --- a/lib/dpif-netdev.c
> +++ b/lib/dpif-netdev.c
> @@ -272,23 +272,6 @@ static bool dpcls_lookup(struct dpcls *cls,
>  #define DP_SUPPORTED_METER_BAND_TYPES           \
>      ( 1 << OFPMBT13_DROP )
>
> -struct dp_meter_band {
> -    struct ofputil_meter_band up; /* type, prec_level, pad, rate, burst_size */
> -    uint32_t bucket; /* In 1/1000 packets (for PKTPS), or in bits (for KBPS) */
> -    uint64_t packet_count;
> -    uint64_t byte_count;
> -};
> -
> -struct dp_meter {
> -    uint16_t flags;
> -    uint16_t n_bands;
> -    uint32_t max_delta_t;
> -    uint64_t used;
> -    uint64_t packet_count;
> -    uint64_t byte_count;
> -    struct dp_meter_band bands[];
> -};
> -
>  struct pmd_auto_lb {
>      bool auto_lb_requested;     /* Auto load balancing requested by user. */
>      bool is_enabled;            /* Current status of Auto load balancing. */
> @@ -424,6 +407,7 @@ struct dp_flow_offload_item {
>      ovs_be32 ori_nw_dst;
>
>      struct ovs_list node;
> +    struct dpif *dpif;
>  };
>
>  struct dp_flow_offload {
> @@ -1640,6 +1624,31 @@ dpif_netdev_open(const struct dpif_class *class, const char *name,
>      return error;
>  }
>
> +static struct dpif *
> +dpif_netdev_dump_create(struct dp_netdev *dp)
> +{
> +    struct dpif *dpif = NULL;
> +
> +    ovs_mutex_lock(&dp_netdev_mutex);
> +    dpif = create_dpif_netdev(dp);
> +    ovs_mutex_unlock(&dp_netdev_mutex);
> +
> +    return dpif;
> +}
> +
> +static void
> +dpif_netdev_dump_destroy(struct dp_netdev *dp, struct dpif *dpif)
> +{
> +    ovs_mutex_lock(&dp_netdev_mutex);
> +
> +    free(dpif->base_name);
> +    free(dpif->full_name);
> +    free(dpif);
> +    ovs_refcount_unref(&dp->ref_cnt);
> +
> +    ovs_mutex_unlock(&dp_netdev_mutex);
> +}
> +
>  static void
>  dp_netdev_destroy_upcall_lock(struct dp_netdev *dp)
>      OVS_NO_THREAD_SAFETY_ANALYSIS
> @@ -1657,6 +1666,15 @@ dp_delete_meter(struct dp_netdev *dp, uint32_t meter_id)
>      OVS_REQUIRES(dp->meter_locks[meter_id % N_METER_LOCKS])
>  {
>      if (dp->meters[meter_id]) {
> +        if (dp->meters[meter_id]->offload) {
> +            struct netdev_offload_meter *nom;
> +
> +            nom = dp->meters[meter_id]->offload;
> +            nom->meter_ops->meter_destroy(nom->priv_data);
> +            free(nom);
> +            dp->meters[meter_id]->offload = NULL;
> +        }
> +
>          free(dp->meters[meter_id]);
>          dp->meters[meter_id] = NULL;
>      }
> @@ -2385,6 +2403,7 @@ dp_netdev_alloc_flow_offload(struct dp_netdev_pmd_thread *pmd,
>      offload->flow = flow;
>      offload->op = op;
>      offload->should_jump = false;
> +    offload->dpif = dpif_netdev_dump_create(pmd->dp);
>
>      dp_netdev_flow_ref(flow);
>      dp_netdev_pmd_try_ref(pmd);
> @@ -2395,6 +2414,7 @@ dp_netdev_alloc_flow_offload(struct dp_netdev_pmd_thread *pmd,
>  static void
>  dp_netdev_free_flow_offload(struct dp_flow_offload_item *offload)
>  {
> +    dpif_netdev_dump_destroy(offload->pmd->dp, offload->dpif);
>      dp_netdev_pmd_unref(offload->pmd);
>      dp_netdev_flow_unref(offload->flow);
>
> @@ -2480,6 +2500,7 @@ dp_netdev_flow_offload_put(struct dp_flow_offload_item *offload)
>          }
>      }
>      info.flow_mark = mark;
> +    info.dpif_class = pmd->dp->class;
>      info.dpif_type_str = dpif_type_str;
>      info.mod_flag = offload->mod_flag;
>      info.nat_action = offload->nat_action;
> @@ -2487,6 +2508,7 @@ dp_netdev_flow_offload_put(struct dp_flow_offload_item *offload)
>      info.ori_nw_dst = offload->ori_nw_dst;
>      info.ct_enable = false;
>      info.group_id = 0;
> +    info.dpif_class = pmd->dp->class;
>
>      port = netdev_ports_get(in_port, dpif_type_str);
>      if (!port) {
> @@ -2495,7 +2517,7 @@ dp_netdev_flow_offload_put(struct dp_flow_offload_item *offload)
>      /* Taking a global 'port_mutex' to fulfill thread safety restrictions for
>       * the netdev-offload-dpdk module. */
>      ovs_mutex_lock(&pmd->dp->port_mutex);
> -    ret = netdev_flow_put(port, &offload->match,
> +    ret = netdev_flow_put(offload->dpif, port, &offload->match,
>                            CONST_CAST(struct nlattr *, offload->actions),
>                            offload->actions_len, &flow->ufid, &info,
>                            NULL);
> @@ -6193,6 +6215,15 @@ dpif_netdev_meter_set(struct dpif *dpif, ofproto_meter_id meter_id,
>      }
>
>      meter_lock(dp, mid);
> +    if (dp->meters[mid] && dp->meters[mid]->offload) {
> +        struct netdev_offload_meter *nom;
> +
> +        nom = dp->meters[mid]->offload;
> +        nom->meter_ops->meter_update(nom->priv_data, config);
> +        meter->offload = nom;
> +        dp->meters[mid]->offload = NULL;
> +    }
> +
>      dp_delete_meter(dp, mid); /* Free existing meter, if any */
>      dp->meters[mid] = meter;
>      meter_unlock(dp, mid);
> @@ -6257,7 +6288,97 @@ dpif_netdev_meter_del(struct dpif *dpif,
>      return error;
>  }
>
> -
> +static int
> +dpif_netdev_meter_get_config(struct dpif *dpif,
> +                             ofproto_meter_id meter_id,
> +                             struct ofputil_meter_config *conf)
> +{
> +    struct dp_netdev *dp = get_dp_netdev(dpif);
> +    uint32_t mid = meter_id.uint32;
> +    struct dp_meter *meter;
> +    int ret = 0, i;
> +
> +    if (mid >= MAX_METERS) {
> +        return -1;
> +    }
> +
> +    meter_lock(dp, mid);
> +    meter = dp->meters[mid];
> +    if (!meter) {
> +        ret = -1;
> +        goto done;
> +    }
> +
> +    conf->n_bands = meter->n_bands;
> +    conf->bands = xzalloc(conf->n_bands * sizeof(struct dp_meter_band));
> +    for (i = 0; i < meter->n_bands; ++i) {
> +        conf->bands[i] = meter->bands[i].up;
> +    }
> +
> +done:
> +    meter_unlock(dp, mid);
> +    return ret;
> +}
> +
> +static int
> +dpif_netdev_meter_set_offload(struct dpif *dpif,
> +                              ofproto_meter_id meter_id,
> +                              void *data)
> +{
> +    struct dp_netdev *dp = get_dp_netdev(dpif);
> +    uint32_t mid = meter_id.uint32;
> +    struct dp_meter *meter;
> +    int ret = 0;
> +
> +    if (mid >= MAX_METERS) {
> +        return -1;
> +    }
> +
> +    meter_lock(dp, mid);
> +    meter = dp->meters[mid];
> +    if (!meter) {
> +        ret = -1;
> +        goto done;
> +    }
> +
> +    meter->offload = (struct netdev_offload_meter *)data;
> +done:
> +    meter_unlock(dp, mid);
> +    return ret;
> +}
> +
> +static int
> +dpif_netdev_meter_get_offload(struct dpif *dpif,
> +                              ofproto_meter_id meter_id,
> +                              void **data, uint16_t size)
> +{
> +    struct dp_netdev *dp = get_dp_netdev(dpif);
> +    uint32_t mid = meter_id.uint32;
> +    struct dp_meter *meter;
> +    int ret = 0;
> +
> +    if (mid >= MAX_METERS) {
> +        return -1;
> +    }
> +
> +    meter_lock(dp, mid);
> +    meter = dp->meters[mid];
> +    if (!meter) {
> +        ret = -1;
> +        goto done;
> +    }
> +
> +    *data = NULL;
> +    if (meter->offload) {
> +        *data = xmemdup(meter->offload, size);
> +    }
> +
> +done:
> +    meter_unlock(dp, mid);
> +    return ret;
> +}
> +
> +
>  static void
>  dpif_netdev_disable_upcall(struct dpif *dpif)
>      OVS_NO_THREAD_SAFETY_ANALYSIS
> @@ -8088,6 +8209,9 @@ const struct dpif_class dpif_netdev_class = {
>      dpif_netdev_meter_set,
>      dpif_netdev_meter_get,
>      dpif_netdev_meter_del,
> +    dpif_netdev_meter_get_config,
> +    dpif_netdev_meter_set_offload,
> +    dpif_netdev_meter_get_offload,
>  };
>
>  static void
> diff --git a/lib/dpif-netdev.h b/lib/dpif-netdev.h
> index 6db6ed2..123cf99 100644
> --- a/lib/dpif-netdev.h
> +++ b/lib/dpif-netdev.h
> @@ -21,6 +21,7 @@
>  #include <stddef.h>
>  #include <stdint.h>
>  #include "dpif.h"
> +#include "netdev-offload-provider.h"
>  #include "openvswitch/types.h"
>  #include "dp-packet.h"
>  #include "packets.h"
> @@ -33,6 +34,24 @@ extern "C" {
>   * headers to be aligned on a 4-byte boundary.  */
>  enum { DP_NETDEV_HEADROOM = 2 + VLAN_HEADER_LEN };
>
> +struct dp_meter_band {
> +    struct ofputil_meter_band up; /* type, prec_level, pad, rate, burst_size */
> +    uint32_t bucket; /* In 1/1000 packets (for PKTPS), or in bits (for KBPS) */
> +    uint64_t packet_count;
> +    uint64_t byte_count;
> +};
> +
> +struct dp_meter {
> +    uint16_t flags;
> +    uint16_t n_bands;
> +    uint32_t max_delta_t;
> +    uint64_t used;
> +    uint64_t packet_count;
> +    uint64_t byte_count;
> +    struct netdev_offload_meter *offload;
> +    struct dp_meter_band bands[];
> +};
> +
>  bool dpif_is_netdev(const struct dpif *);
>
>  #define NR_QUEUE   1
> diff --git a/lib/dpif-netlink.c b/lib/dpif-netlink.c
> index 57ad657..7c921bd 100644
> --- a/lib/dpif-netlink.c
> +++ b/lib/dpif-netlink.c
> @@ -2093,7 +2093,7 @@ parse_flow_put(struct dpif_netlink *dpif, struct dpif_flow_put *put)
>      info.recirc_id_shared_with_tc = (dpif->user_features
>                                       & OVS_DP_F_TC_RECIRC_SHARING);
>      info.tc_modify_flow_deleted = false;
> -    err = netdev_flow_put(dev, &match,
> +    err = netdev_flow_put(NULL, dev, &match,
>                            CONST_CAST(struct nlattr *, put->actions),
>                            put->actions_len,
>                            CONST_CAST(ovs_u128 *, put->ufid),
> @@ -4001,6 +4001,9 @@ const struct dpif_class dpif_netlink_class = {
>      dpif_netlink_meter_set,
>      dpif_netlink_meter_get,
>      dpif_netlink_meter_del,
> +    NULL,
> +    NULL,
> +    NULL,
>  };
>
>  static int
> diff --git a/lib/dpif-provider.h b/lib/dpif-provider.h
> index b77317b..50fe086 100644
> --- a/lib/dpif-provider.h
> +++ b/lib/dpif-provider.h
> @@ -616,6 +616,11 @@ struct dpif_class {
>       * zero. */
>      int (*meter_del)(struct dpif *, ofproto_meter_id meter_id,
>                       struct ofputil_meter_stats *, uint16_t n_bands);
> +    int (*meter_get_config)(struct dpif *, ofproto_meter_id meter_id,
> +                            struct ofputil_meter_config *);
> +    int (*meter_set_offload)(struct dpif *, ofproto_meter_id meter_id, void *);
> +    int (*meter_get_offload)(struct dpif *, ofproto_meter_id meter_id, void **,
> +                             uint16_t size);
>  };
>
>  extern const struct dpif_class dpif_netlink_class;
> diff --git a/lib/dpif.c b/lib/dpif.c
> index 8264bb5..9a7d312 100644
> --- a/lib/dpif.c
> +++ b/lib/dpif.c
> @@ -66,6 +66,9 @@ COVERAGE_DEFINE(dpif_execute_with_help);
>  COVERAGE_DEFINE(dpif_meter_set);
>  COVERAGE_DEFINE(dpif_meter_get);
>  COVERAGE_DEFINE(dpif_meter_del);
> +COVERAGE_DEFINE(dpif_meter_get_config);
> +COVERAGE_DEFINE(dpif_meter_set_offload);
> +COVERAGE_DEFINE(dpif_meter_get_offload);
>
>  static const struct dpif_class *base_dpif_classes[] = {
>  #if defined(__linux__) || defined(_WIN32)
> @@ -1979,3 +1982,66 @@ dpif_meter_del(struct dpif *dpif, ofproto_meter_id meter_id,
>      }
>      return error;
>  }
> +
> +int
> +dpif_meter_get_config(struct dpif *dpif, ofproto_meter_id meter_id,
> +                      struct ofputil_meter_config *config)
> +{
> +    COVERAGE_INC(dpif_meter_get_config);
> +
> +    int error = (dpif->dpif_class->meter_get_config
> +                 ? dpif->dpif_class->meter_get_config(dpif, meter_id,
> +                                                      config)
> +                 : EOPNOTSUPP);
> +    if (!error) {
> +        VLOG_DBG_RL(&dpmsg_rl, "%s: DPIF meter %"PRIu32" get config",
> +                    dpif_name(dpif), meter_id.uint32);
> +    } else {
> +        VLOG_WARN_RL(&error_rl, "%s: failed to get DPIF meter config %"PRIu32": %s",
> +                     dpif_name(dpif), meter_id.uint32, ovs_strerror(error));
> +    }
> +
> +    return error;
> +}
> +
> +int
> +dpif_meter_set_offload(struct dpif *dpif, ofproto_meter_id meter_id,
> +                       void *data)
> +{
> +    COVERAGE_INC(dpif_meter_set_offload);
> +
> +    int error = (dpif->dpif_class->meter_set_offload
> +                 ? dpif->dpif_class->meter_set_offload(dpif, meter_id,
> +                                                       data)
> +                 : EOPNOTSUPP);
> +    if (!error) {
> +        VLOG_DBG_RL(&dpmsg_rl, "%s: DPIF meter %"PRIu32" offload set",
> +                    dpif_name(dpif), meter_id.uint32);
> +    } else {
> +        VLOG_WARN_RL(&error_rl, "%s: failed to offload set DPIF meter %"PRIu32": %s",
> +                     dpif_name(dpif), meter_id.uint32, ovs_strerror(error));
> +    }
> +
> +    return error;
> +}
> +
> +int
> +dpif_meter_get_offload(struct dpif *dpif, ofproto_meter_id meter_id,
> +                       void **data, uint16_t size)
> +{
> +    COVERAGE_INC(dpif_meter_get_offload);
> +
> +    int error = (dpif->dpif_class->meter_get_offload
> +                 ? dpif->dpif_class->meter_get_offload(dpif, meter_id,
> +                                                       data, size)
> +                 : EOPNOTSUPP);
> +    if (!error) {
> +        VLOG_DBG_RL(&dpmsg_rl, "%s: DPIF meter %"PRIu32" offload get",
> +                    dpif_name(dpif), meter_id.uint32);
> +    } else {
> +        VLOG_WARN_RL(&error_rl, "%s: failed to offload get DPIF meter %"PRIu32": %s",
> +                     dpif_name(dpif), meter_id.uint32, ovs_strerror(error));
> +    }
> +
> +    return error;
> +}
> diff --git a/lib/dpif.h b/lib/dpif.h
> index 4df8f7c..c90ca71 100644
> --- a/lib/dpif.h
> +++ b/lib/dpif.h
> @@ -891,6 +891,12 @@ int dpif_meter_get(const struct dpif *, ofproto_meter_id meter_id,
>                     struct ofputil_meter_stats *, uint16_t n_bands);
>  int dpif_meter_del(struct dpif *, ofproto_meter_id meter_id,
>                     struct ofputil_meter_stats *, uint16_t n_bands);
> +int dpif_meter_get_config(struct dpif *dpif, ofproto_meter_id meter_id,
> +                          struct ofputil_meter_config *config);
> +int dpif_meter_set_offload(struct dpif *dpif, ofproto_meter_id meter_id,
> +                           void *data);
> +int dpif_meter_get_offload(struct dpif *dpif, ofproto_meter_id meter_id,
> +                           void **data, uint16_t size);
>
>  /* Miscellaneous. */
>
> diff --git a/lib/netdev-dpdk.c b/lib/netdev-dpdk.c
> index 45fc908..9f382ea 100644
> --- a/lib/netdev-dpdk.c
> +++ b/lib/netdev-dpdk.c
> @@ -1205,6 +1205,12 @@ netdev_dpdk_cast(const struct netdev *netdev)
>      return CONTAINER_OF(netdev, struct netdev_dpdk, up);
>  }
>
> +int
> +netdev_dpdk_get_portid(const struct netdev *netdev)
> +{
> +    return netdev_dpdk_cast(netdev)->port_id;
> +}
> +
>  static struct netdev *
>  netdev_dpdk_alloc(void)
>  {
> diff --git a/lib/netdev-dpdk.h b/lib/netdev-dpdk.h
> index 811992c..7c82bc6 100644
> --- a/lib/netdev-dpdk.h
> +++ b/lib/netdev-dpdk.h
> @@ -37,6 +37,7 @@ void netdev_dpdk_register(void);
>  void free_dpdk_buf(struct dp_packet *);
>
>  bool netdev_dpdk_flow_api_supported(struct netdev *);
> +int netdev_dpdk_get_portid(const struct netdev *netdev);
>
>  int
>  netdev_dpdk_rte_flow_destroy(struct netdev *netdev,
> diff --git a/lib/netdev-dummy.c b/lib/netdev-dummy.c
> index 71df291..1d203c7 100644
> --- a/lib/netdev-dummy.c
> +++ b/lib/netdev-dummy.c
> @@ -1430,10 +1430,10 @@ find_offloaded_flow(const struct hmap *offloaded_flows, const ovs_u128 *ufid)
>  }
>
>  static int
> -netdev_dummy_flow_put(struct netdev *netdev, struct match *match,
> -                      struct nlattr *actions OVS_UNUSED,
> -                      size_t actions_len OVS_UNUSED,
> -                      const ovs_u128 *ufid, struct offload_info *info,
> +netdev_dummy_flow_put(struct dpif *dpif OVS_UNUSED, struct netdev *netdev,
> +                      struct match *match, struct nlattr *actions OVS_UNUSED,
> +                      size_t actions_len OVS_UNUSED, const ovs_u128 *ufid,
> +                      struct offload_info *info,
>                        struct dpif_flow_stats *stats)
>  {
>      struct netdev_dummy *dev = netdev_dummy_cast(netdev);
> diff --git a/lib/netdev-offload-dpdk.c b/lib/netdev-offload-dpdk.c
> index f7c8f2b..0a440eb 100644
> --- a/lib/netdev-offload-dpdk.c
> +++ b/lib/netdev-offload-dpdk.c
> @@ -14,17 +14,19 @@
>   * See the License for the specific language governing permissions and
>   * limitations under the License.
>   */
> -#include <config.h>
> -
> +#include "netdev-vport.h"
>  #include <sys/types.h>
>  #include <netinet/ip6.h>
> +#include <config.h>
> +
>  #include <rte_flow.h>
> +#include <rte_mtr.h>
>
>  #include "cmap.h"
>  #include "dpif-netdev.h"
>  #include "netdev-offload-provider.h"
> +#include "openvswitch/ofp-meter.h"
>  #include "netdev-provider.h"
> -#include "netdev-vport.h"
>  #include "openvswitch/match.h"
>  #include "openvswitch/vlog.h"
>  #include "packets.h"
> @@ -1173,6 +1175,28 @@ parse_flow_match(struct flow_patterns *patterns,
>      return 0;
>  }
>
> +struct dpdk_meter_offload {
> +    uint32_t port_id;
> +    uint32_t max_rate;
> +    uint32_t mp_id;
> +    struct rte_flow_action_meter mc;
> +};
> +
> +static void dpdk_meter_destroy(void *priv_data)
> +{
> +    struct dpdk_meter_offload *dmo = priv_data;
> +    struct rte_mtr_error mtr_error;
> +
> +    if (dmo) {
> +        rte_mtr_meter_profile_delete(dmo->port_id,
> +                                     dmo->mc.mtr_id,
> +                                     &mtr_error);
> +        rte_mtr_destroy(dmo->port_id, dmo->mc.mtr_id,
> +                        &mtr_error);
> +        free(dmo);
> +    }
> +}
> +
>  static void
>  add_flow_mark_rss_actions(struct flow_actions *actions,
>                            uint32_t flow_mark,
> @@ -1766,8 +1790,122 @@ parse_clone_actions(struct netdev *netdev,
>      return 0;
>  }
>
> +static struct rte_flow_action_meter*
> +dpdk_meter_create(struct dpif *dpif, struct netdev *netdev, uint32_t mid)
> +{
> +    uint32_t port_id = netdev_dpdk_get_portid(netdev);
> +    struct netdev_offload_meter *nom;
> +    struct dpdk_meter_offload *dmo;
> +    struct ofputil_meter_config config;
> +    ofproto_meter_id meter_id;
> +    struct rte_mtr_meter_profile mp;
> +    struct rte_mtr_params params;
> +    struct rte_mtr_error mtr_error;
> +    uint32_t max_rate;
> +    int ret;
> +    static struct netdev_offload_meter_api dpdk_meter_offload_api;
> +
> +    meter_id.uint32 = mid;
> +
> +    if (dpif_meter_get_config(dpif, meter_id, &config)) {
> +        return NULL;
> +    }
> +
> +    nom = xmalloc(sizeof *nom);
> +    dmo = xmalloc(sizeof *dmo);
> +
> +    nom->meter_ops = &dpdk_meter_offload_api;
> +    nom->priv_data = dmo;
> +
> +    memset(&mp, 0, sizeof(struct rte_mtr_meter_profile));
> +    max_rate = ofputil_meter_config_max_rate(&config);
> +
> +    dmo->mc.mtr_id = mid;
> +    dmo->port_id = port_id;
> +    dmo->max_rate = max_rate;
> +    dmo->mp_id = mid;
> +
> +    mp.alg = RTE_MTR_SRTCM_RFC2697;
> +    mp.srtcm_rfc2697.cir = max_rate *1024 /8; /* rate_max Kbps*/
> +    mp.srtcm_rfc2697.cbs = max_rate *1024 /8;
> +    mp.srtcm_rfc2697.ebs = 0;
> +
> +    ret = rte_mtr_meter_profile_add(dmo->port_id, dmo->mc.mtr_id,
> +                                    &mp, &mtr_error);
> +    if (ret && ret != -EEXIST) {
> +        VLOG_ERR("rte_mtr_meter_profile_add fail: err_type: %d err_msg: %s, portid: %d\n",
> +                   mtr_error.type, mtr_error.message, netdev_dpdk_get_portid(netdev));
> +        goto profile_err;
> +    }
> +
> +    enum rte_color dscp_table[2];
> +    dscp_table[0] = RTE_COLOR_YELLOW;
> +    dscp_table[1] = RTE_COLOR_RED;
> +
> +    params.meter_profile_id = dmo->mc.mtr_id;
> +    params.dscp_table = dscp_table;
> +    params.meter_enable = 1;
> +    params.use_prev_mtr_color = 0;
> +    params.action[RTE_COLOR_GREEN]  = MTR_POLICER_ACTION_COLOR_GREEN;
> +    params.action[RTE_COLOR_YELLOW] = MTR_POLICER_ACTION_DROP;
> +    params.action[RTE_COLOR_RED]    = MTR_POLICER_ACTION_DROP;
> +
> +    ret = rte_mtr_create(dmo->port_id, dmo->mc.mtr_id, &params, 0, &mtr_error);
> +    if (ret && ret != -EEXIST) {
> +        VLOG_ERR("rte_mtr_create fail: err_type: %d err_msg: %s, portid: %d\n",
> +                   mtr_error.type, mtr_error.message, netdev_dpdk_get_portid(netdev));
> +        goto mtr_err;
> +    }
> +
> +    dpif_meter_set_offload(dpif, meter_id, nom);
> +
> +    free(config.bands);
> +    return &dmo->mc;
> +
> +mtr_err:
> +    rte_mtr_meter_profile_delete(dmo->port_id, dmo->mc.mtr_id, &mtr_error);
> +
> +profile_err:
> +    free(nom);
> +    free(dmo);
> +    free(config.bands);
> +    return NULL;
> +}
> +
> +static struct rte_flow_action_meter *
> +netdev_offload_dpdk_meter_conf(struct dpif *dpif, struct netdev *netdev, uint32_t mid)
> +{
> +    uint32_t port_id = netdev_dpdk_get_portid(netdev);
> +    struct netdev_offload_meter *nom = NULL;
> +    struct dpdk_meter_offload *dmo;
> +    ofproto_meter_id meter_id;
> +    uint32_t ret;
> +    meter_id.uint32 = mid;
> +
> +    ret = dpif_meter_get_offload(dpif, meter_id, (void **)&nom,
> +                                 sizeof *nom + sizeof *dmo);
> +    if (ret) {
> +        VLOG_INFO("netdev offload dpdk meter, can't get the meter");
> +        return NULL;
> +    }
> +
> +    if (!nom) {
> +        return dpdk_meter_create(dpif, netdev, mid);
> +    }
> +
> +    dmo = (struct dpdk_meter_offload *)nom->priv_data;
> +    if (port_id != dmo->port_id) {
> +        VLOG_INFO("dpdk meter %d is used on %d, can't be used for : %d",
> +                  mid, dmo->port_id, port_id);
> +        return NULL;
> +    }
> +
> +    return &dmo->mc;
> +}
> +
>  static int
> -parse_flow_actions(struct netdev *netdev,
> +parse_flow_actions(struct dpif *dpif,
> +                   struct netdev *netdev,
>                     struct flow_actions *actions,
>                     struct nlattr *nl_actions,
>                     size_t nl_actions_len,
> @@ -1840,7 +1978,17 @@ parse_flow_actions(struct netdev *netdev,
>              add_flow_action(actions, RTE_FLOW_ACTION_TYPE_JUMP, jump);
>
>          /* for ct && ct clear no need to translate*/
> -        } else if (nl_attr_type(nla) == OVS_ACTION_ATTR_CT_CLEAR ||
> +        } else if (nl_attr_type(nla) == OVS_ACTION_ATTR_METER) {
> +            struct rte_flow_action_meter *mc;
> +            mc = netdev_offload_dpdk_meter_conf(dpif,
> +                                                netdev,
> +                                                nl_attr_get_u32(nla));
> +            if (mc) {
> +                add_flow_action(actions, RTE_FLOW_ACTION_TYPE_METER, mc);
> +            }
> +
> +        }
> +          else if (nl_attr_type(nla) == OVS_ACTION_ATTR_CT_CLEAR ||
>                     nl_attr_type(nla) == OVS_ACTION_ATTR_CT ) {
>          } else {
>              VLOG_DBG_RL(&rl, "Unsupported action type %d", nl_attr_type(nla));
> @@ -1875,7 +2023,8 @@ parse_flow_actions(struct netdev *netdev,
>  }
>
>  static struct rte_flow *
> -netdev_offload_dpdk_actions(struct netdev *netdev,
> +netdev_offload_dpdk_actions(struct dpif *dpif,
> +                            struct netdev *netdev,
>                              struct flow_patterns *patterns,
>                              struct nlattr *nl_actions,
>                              size_t actions_len,
> @@ -1889,7 +2038,7 @@ netdev_offload_dpdk_actions(struct netdev *netdev,
>      struct rte_flow_error error;
>      int ret,i;
>
> -    ret = parse_flow_actions(netdev, &actions, nl_actions, actions_len, info);
> +    ret = parse_flow_actions(dpif, netdev, &actions, nl_actions, actions_len, info);
>      if (ret) {
>          VLOG_DBG("TIMO DBG: after parse_flow_actions, ret is -1");
>          goto out;
> @@ -1927,13 +2076,79 @@ out:
>      return flow;
>  }
>
> +#define DPDK_METER_UPATE_UP 65536
> +
> +static void dpdk_meter_update(void *priv_data, void *config)
> +{
> +    struct dpdk_meter_offload *dmo = priv_data;
> +    struct rte_mtr_meter_profile mp;
> +    struct rte_mtr_error mtr_error;
> +    uint32_t mp_id, new_mp_id;
> +    uint32_t max_rate;
> +    uint32_t ret;
> +
> +    if (!priv_data || !config) {
> +        return;
> +    }
> +
> +    max_rate = ofputil_meter_config_max_rate(config);
> +    if (dmo->max_rate == max_rate) {
> +        return;
> +    }
> +
> +    memset(&mp, 0, sizeof(struct rte_mtr_meter_profile));
> +    mp.alg = RTE_MTR_SRTCM_RFC2697;
> +    mp.srtcm_rfc2697.cir = max_rate *1024 /8;
> +    mp.srtcm_rfc2697.cbs = max_rate *1024 /8;
> +    mp.srtcm_rfc2697.ebs = 0;
> +
> +    if (dmo->mp_id < DPDK_METER_UPATE_UP) {
> +        new_mp_id = dmo->mp_id + DPDK_METER_UPATE_UP;
> +    } else {
> +        new_mp_id = dmo->mp_id - DPDK_METER_UPATE_UP;
> +    }
> +
> +    ret = rte_mtr_meter_profile_add(dmo->port_id, new_mp_id,
> +                                    &mp, &mtr_error);
> +    if (ret) {
> +        VLOG_ERR("rte_mtr_meter_profile_add fail: err_type: %d err_msg: %s\n",
> +                 mtr_error.type, mtr_error.message);
> +        return;
> +    }
> +
> +    ret = rte_mtr_meter_profile_update(dmo->port_id, dmo->mc.mtr_id,
> +                                       new_mp_id, &mtr_error);
> +    if (ret) {
> +        VLOG_ERR("rte_mtr_meter_profile_update fail: err_type: %d err_msg: %s\n",
> +                 mtr_error.type, mtr_error.message);
> +        mp_id = new_mp_id;
> +        goto out;
> +    }
> +
> +    mp_id = dmo->mp_id;
> +    dmo->mp_id = new_mp_id;
> +out:
> +    ret = rte_mtr_meter_profile_delete(dmo->port_id, mp_id, &mtr_error);
> +    if (ret) {
> +        VLOG_ERR("rte_mtr_meter_profile_update fail: err_type: %d err_msg: %s\n",
> +                 mtr_error.type, mtr_error.message);
> +    }
> +}
> +
> +static struct netdev_offload_meter_api dpdk_meter_offload_api = {
> +    .meter_destroy  = dpdk_meter_destroy,
> +    .meter_update   = dpdk_meter_update,
> +};
> +
> +
> +
>  static int
> -netdev_offload_dpdk_add_flow(struct netdev *netdev,
> +netdev_offload_dpdk_add_flow(struct dpif *dpif, struct netdev *netdev,
>                               struct match *match,
>                               struct nlattr *nl_actions,
>                               size_t actions_len,
>                               const ovs_u128 *ufid,
> -                             struct offload_info *info)
> +                             struct offload_info *info OVS_UNUSED)
>  {
>      struct flow_patterns patterns = { .items = NULL, .cnt = 0 };
>      bool actions_offloaded = true;
> @@ -1943,10 +2158,12 @@ netdev_offload_dpdk_add_flow(struct netdev *netdev,
>      ret = parse_flow_match(&patterns, match, netdev, info);
>      if (ret) {
>          goto out;
> +        VLOG_DBG("yunxiang_debug_netdev_offload_dpdk_add_flow: ret %p \n",ret);
>      }
> -
> -    flow = netdev_offload_dpdk_actions(netdev, &patterns, nl_actions,
> +
> +    flow = netdev_offload_dpdk_actions(dpif, netdev, &patterns, nl_actions,
>                                         actions_len, info);
> +    VLOG_DBG("yunxiang_debug_netdev_offload_dpdk_add_flow: flow %p \n",flow);
>      if (!flow) {
>          /* If we failed to offload the rule actions fallback to MARK+RSS
>           * actions.
> @@ -2075,9 +2292,10 @@ netdev_offload_dpdk_destroy_flow(struct netdev *netdev,
>  }
>
>  static int
> -netdev_offload_dpdk_flow_put(struct netdev *netdev, struct match *match,
> -                             struct nlattr *actions, size_t actions_len,
> -                             const ovs_u128 *ufid, struct offload_info *info,
> +netdev_offload_dpdk_flow_put(struct dpif *dpif, struct netdev *netdev,
> +                             struct match *match, struct nlattr *actions,
> +                             size_t actions_len, const ovs_u128 *ufid,
> +                             struct offload_info *info,
>                               struct dpif_flow_stats *stats)
>  {
>      struct ufid_to_rte_flow_data *rte_flow_data;
> @@ -2088,10 +2306,12 @@ netdev_offload_dpdk_flow_put(struct netdev *netdev, struct match *match,
>       * Here destroy the old rte flow first before adding a new one.
>       */
>      rte_flow_data = ufid_to_rte_flow_data_find(ufid);
> +    VLOG_DBG("yunxiang_debug_netdev_offload_dpdk_flow_put,rte_flow_data=%p",rte_flow_data);
>      if (rte_flow_data && rte_flow_data->rte_flow) {
>          ret = netdev_offload_dpdk_destroy_flow(netdev, ufid,
>                                                 rte_flow_data->rte_flow);
>          if (ret < 0) {
> +            VLOG_DBG("yunxiang_debug_netdev_offload_dpdk_flow_put,ret=%p",ret);
>              return ret;
>          }
>      }
> @@ -2104,7 +2324,7 @@ netdev_offload_dpdk_flow_put(struct netdev *netdev, struct match *match,
>      if (stats) {
>          memset(stats, 0, sizeof *stats);
>      }
> -    return netdev_offload_dpdk_add_flow(netdev, match, actions,
> +    return netdev_offload_dpdk_add_flow(dpif, netdev, match, actions,
>                                          actions_len, ufid, info);
>  }
>
> diff --git a/lib/netdev-offload-provider.h b/lib/netdev-offload-provider.h
> index 5a809c0..f1830f8 100644
> --- a/lib/netdev-offload-provider.h
> +++ b/lib/netdev-offload-provider.h
> @@ -63,9 +63,10 @@ struct netdev_flow_api {
>       * 'stats' is populated according to the rules set out in the description
>       * above 'struct dpif_flow_put'.
>       * Return 0 if successful, otherwise returns a positive errno value. */
> -    int (*flow_put)(struct netdev *, struct match *, struct nlattr *actions,
> -                    size_t actions_len, const ovs_u128 *ufid,
> -                    struct offload_info *info, struct dpif_flow_stats *);
> +    int (*flow_put)(struct dpif *, struct netdev *, struct match *,
> +                    struct nlattr *actions, size_t actions_len,
> +                    const ovs_u128 *ufid, struct offload_info *info,
> +                    struct dpif_flow_stats *);
>
>      /* Queries a flow specified by ufid on netdev.
>       * Fills output buffer as 'wbuffer' in flow_dump_next, which
> @@ -87,6 +88,16 @@ struct netdev_flow_api {
>      int (*init_flow_api)(struct netdev *);
>  };
>
> +struct netdev_offload_meter_api {
> +    void (*meter_destroy)(void *);
> +    void (*meter_update)(void *, void *);
> +};
> +
> +struct netdev_offload_meter {
> +    void *priv_data;
> +    struct netdev_offload_meter_api *meter_ops;
> +};
> +
>  int netdev_register_flow_api_provider(const struct netdev_flow_api *);
>  int netdev_unregister_flow_api_provider(const char *type);
>  bool netdev_flow_api_equals(const struct netdev *, const struct netdev *);
> diff --git a/lib/netdev-offload-tc.c b/lib/netdev-offload-tc.c
> index 6c76f69..ba4bd67 100644
> --- a/lib/netdev-offload-tc.c
> +++ b/lib/netdev-offload-tc.c
> @@ -1354,9 +1354,10 @@ flower_match_to_tun_opt(struct tc_flower *flower, const struct flow_tnl *tnl,
>  }
>
>  static int
> -netdev_tc_flow_put(struct netdev *netdev, struct match *match,
> -                   struct nlattr *actions, size_t actions_len,
> -                   const ovs_u128 *ufid, struct offload_info *info,
> +netdev_tc_flow_put(struct dpif *dpif OVS_UNUSED, struct netdev *netdev,
> +                   struct match *match, struct nlattr *actions,
> +                   size_t actions_len, const ovs_u128 *ufid,
> +                   struct offload_info *info,
>                     struct dpif_flow_stats *stats)
>  {
>      static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(5, 20);
> diff --git a/lib/netdev-offload.c b/lib/netdev-offload.c
> index e054bb0..135a29a 100644
> --- a/lib/netdev-offload.c
> +++ b/lib/netdev-offload.c
> @@ -239,17 +239,18 @@ netdev_flow_dump_next(struct netdev_flow_dump *dump, struct match *match,
>
>
>  int
> -netdev_flow_put(struct netdev *netdev, struct match *match,
> -                struct nlattr *actions, size_t act_len,
> -                const ovs_u128 *ufid, struct offload_info *info,
> +netdev_flow_put(struct dpif *dpif, struct netdev *netdev,
> +                struct match *match, struct nlattr *actions,
> +                size_t act_len, const ovs_u128 *ufid,
> +                struct offload_info *info,
>                  struct dpif_flow_stats *stats)
>  {
>      const struct netdev_flow_api *flow_api =
>          ovsrcu_get(const struct netdev_flow_api *, &netdev->flow_api);
>
>      return (flow_api && flow_api->flow_put)
> -           ? flow_api->flow_put(netdev, match, actions, act_len, ufid,
> -                                info, stats)
> +           ? flow_api->flow_put(dpif, netdev, match, actions,
> +                                act_len, ufid, info, stats)
>             : EOPNOTSUPP;
>  }
>
> diff --git a/lib/netdev-offload.h b/lib/netdev-offload.h
> index f2c4fbc..2d43059 100644
> --- a/lib/netdev-offload.h
> +++ b/lib/netdev-offload.h
> @@ -20,8 +20,12 @@
>
>  #include "openvswitch/netdev.h"
>  #include "openvswitch/types.h"
> +#include "dpif-provider.h"
>  #include "packets.h"
>  #include "flow.h"
> +
> +
> +
>  #include "conntrack.h"
>
>  #ifdef  __cplusplus
> @@ -62,6 +66,7 @@ struct netdev_flow_dump {
>
>  /* Flow offloading. */
>  struct offload_info {
> +    const struct dpif_class *dpif_class;
>      ovs_be16 tp_dst_port; /* Destination port for tunnel in SET action */
>      uint8_t tunnel_csum_on; /* Tunnel header with checksum */
>
> @@ -96,7 +101,8 @@ bool netdev_flow_dump_next(struct netdev_flow_dump *, struct match *,
>                            struct nlattr **actions, struct dpif_flow_stats *,
>                            struct dpif_flow_attrs *, ovs_u128 *ufid,
>                            struct ofpbuf *rbuffer, struct ofpbuf *wbuffer);
> -int netdev_flow_put(struct netdev *, struct match *, struct nlattr *actions,
> +int netdev_flow_put(struct dpif *, struct netdev *,
> +                    struct match *, struct nlattr *actions,
>                      size_t actions_len, const ovs_u128 *,
>                      struct offload_info *, struct dpif_flow_stats *);
>  int netdev_flow_get(struct netdev *, struct match *, struct nlattr **actions,
> diff --git a/lib/ofp-meter.c b/lib/ofp-meter.c
> index 9ea40a0..633924e 100644
> --- a/lib/ofp-meter.c
> +++ b/lib/ofp-meter.c
> @@ -807,3 +807,17 @@ ofputil_format_meter_mod(struct ds *s, const struct ofputil_meter_mod *mm)
>
>      ofputil_format_meter_config(s, &mm->meter);
>  }
> +
> +uint32_t
> +ofputil_meter_config_max_rate(struct ofputil_meter_config *conf)
> +{
> +    uint32_t i, max_rate = 0;
> +    for (i = 0; i < conf->n_bands; i++) {
> +        if (max_rate < conf->bands[i].rate) {
> +            max_rate = conf->bands[i].rate;
> +        }
> +    }
> +
> +    return max_rate;
> +}
> +
> --
> 1.8.3.1
>
>
>
> _______________________________________________
> dev mailing list
> dev@openvswitch.org
> https://mail.openvswitch.org/mailman/listinfo/ovs-dev

Yes. I test your summit,and compose others to a big patch. This is a test .


--------------
taoyunxiang@cmss.chinamobile.com
>On Tue, Dec 29, 2020 at 2:13 AM Tao YunXiang
><taoyunxiang@cmss.chinamobile.com> wrote:
>>
>> From: Taoyunxiang <taoyunxiang@cmss.chinamobile.com>
>>
>> Code Source From: Self Code
>> Description:
>> This is the init version to support QoS function offload,
>> which is handled by meter aciton. So this change offload meter
>> action from openflow.
>> Jira:  #[Optional]
>> 市场项目编号（名称）：[Optional]
>>
>> Add QoS function offload support by meter.
>Hi
>this version is same as what I posted
>http://patchwork.ozlabs.org/project/openvswitch/list/?series=150532
>> ---
>>  include/openvswitch/ofp-meter.h |   1 +
>>  lib/dpif-netdev.c               | 162 +++++++++++++++++++++++---
>>  lib/dpif-netdev.h               |  19 +++
>>  lib/dpif-netlink.c              |   5 +-
>>  lib/dpif-provider.h             |   5 +
>>  lib/dpif.c                      |  66 +++++++++++
>>  lib/dpif.h                      |   6 +
>>  lib/netdev-dpdk.c               |   6 +
>>  lib/netdev-dpdk.h               |   1 +
>>  lib/netdev-dummy.c              |   8 +-
>>  lib/netdev-offload-dpdk.c       | 250 +++++++++++++++++++++++++++++++++++++---
>>  lib/netdev-offload-provider.h   |  17 ++-
>>  lib/netdev-offload-tc.c         |   7 +-
>>  lib/netdev-offload.c            |  11 +-
>>  lib/netdev-offload.h            |   8 +-
>>  lib/ofp-meter.c                 |  14 +++
>>  16 files changed, 535 insertions(+), 51 deletions(-)
>>
>> diff --git a/include/openvswitch/ofp-meter.h b/include/openvswitch/ofp-meter.h
>> index 6776eae..c6678f5 100644
>> --- a/include/openvswitch/ofp-meter.h
>> +++ b/include/openvswitch/ofp-meter.h
>> @@ -62,6 +62,7 @@ int ofputil_decode_meter_config(struct ofpbuf *,
>>                                  struct ofpbuf *bands);
>>  void ofputil_format_meter_config(struct ds *,
>>                                   const struct ofputil_meter_config *);
>> +uint32_t ofputil_meter_config_max_rate(struct ofputil_meter_config *conf);
>>
>>  struct ofputil_meter_mod {
>>      uint16_t command;
>> diff --git a/lib/dpif-netdev.c b/lib/dpif-netdev.c
>> index a40441f..b03a2ab 100644
>> --- a/lib/dpif-netdev.c
>> +++ b/lib/dpif-netdev.c
>> @@ -272,23 +272,6 @@ static bool dpcls_lookup(struct dpcls *cls,
>>  #define DP_SUPPORTED_METER_BAND_TYPES           \
>>      ( 1 << OFPMBT13_DROP )
>>
>> -struct dp_meter_band {
>> -    struct ofputil_meter_band up; /* type, prec_level, pad, rate, burst_size */
>> -    uint32_t bucket; /* In 1/1000 packets (for PKTPS), or in bits (for KBPS) */
>> -    uint64_t packet_count;
>> -    uint64_t byte_count;
>> -};
>> -
>> -struct dp_meter {
>> -    uint16_t flags;
>> -    uint16_t n_bands;
>> -    uint32_t max_delta_t;
>> -    uint64_t used;
>> -    uint64_t packet_count;
>> -    uint64_t byte_count;
>> -    struct dp_meter_band bands[];
>> -};
>> -
>>  struct pmd_auto_lb {
>>      bool auto_lb_requested;     /* Auto load balancing requested by user. */
>>      bool is_enabled;            /* Current status of Auto load balancing. */
>> @@ -424,6 +407,7 @@ struct dp_flow_offload_item {
>>      ovs_be32 ori_nw_dst;
>>
>>      struct ovs_list node;
>> +    struct dpif *dpif;
>>  };
>>
>>  struct dp_flow_offload {
>> @@ -1640,6 +1624,31 @@ dpif_netdev_open(const struct dpif_class *class, const char *name,
>>      return error;
>>  }
>>
>> +static struct dpif *
>> +dpif_netdev_dump_create(struct dp_netdev *dp)
>> +{
>> +    struct dpif *dpif = NULL;
>> +
>> +    ovs_mutex_lock(&dp_netdev_mutex);
>> +    dpif = create_dpif_netdev(dp);
>> +    ovs_mutex_unlock(&dp_netdev_mutex);
>> +
>> +    return dpif;
>> +}
>> +
>> +static void
>> +dpif_netdev_dump_destroy(struct dp_netdev *dp, struct dpif *dpif)
>> +{
>> +    ovs_mutex_lock(&dp_netdev_mutex);
>> +
>> +    free(dpif->base_name);
>> +    free(dpif->full_name);
>> +    free(dpif);
>> +    ovs_refcount_unref(&dp->ref_cnt);
>> +
>> +    ovs_mutex_unlock(&dp_netdev_mutex);
>> +}
>> +
>>  static void
>>  dp_netdev_destroy_upcall_lock(struct dp_netdev *dp)
>>      OVS_NO_THREAD_SAFETY_ANALYSIS
>> @@ -1657,6 +1666,15 @@ dp_delete_meter(struct dp_netdev *dp, uint32_t meter_id)
>>      OVS_REQUIRES(dp->meter_locks[meter_id % N_METER_LOCKS])
>>  {
>>      if (dp->meters[meter_id]) {
>> +        if (dp->meters[meter_id]->offload) {
>> +            struct netdev_offload_meter *nom;
>> +
>> +            nom = dp->meters[meter_id]->offload;
>> +            nom->meter_ops->meter_destroy(nom->priv_data);
>> +            free(nom);
>> +            dp->meters[meter_id]->offload = NULL;
>> +        }
>> +
>>          free(dp->meters[meter_id]);
>>          dp->meters[meter_id] = NULL;
>>      }
>> @@ -2385,6 +2403,7 @@ dp_netdev_alloc_flow_offload(struct dp_netdev_pmd_thread *pmd,
>>      offload->flow = flow;
>>      offload->op = op;
>>      offload->should_jump = false;
>> +    offload->dpif = dpif_netdev_dump_create(pmd->dp);
>>
>>      dp_netdev_flow_ref(flow);
>>      dp_netdev_pmd_try_ref(pmd);
>> @@ -2395,6 +2414,7 @@ dp_netdev_alloc_flow_offload(struct dp_netdev_pmd_thread *pmd,
>>  static void
>>  dp_netdev_free_flow_offload(struct dp_flow_offload_item *offload)
>>  {
>> +    dpif_netdev_dump_destroy(offload->pmd->dp, offload->dpif);
>>      dp_netdev_pmd_unref(offload->pmd);
>>      dp_netdev_flow_unref(offload->flow);
>>
>> @@ -2480,6 +2500,7 @@ dp_netdev_flow_offload_put(struct dp_flow_offload_item *offload)
>>          }
>>      }
>>      info.flow_mark = mark;
>> +    info.dpif_class = pmd->dp->class;
>>      info.dpif_type_str = dpif_type_str;
>>      info.mod_flag = offload->mod_flag;
>>      info.nat_action = offload->nat_action;
>> @@ -2487,6 +2508,7 @@ dp_netdev_flow_offload_put(struct dp_flow_offload_item *offload)
>>      info.ori_nw_dst = offload->ori_nw_dst;
>>      info.ct_enable = false;
>>      info.group_id = 0;
>> +    info.dpif_class = pmd->dp->class;
>>
>>      port = netdev_ports_get(in_port, dpif_type_str);
>>      if (!port) {
>> @@ -2495,7 +2517,7 @@ dp_netdev_flow_offload_put(struct dp_flow_offload_item *offload)
>>      /* Taking a global 'port_mutex' to fulfill thread safety restrictions for
>>       * the netdev-offload-dpdk module. */
>>      ovs_mutex_lock(&pmd->dp->port_mutex);
>> -    ret = netdev_flow_put(port, &offload->match,
>> +    ret = netdev_flow_put(offload->dpif, port, &offload->match,
>>                            CONST_CAST(struct nlattr *, offload->actions),
>>                            offload->actions_len, &flow->ufid, &info,
>>                            NULL);
>> @@ -6193,6 +6215,15 @@ dpif_netdev_meter_set(struct dpif *dpif, ofproto_meter_id meter_id,
>>      }
>>
>>      meter_lock(dp, mid);
>> +    if (dp->meters[mid] && dp->meters[mid]->offload) {
>> +        struct netdev_offload_meter *nom;
>> +
>> +        nom = dp->meters[mid]->offload;
>> +        nom->meter_ops->meter_update(nom->priv_data, config);
>> +        meter->offload = nom;
>> +        dp->meters[mid]->offload = NULL;
>> +    }
>> +
>>      dp_delete_meter(dp, mid); /* Free existing meter, if any */
>>      dp->meters[mid] = meter;
>>      meter_unlock(dp, mid);
>> @@ -6257,7 +6288,97 @@ dpif_netdev_meter_del(struct dpif *dpif,
>>      return error;
>>  }
>>
>> -
>> +static int
>> +dpif_netdev_meter_get_config(struct dpif *dpif,
>> +                             ofproto_meter_id meter_id,
>> +                             struct ofputil_meter_config *conf)
>> +{
>> +    struct dp_netdev *dp = get_dp_netdev(dpif);
>> +    uint32_t mid = meter_id.uint32;
>> +    struct dp_meter *meter;
>> +    int ret = 0, i;
>> +
>> +    if (mid >= MAX_METERS) {
>> +        return -1;
>> +    }
>> +
>> +    meter_lock(dp, mid);
>> +    meter = dp->meters[mid];
>> +    if (!meter) {
>> +        ret = -1;
>> +        goto done;
>> +    }
>> +
>> +    conf->n_bands = meter->n_bands;
>> +    conf->bands = xzalloc(conf->n_bands * sizeof(struct dp_meter_band));
>> +    for (i = 0; i < meter->n_bands; ++i) {
>> +        conf->bands[i] = meter->bands[i].up;
>> +    }
>> +
>> +done:
>> +    meter_unlock(dp, mid);
>> +    return ret;
>> +}
>> +
>> +static int
>> +dpif_netdev_meter_set_offload(struct dpif *dpif,
>> +                              ofproto_meter_id meter_id,
>> +                              void *data)
>> +{
>> +    struct dp_netdev *dp = get_dp_netdev(dpif);
>> +    uint32_t mid = meter_id.uint32;
>> +    struct dp_meter *meter;
>> +    int ret = 0;
>> +
>> +    if (mid >= MAX_METERS) {
>> +        return -1;
>> +    }
>> +
>> +    meter_lock(dp, mid);
>> +    meter = dp->meters[mid];
>> +    if (!meter) {
>> +        ret = -1;
>> +        goto done;
>> +    }
>> +
>> +    meter->offload = (struct netdev_offload_meter *)data;
>> +done:
>> +    meter_unlock(dp, mid);
>> +    return ret;
>> +}
>> +
>> +static int
>> +dpif_netdev_meter_get_offload(struct dpif *dpif,
>> +                              ofproto_meter_id meter_id,
>> +                              void **data, uint16_t size)
>> +{
>> +    struct dp_netdev *dp = get_dp_netdev(dpif);
>> +    uint32_t mid = meter_id.uint32;
>> +    struct dp_meter *meter;
>> +    int ret = 0;
>> +
>> +    if (mid >= MAX_METERS) {
>> +        return -1;
>> +    }
>> +
>> +    meter_lock(dp, mid);
>> +    meter = dp->meters[mid];
>> +    if (!meter) {
>> +        ret = -1;
>> +        goto done;
>> +    }
>> +
>> +    *data = NULL;
>> +    if (meter->offload) {
>> +        *data = xmemdup(meter->offload, size);
>> +    }
>> +
>> +done:
>> +    meter_unlock(dp, mid);
>> +    return ret;
>> +}
>> +
>> +
>>  static void
>>  dpif_netdev_disable_upcall(struct dpif *dpif)
>>      OVS_NO_THREAD_SAFETY_ANALYSIS
>> @@ -8088,6 +8209,9 @@ const struct dpif_class dpif_netdev_class = {
>>      dpif_netdev_meter_set,
>>      dpif_netdev_meter_get,
>>      dpif_netdev_meter_del,
>> +    dpif_netdev_meter_get_config,
>> +    dpif_netdev_meter_set_offload,
>> +    dpif_netdev_meter_get_offload,
>>  };
>>
>>  static void
>> diff --git a/lib/dpif-netdev.h b/lib/dpif-netdev.h
>> index 6db6ed2..123cf99 100644
>> --- a/lib/dpif-netdev.h
>> +++ b/lib/dpif-netdev.h
>> @@ -21,6 +21,7 @@
>>  #include <stddef.h>
>>  #include <stdint.h>
>>  #include "dpif.h"
>> +#include "netdev-offload-provider.h"
>>  #include "openvswitch/types.h"
>>  #include "dp-packet.h"
>>  #include "packets.h"
>> @@ -33,6 +34,24 @@ extern "C" {
>>   * headers to be aligned on a 4-byte boundary.  */
>>  enum { DP_NETDEV_HEADROOM = 2 + VLAN_HEADER_LEN };
>>
>> +struct dp_meter_band {
>> +    struct ofputil_meter_band up; /* type, prec_level, pad, rate, burst_size */
>> +    uint32_t bucket; /* In 1/1000 packets (for PKTPS), or in bits (for KBPS) */
>> +    uint64_t packet_count;
>> +    uint64_t byte_count;
>> +};
>> +
>> +struct dp_meter {
>> +    uint16_t flags;
>> +    uint16_t n_bands;
>> +    uint32_t max_delta_t;
>> +    uint64_t used;
>> +    uint64_t packet_count;
>> +    uint64_t byte_count;
>> +    struct netdev_offload_meter *offload;
>> +    struct dp_meter_band bands[];
>> +};
>> +
>>  bool dpif_is_netdev(const struct dpif *);
>>
>>  #define NR_QUEUE   1
>> diff --git a/lib/dpif-netlink.c b/lib/dpif-netlink.c
>> index 57ad657..7c921bd 100644
>> --- a/lib/dpif-netlink.c
>> +++ b/lib/dpif-netlink.c
>> @@ -2093,7 +2093,7 @@ parse_flow_put(struct dpif_netlink *dpif, struct dpif_flow_put *put)
>>      info.recirc_id_shared_with_tc = (dpif->user_features
>>                                       & OVS_DP_F_TC_RECIRC_SHARING);
>>      info.tc_modify_flow_deleted = false;
>> -    err = netdev_flow_put(dev, &match,
>> +    err = netdev_flow_put(NULL, dev, &match,
>>                            CONST_CAST(struct nlattr *, put->actions),
>>                            put->actions_len,
>>                            CONST_CAST(ovs_u128 *, put->ufid),
>> @@ -4001,6 +4001,9 @@ const struct dpif_class dpif_netlink_class = {
>>      dpif_netlink_meter_set,
>>      dpif_netlink_meter_get,
>>      dpif_netlink_meter_del,
>> +    NULL,
>> +    NULL,
>> +    NULL,
>>  };
>>
>>  static int
>> diff --git a/lib/dpif-provider.h b/lib/dpif-provider.h
>> index b77317b..50fe086 100644
>> --- a/lib/dpif-provider.h
>> +++ b/lib/dpif-provider.h
>> @@ -616,6 +616,11 @@ struct dpif_class {
>>       * zero. */
>>      int (*meter_del)(struct dpif *, ofproto_meter_id meter_id,
>>                       struct ofputil_meter_stats *, uint16_t n_bands);
>> +    int (*meter_get_config)(struct dpif *, ofproto_meter_id meter_id,
>> +                            struct ofputil_meter_config *);
>> +    int (*meter_set_offload)(struct dpif *, ofproto_meter_id meter_id, void *);
>> +    int (*meter_get_offload)(struct dpif *, ofproto_meter_id meter_id, void **,
>> +                             uint16_t size);
>>  };
>>
>>  extern const struct dpif_class dpif_netlink_class;
>> diff --git a/lib/dpif.c b/lib/dpif.c
>> index 8264bb5..9a7d312 100644
>> --- a/lib/dpif.c
>> +++ b/lib/dpif.c
>> @@ -66,6 +66,9 @@ COVERAGE_DEFINE(dpif_execute_with_help);
>>  COVERAGE_DEFINE(dpif_meter_set);
>>  COVERAGE_DEFINE(dpif_meter_get);
>>  COVERAGE_DEFINE(dpif_meter_del);
>> +COVERAGE_DEFINE(dpif_meter_get_config);
>> +COVERAGE_DEFINE(dpif_meter_set_offload);
>> +COVERAGE_DEFINE(dpif_meter_get_offload);
>>
>>  static const struct dpif_class *base_dpif_classes[] = {
>>  #if defined(__linux__) || defined(_WIN32)
>> @@ -1979,3 +1982,66 @@ dpif_meter_del(struct dpif *dpif, ofproto_meter_id meter_id,
>>      }
>>      return error;
>>  }
>> +
>> +int
>> +dpif_meter_get_config(struct dpif *dpif, ofproto_meter_id meter_id,
>> +                      struct ofputil_meter_config *config)
>> +{
>> +    COVERAGE_INC(dpif_meter_get_config);
>> +
>> +    int error = (dpif->dpif_class->meter_get_config
>> +                 ? dpif->dpif_class->meter_get_config(dpif, meter_id,
>> +                                                      config)
>> +                 : EOPNOTSUPP);
>> +    if (!error) {
>> +        VLOG_DBG_RL(&dpmsg_rl, "%s: DPIF meter %"PRIu32" get config",
>> +                    dpif_name(dpif), meter_id.uint32);
>> +    } else {
>> +        VLOG_WARN_RL(&error_rl, "%s: failed to get DPIF meter config %"PRIu32": %s",
>> +                     dpif_name(dpif), meter_id.uint32, ovs_strerror(error));
>> +    }
>> +
>> +    return error;
>> +}
>> +
>> +int
>> +dpif_meter_set_offload(struct dpif *dpif, ofproto_meter_id meter_id,
>> +                       void *data)
>> +{
>> +    COVERAGE_INC(dpif_meter_set_offload);
>> +
>> +    int error = (dpif->dpif_class->meter_set_offload
>> +                 ? dpif->dpif_class->meter_set_offload(dpif, meter_id,
>> +                                                       data)
>> +                 : EOPNOTSUPP);
>> +    if (!error) {
>> +        VLOG_DBG_RL(&dpmsg_rl, "%s: DPIF meter %"PRIu32" offload set",
>> +                    dpif_name(dpif), meter_id.uint32);
>> +    } else {
>> +        VLOG_WARN_RL(&error_rl, "%s: failed to offload set DPIF meter %"PRIu32": %s",
>> +                     dpif_name(dpif), meter_id.uint32, ovs_strerror(error));
>> +    }
>> +
>> +    return error;
>> +}
>> +
>> +int
>> +dpif_meter_get_offload(struct dpif *dpif, ofproto_meter_id meter_id,
>> +                       void **data, uint16_t size)
>> +{
>> +    COVERAGE_INC(dpif_meter_get_offload);
>> +
>> +    int error = (dpif->dpif_class->meter_get_offload
>> +                 ? dpif->dpif_class->meter_get_offload(dpif, meter_id,
>> +                                                       data, size)
>> +                 : EOPNOTSUPP);
>> +    if (!error) {
>> +        VLOG_DBG_RL(&dpmsg_rl, "%s: DPIF meter %"PRIu32" offload get",
>> +                    dpif_name(dpif), meter_id.uint32);
>> +    } else {
>> +        VLOG_WARN_RL(&error_rl, "%s: failed to offload get DPIF meter %"PRIu32": %s",
>> +                     dpif_name(dpif), meter_id.uint32, ovs_strerror(error));
>> +    }
>> +
>> +    return error;
>> +}
>> diff --git a/lib/dpif.h b/lib/dpif.h
>> index 4df8f7c..c90ca71 100644
>> --- a/lib/dpif.h
>> +++ b/lib/dpif.h
>> @@ -891,6 +891,12 @@ int dpif_meter_get(const struct dpif *, ofproto_meter_id meter_id,
>>                     struct ofputil_meter_stats *, uint16_t n_bands);
>>  int dpif_meter_del(struct dpif *, ofproto_meter_id meter_id,
>>                     struct ofputil_meter_stats *, uint16_t n_bands);
>> +int dpif_meter_get_config(struct dpif *dpif, ofproto_meter_id meter_id,
>> +                          struct ofputil_meter_config *config);
>> +int dpif_meter_set_offload(struct dpif *dpif, ofproto_meter_id meter_id,
>> +                           void *data);
>> +int dpif_meter_get_offload(struct dpif *dpif, ofproto_meter_id meter_id,
>> +                           void **data, uint16_t size);
>>
>>  /* Miscellaneous. */
>>
>> diff --git a/lib/netdev-dpdk.c b/lib/netdev-dpdk.c
>> index 45fc908..9f382ea 100644
>> --- a/lib/netdev-dpdk.c
>> +++ b/lib/netdev-dpdk.c
>> @@ -1205,6 +1205,12 @@ netdev_dpdk_cast(const struct netdev *netdev)
>>      return CONTAINER_OF(netdev, struct netdev_dpdk, up);
>>  }
>>
>> +int
>> +netdev_dpdk_get_portid(const struct netdev *netdev)
>> +{
>> +    return netdev_dpdk_cast(netdev)->port_id;
>> +}
>> +
>>  static struct netdev *
>>  netdev_dpdk_alloc(void)
>>  {
>> diff --git a/lib/netdev-dpdk.h b/lib/netdev-dpdk.h
>> index 811992c..7c82bc6 100644
>> --- a/lib/netdev-dpdk.h
>> +++ b/lib/netdev-dpdk.h
>> @@ -37,6 +37,7 @@ void netdev_dpdk_register(void);
>>  void free_dpdk_buf(struct dp_packet *);
>>
>>  bool netdev_dpdk_flow_api_supported(struct netdev *);
>> +int netdev_dpdk_get_portid(const struct netdev *netdev);
>>
>>  int
>>  netdev_dpdk_rte_flow_destroy(struct netdev *netdev,
>> diff --git a/lib/netdev-dummy.c b/lib/netdev-dummy.c
>> index 71df291..1d203c7 100644
>> --- a/lib/netdev-dummy.c
>> +++ b/lib/netdev-dummy.c
>> @@ -1430,10 +1430,10 @@ find_offloaded_flow(const struct hmap *offloaded_flows, const ovs_u128 *ufid)
>>  }
>>
>>  static int
>> -netdev_dummy_flow_put(struct netdev *netdev, struct match *match,
>> -                      struct nlattr *actions OVS_UNUSED,
>> -                      size_t actions_len OVS_UNUSED,
>> -                      const ovs_u128 *ufid, struct offload_info *info,
>> +netdev_dummy_flow_put(struct dpif *dpif OVS_UNUSED, struct netdev *netdev,
>> +                      struct match *match, struct nlattr *actions OVS_UNUSED,
>> +                      size_t actions_len OVS_UNUSED, const ovs_u128 *ufid,
>> +                      struct offload_info *info,
>>                        struct dpif_flow_stats *stats)
>>  {
>>      struct netdev_dummy *dev = netdev_dummy_cast(netdev);
>> diff --git a/lib/netdev-offload-dpdk.c b/lib/netdev-offload-dpdk.c
>> index f7c8f2b..0a440eb 100644
>> --- a/lib/netdev-offload-dpdk.c
>> +++ b/lib/netdev-offload-dpdk.c
>> @@ -14,17 +14,19 @@
>>   * See the License for the specific language governing permissions and
>>   * limitations under the License.
>>   */
>> -#include <config.h>
>> -
>> +#include "netdev-vport.h"
>>  #include <sys/types.h>
>>  #include <netinet/ip6.h>
>> +#include <config.h>
>> +
>>  #include <rte_flow.h>
>> +#include <rte_mtr.h>
>>
>>  #include "cmap.h"
>>  #include "dpif-netdev.h"
>>  #include "netdev-offload-provider.h"
>> +#include "openvswitch/ofp-meter.h"
>>  #include "netdev-provider.h"
>> -#include "netdev-vport.h"
>>  #include "openvswitch/match.h"
>>  #include "openvswitch/vlog.h"
>>  #include "packets.h"
>> @@ -1173,6 +1175,28 @@ parse_flow_match(struct flow_patterns *patterns,
>>      return 0;
>>  }
>>
>> +struct dpdk_meter_offload {
>> +    uint32_t port_id;
>> +    uint32_t max_rate;
>> +    uint32_t mp_id;
>> +    struct rte_flow_action_meter mc;
>> +};
>> +
>> +static void dpdk_meter_destroy(void *priv_data)
>> +{
>> +    struct dpdk_meter_offload *dmo = priv_data;
>> +    struct rte_mtr_error mtr_error;
>> +
>> +    if (dmo) {
>> +        rte_mtr_meter_profile_delete(dmo->port_id,
>> +                                     dmo->mc.mtr_id,
>> +                                     &mtr_error);
>> +        rte_mtr_destroy(dmo->port_id, dmo->mc.mtr_id,
>> +                        &mtr_error);
>> +        free(dmo);
>> +    }
>> +}
>> +
>>  static void
>>  add_flow_mark_rss_actions(struct flow_actions *actions,
>>                            uint32_t flow_mark,
>> @@ -1766,8 +1790,122 @@ parse_clone_actions(struct netdev *netdev,
>>      return 0;
>>  }
>>
>> +static struct rte_flow_action_meter*
>> +dpdk_meter_create(struct dpif *dpif, struct netdev *netdev, uint32_t mid)
>> +{
>> +    uint32_t port_id = netdev_dpdk_get_portid(netdev);
>> +    struct netdev_offload_meter *nom;
>> +    struct dpdk_meter_offload *dmo;
>> +    struct ofputil_meter_config config;
>> +    ofproto_meter_id meter_id;
>> +    struct rte_mtr_meter_profile mp;
>> +    struct rte_mtr_params params;
>> +    struct rte_mtr_error mtr_error;
>> +    uint32_t max_rate;
>> +    int ret;
>> +    static struct netdev_offload_meter_api dpdk_meter_offload_api;
>> +
>> +    meter_id.uint32 = mid;
>> +
>> +    if (dpif_meter_get_config(dpif, meter_id, &config)) {
>> +        return NULL;
>> +    }
>> +
>> +    nom = xmalloc(sizeof *nom);
>> +    dmo = xmalloc(sizeof *dmo);
>> +
>> +    nom->meter_ops = &dpdk_meter_offload_api;
>> +    nom->priv_data = dmo;
>> +
>> +    memset(&mp, 0, sizeof(struct rte_mtr_meter_profile));
>> +    max_rate = ofputil_meter_config_max_rate(&config);
>> +
>> +    dmo->mc.mtr_id = mid;
>> +    dmo->port_id = port_id;
>> +    dmo->max_rate = max_rate;
>> +    dmo->mp_id = mid;
>> +
>> +    mp.alg = RTE_MTR_SRTCM_RFC2697;
>> +    mp.srtcm_rfc2697.cir = max_rate *1024 /8; /* rate_max Kbps*/
>> +    mp.srtcm_rfc2697.cbs = max_rate *1024 /8;
>> +    mp.srtcm_rfc2697.ebs = 0;
>> +
>> +    ret = rte_mtr_meter_profile_add(dmo->port_id, dmo->mc.mtr_id,
>> +                                    &mp, &mtr_error);
>> +    if (ret && ret != -EEXIST) {
>> +        VLOG_ERR("rte_mtr_meter_profile_add fail: err_type: %d err_msg: %s, portid: %d\n",
>> +                   mtr_error.type, mtr_error.message, netdev_dpdk_get_portid(netdev));
>> +        goto profile_err;
>> +    }
>> +
>> +    enum rte_color dscp_table[2];
>> +    dscp_table[0] = RTE_COLOR_YELLOW;
>> +    dscp_table[1] = RTE_COLOR_RED;
>> +
>> +    params.meter_profile_id = dmo->mc.mtr_id;
>> +    params.dscp_table = dscp_table;
>> +    params.meter_enable = 1;
>> +    params.use_prev_mtr_color = 0;
>> +    params.action[RTE_COLOR_GREEN]  = MTR_POLICER_ACTION_COLOR_GREEN;
>> +    params.action[RTE_COLOR_YELLOW] = MTR_POLICER_ACTION_DROP;
>> +    params.action[RTE_COLOR_RED]    = MTR_POLICER_ACTION_DROP;
>> +
>> +    ret = rte_mtr_create(dmo->port_id, dmo->mc.mtr_id, &params, 0, &mtr_error);
>> +    if (ret && ret != -EEXIST) {
>> +        VLOG_ERR("rte_mtr_create fail: err_type: %d err_msg: %s, portid: %d\n",
>> +                   mtr_error.type, mtr_error.message, netdev_dpdk_get_portid(netdev));
>> +        goto mtr_err;
>> +    }
>> +
>> +    dpif_meter_set_offload(dpif, meter_id, nom);
>> +
>> +    free(config.bands);
>> +    return &dmo->mc;
>> +
>> +mtr_err:
>> +    rte_mtr_meter_profile_delete(dmo->port_id, dmo->mc.mtr_id, &mtr_error);
>> +
>> +profile_err:
>> +    free(nom);
>> +    free(dmo);
>> +    free(config.bands);
>> +    return NULL;
>> +}
>> +
>> +static struct rte_flow_action_meter *
>> +netdev_offload_dpdk_meter_conf(struct dpif *dpif, struct netdev *netdev, uint32_t mid)
>> +{
>> +    uint32_t port_id = netdev_dpdk_get_portid(netdev);
>> +    struct netdev_offload_meter *nom = NULL;
>> +    struct dpdk_meter_offload *dmo;
>> +    ofproto_meter_id meter_id;
>> +    uint32_t ret;
>> +    meter_id.uint32 = mid;
>> +
>> +    ret = dpif_meter_get_offload(dpif, meter_id, (void **)&nom,
>> +                                 sizeof *nom + sizeof *dmo);
>> +    if (ret) {
>> +        VLOG_INFO("netdev offload dpdk meter, can't get the meter");
>> +        return NULL;
>> +    }
>> +
>> +    if (!nom) {
>> +        return dpdk_meter_create(dpif, netdev, mid);
>> +    }
>> +
>> +    dmo = (struct dpdk_meter_offload *)nom->priv_data;
>> +    if (port_id != dmo->port_id) {
>> +        VLOG_INFO("dpdk meter %d is used on %d, can't be used for : %d",
>> +                  mid, dmo->port_id, port_id);
>> +        return NULL;
>> +    }
>> +
>> +    return &dmo->mc;
>> +}
>> +
>>  static int
>> -parse_flow_actions(struct netdev *netdev,
>> +parse_flow_actions(struct dpif *dpif,
>> +                   struct netdev *netdev,
>>                     struct flow_actions *actions,
>>                     struct nlattr *nl_actions,
>>                     size_t nl_actions_len,
>> @@ -1840,7 +1978,17 @@ parse_flow_actions(struct netdev *netdev,
>>              add_flow_action(actions, RTE_FLOW_ACTION_TYPE_JUMP, jump);
>>
>>          /* for ct && ct clear no need to translate*/
>> -        } else if (nl_attr_type(nla) == OVS_ACTION_ATTR_CT_CLEAR ||
>> +        } else if (nl_attr_type(nla) == OVS_ACTION_ATTR_METER) {
>> +            struct rte_flow_action_meter *mc;
>> +            mc = netdev_offload_dpdk_meter_conf(dpif,
>> +                                                netdev,
>> +                                                nl_attr_get_u32(nla));
>> +            if (mc) {
>> +                add_flow_action(actions, RTE_FLOW_ACTION_TYPE_METER, mc);
>> +            }
>> +
>> +        }
>> +          else if (nl_attr_type(nla) == OVS_ACTION_ATTR_CT_CLEAR ||
>>                     nl_attr_type(nla) == OVS_ACTION_ATTR_CT ) {
>>          } else {
>>              VLOG_DBG_RL(&rl, "Unsupported action type %d", nl_attr_type(nla));
>> @@ -1875,7 +2023,8 @@ parse_flow_actions(struct netdev *netdev,
>>  }
>>
>>  static struct rte_flow *
>> -netdev_offload_dpdk_actions(struct netdev *netdev,
>> +netdev_offload_dpdk_actions(struct dpif *dpif,
>> +                            struct netdev *netdev,
>>                              struct flow_patterns *patterns,
>>                              struct nlattr *nl_actions,
>>                              size_t actions_len,
>> @@ -1889,7 +2038,7 @@ netdev_offload_dpdk_actions(struct netdev *netdev,
>>      struct rte_flow_error error;
>>      int ret,i;
>>
>> -    ret = parse_flow_actions(netdev, &actions, nl_actions, actions_len, info);
>> +    ret = parse_flow_actions(dpif, netdev, &actions, nl_actions, actions_len, info);
>>      if (ret) {
>>          VLOG_DBG("TIMO DBG: after parse_flow_actions, ret is -1");
>>          goto out;
>> @@ -1927,13 +2076,79 @@ out:
>>      return flow;
>>  }
>>
>> +#define DPDK_METER_UPATE_UP 65536
>> +
>> +static void dpdk_meter_update(void *priv_data, void *config)
>> +{
>> +    struct dpdk_meter_offload *dmo = priv_data;
>> +    struct rte_mtr_meter_profile mp;
>> +    struct rte_mtr_error mtr_error;
>> +    uint32_t mp_id, new_mp_id;
>> +    uint32_t max_rate;
>> +    uint32_t ret;
>> +
>> +    if (!priv_data || !config) {
>> +        return;
>> +    }
>> +
>> +    max_rate = ofputil_meter_config_max_rate(config);
>> +    if (dmo->max_rate == max_rate) {
>> +        return;
>> +    }
>> +
>> +    memset(&mp, 0, sizeof(struct rte_mtr_meter_profile));
>> +    mp.alg = RTE_MTR_SRTCM_RFC2697;
>> +    mp.srtcm_rfc2697.cir = max_rate *1024 /8;
>> +    mp.srtcm_rfc2697.cbs = max_rate *1024 /8;
>> +    mp.srtcm_rfc2697.ebs = 0;
>> +
>> +    if (dmo->mp_id < DPDK_METER_UPATE_UP) {
>> +        new_mp_id = dmo->mp_id + DPDK_METER_UPATE_UP;
>> +    } else {
>> +        new_mp_id = dmo->mp_id - DPDK_METER_UPATE_UP;
>> +    }
>> +
>> +    ret = rte_mtr_meter_profile_add(dmo->port_id, new_mp_id,
>> +                                    &mp, &mtr_error);
>> +    if (ret) {
>> +        VLOG_ERR("rte_mtr_meter_profile_add fail: err_type: %d err_msg: %s\n",
>> +                 mtr_error.type, mtr_error.message);
>> +        return;
>> +    }
>> +
>> +    ret = rte_mtr_meter_profile_update(dmo->port_id, dmo->mc.mtr_id,
>> +                                       new_mp_id, &mtr_error);
>> +    if (ret) {
>> +        VLOG_ERR("rte_mtr_meter_profile_update fail: err_type: %d err_msg: %s\n",
>> +                 mtr_error.type, mtr_error.message);
>> +        mp_id = new_mp_id;
>> +        goto out;
>> +    }
>> +
>> +    mp_id = dmo->mp_id;
>> +    dmo->mp_id = new_mp_id;
>> +out:
>> +    ret = rte_mtr_meter_profile_delete(dmo->port_id, mp_id, &mtr_error);
>> +    if (ret) {
>> +        VLOG_ERR("rte_mtr_meter_profile_update fail: err_type: %d err_msg: %s\n",
>> +                 mtr_error.type, mtr_error.message);
>> +    }
>> +}
>> +
>> +static struct netdev_offload_meter_api dpdk_meter_offload_api = {
>> +    .meter_destroy  = dpdk_meter_destroy,
>> +    .meter_update   = dpdk_meter_update,
>> +};
>> +
>> +
>> +
>>  static int
>> -netdev_offload_dpdk_add_flow(struct netdev *netdev,
>> +netdev_offload_dpdk_add_flow(struct dpif *dpif, struct netdev *netdev,
>>                               struct match *match,
>>                               struct nlattr *nl_actions,
>>                               size_t actions_len,
>>                               const ovs_u128 *ufid,
>> -                             struct offload_info *info)
>> +                             struct offload_info *info OVS_UNUSED)
>>  {
>>      struct flow_patterns patterns = { .items = NULL, .cnt = 0 };
>>      bool actions_offloaded = true;
>> @@ -1943,10 +2158,12 @@ netdev_offload_dpdk_add_flow(struct netdev *netdev,
>>      ret = parse_flow_match(&patterns, match, netdev, info);
>>      if (ret) {
>>          goto out;
>> +        VLOG_DBG("yunxiang_debug_netdev_offload_dpdk_add_flow: ret %p \n",ret);
>>      }
>> -
>> -    flow = netdev_offload_dpdk_actions(netdev, &patterns, nl_actions,
>> +
>> +    flow = netdev_offload_dpdk_actions(dpif, netdev, &patterns, nl_actions,
>>                                         actions_len, info);
>> +    VLOG_DBG("yunxiang_debug_netdev_offload_dpdk_add_flow: flow %p \n",flow);
>>      if (!flow) {
>>          /* If we failed to offload the rule actions fallback to MARK+RSS
>>           * actions.
>> @@ -2075,9 +2292,10 @@ netdev_offload_dpdk_destroy_flow(struct netdev *netdev,
>>  }
>>
>>  static int
>> -netdev_offload_dpdk_flow_put(struct netdev *netdev, struct match *match,
>> -                             struct nlattr *actions, size_t actions_len,
>> -                             const ovs_u128 *ufid, struct offload_info *info,
>> +netdev_offload_dpdk_flow_put(struct dpif *dpif, struct netdev *netdev,
>> +                             struct match *match, struct nlattr *actions,
>> +                             size_t actions_len, const ovs_u128 *ufid,
>> +                             struct offload_info *info,
>>                               struct dpif_flow_stats *stats)
>>  {
>>      struct ufid_to_rte_flow_data *rte_flow_data;
>> @@ -2088,10 +2306,12 @@ netdev_offload_dpdk_flow_put(struct netdev *netdev, struct match *match,
>>       * Here destroy the old rte flow first before adding a new one.
>>       */
>>      rte_flow_data = ufid_to_rte_flow_data_find(ufid);
>> +    VLOG_DBG("yunxiang_debug_netdev_offload_dpdk_flow_put,rte_flow_data=%p",rte_flow_data);
>>      if (rte_flow_data && rte_flow_data->rte_flow) {
>>          ret = netdev_offload_dpdk_destroy_flow(netdev, ufid,
>>                                                 rte_flow_data->rte_flow);
>>          if (ret < 0) {
>> +            VLOG_DBG("yunxiang_debug_netdev_offload_dpdk_flow_put,ret=%p",ret);
>>              return ret;
>>          }
>>      }
>> @@ -2104,7 +2324,7 @@ netdev_offload_dpdk_flow_put(struct netdev *netdev, struct match *match,
>>      if (stats) {
>>          memset(stats, 0, sizeof *stats);
>>      }
>> -    return netdev_offload_dpdk_add_flow(netdev, match, actions,
>> +    return netdev_offload_dpdk_add_flow(dpif, netdev, match, actions,
>>                                          actions_len, ufid, info);
>>  }
>>
>> diff --git a/lib/netdev-offload-provider.h b/lib/netdev-offload-provider.h
>> index 5a809c0..f1830f8 100644
>> --- a/lib/netdev-offload-provider.h
>> +++ b/lib/netdev-offload-provider.h
>> @@ -63,9 +63,10 @@ struct netdev_flow_api {
>>       * 'stats' is populated according to the rules set out in the description
>>       * above 'struct dpif_flow_put'.
>>       * Return 0 if successful, otherwise returns a positive errno value. */
>> -    int (*flow_put)(struct netdev *, struct match *, struct nlattr *actions,
>> -                    size_t actions_len, const ovs_u128 *ufid,
>> -                    struct offload_info *info, struct dpif_flow_stats *);
>> +    int (*flow_put)(struct dpif *, struct netdev *, struct match *,
>> +                    struct nlattr *actions, size_t actions_len,
>> +                    const ovs_u128 *ufid, struct offload_info *info,
>> +                    struct dpif_flow_stats *);
>>
>>      /* Queries a flow specified by ufid on netdev.
>>       * Fills output buffer as 'wbuffer' in flow_dump_next, which
>> @@ -87,6 +88,16 @@ struct netdev_flow_api {
>>      int (*init_flow_api)(struct netdev *);
>>  };
>>
>> +struct netdev_offload_meter_api {
>> +    void (*meter_destroy)(void *);
>> +    void (*meter_update)(void *, void *);
>> +};
>> +
>> +struct netdev_offload_meter {
>> +    void *priv_data;
>> +    struct netdev_offload_meter_api *meter_ops;
>> +};
>> +
>>  int netdev_register_flow_api_provider(const struct netdev_flow_api *);
>>  int netdev_unregister_flow_api_provider(const char *type);
>>  bool netdev_flow_api_equals(const struct netdev *, const struct netdev *);
>> diff --git a/lib/netdev-offload-tc.c b/lib/netdev-offload-tc.c
>> index 6c76f69..ba4bd67 100644
>> --- a/lib/netdev-offload-tc.c
>> +++ b/lib/netdev-offload-tc.c
>> @@ -1354,9 +1354,10 @@ flower_match_to_tun_opt(struct tc_flower *flower, const struct flow_tnl *tnl,
>>  }
>>
>>  static int
>> -netdev_tc_flow_put(struct netdev *netdev, struct match *match,
>> -                   struct nlattr *actions, size_t actions_len,
>> -                   const ovs_u128 *ufid, struct offload_info *info,
>> +netdev_tc_flow_put(struct dpif *dpif OVS_UNUSED, struct netdev *netdev,
>> +                   struct match *match, struct nlattr *actions,
>> +                   size_t actions_len, const ovs_u128 *ufid,
>> +                   struct offload_info *info,
>>                     struct dpif_flow_stats *stats)
>>  {
>>      static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(5, 20);
>> diff --git a/lib/netdev-offload.c b/lib/netdev-offload.c
>> index e054bb0..135a29a 100644
>> --- a/lib/netdev-offload.c
>> +++ b/lib/netdev-offload.c
>> @@ -239,17 +239,18 @@ netdev_flow_dump_next(struct netdev_flow_dump *dump, struct match *match,
>>
>>
>>  int
>> -netdev_flow_put(struct netdev *netdev, struct match *match,
>> -                struct nlattr *actions, size_t act_len,
>> -                const ovs_u128 *ufid, struct offload_info *info,
>> +netdev_flow_put(struct dpif *dpif, struct netdev *netdev,
>> +                struct match *match, struct nlattr *actions,
>> +                size_t act_len, const ovs_u128 *ufid,
>> +                struct offload_info *info,
>>                  struct dpif_flow_stats *stats)
>>  {
>>      const struct netdev_flow_api *flow_api =
>>          ovsrcu_get(const struct netdev_flow_api *, &netdev->flow_api);
>>
>>      return (flow_api && flow_api->flow_put)
>> -           ? flow_api->flow_put(netdev, match, actions, act_len, ufid,
>> -                                info, stats)
>> +           ? flow_api->flow_put(dpif, netdev, match, actions,
>> +                                act_len, ufid, info, stats)
>>             : EOPNOTSUPP;
>>  }
>>
>> diff --git a/lib/netdev-offload.h b/lib/netdev-offload.h
>> index f2c4fbc..2d43059 100644
>> --- a/lib/netdev-offload.h
>> +++ b/lib/netdev-offload.h
>> @@ -20,8 +20,12 @@
>>
>>  #include "openvswitch/netdev.h"
>>  #include "openvswitch/types.h"
>> +#include "dpif-provider.h"
>>  #include "packets.h"
>>  #include "flow.h"
>> +
>> +
>> +
>>  #include "conntrack.h"
>>
>>  #ifdef  __cplusplus
>> @@ -62,6 +66,7 @@ struct netdev_flow_dump {
>>
>>  /* Flow offloading. */
>>  struct offload_info {
>> +    const struct dpif_class *dpif_class;
>>      ovs_be16 tp_dst_port; /* Destination port for tunnel in SET action */
>>      uint8_t tunnel_csum_on; /* Tunnel header with checksum */
>>
>> @@ -96,7 +101,8 @@ bool netdev_flow_dump_next(struct netdev_flow_dump *, struct match *,
>>                            struct nlattr **actions, struct dpif_flow_stats *,
>>                            struct dpif_flow_attrs *, ovs_u128 *ufid,
>>                            struct ofpbuf *rbuffer, struct ofpbuf *wbuffer);
>> -int netdev_flow_put(struct netdev *, struct match *, struct nlattr *actions,
>> +int netdev_flow_put(struct dpif *, struct netdev *,
>> +                    struct match *, struct nlattr *actions,
>>                      size_t actions_len, const ovs_u128 *,
>>                      struct offload_info *, struct dpif_flow_stats *);
>>  int netdev_flow_get(struct netdev *, struct match *, struct nlattr **actions,
>> diff --git a/lib/ofp-meter.c b/lib/ofp-meter.c
>> index 9ea40a0..633924e 100644
>> --- a/lib/ofp-meter.c
>> +++ b/lib/ofp-meter.c
>> @@ -807,3 +807,17 @@ ofputil_format_meter_mod(struct ds *s, const struct ofputil_meter_mod *mm)
>>
>>      ofputil_format_meter_config(s, &mm->meter);
>>  }
>> +
>> +uint32_t
>> +ofputil_meter_config_max_rate(struct ofputil_meter_config *conf)
>> +{
>> +    uint32_t i, max_rate = 0;
>> +    for (i = 0; i < conf->n_bands; i++) {
>> +        if (max_rate < conf->bands[i].rate) {
>> +            max_rate = conf->bands[i].rate;
>> +        }
>> +    }
>> +
>> +    return max_rate;
>> +}
>> +
>> --
>> 1.8.3.1
>>
>>
>>
>> _______________________________________________
>> dev mailing list
>> dev@openvswitch.org
>> https://mail.openvswitch.org/mailman/listinfo/ovs-dev
>
>
>
>--
>Best regards, Tonghao