diff --git a/lib/dpif-netdev.c b/lib/dpif-netdev.c
index 13fd012..ef5c2e9 100644
--- a/lib/dpif-netdev.c
+++ b/lib/dpif-netdev.c
@@ -1949,6 +1949,223 @@ dp_netdev_pmd_find_flow_by_mark(const uint32_t mark)
     return NULL;
 }
 
+struct dp_flow_offload_item {
+    struct dp_netdev_pmd_thread *pmd;
+    odp_port_t in_port;
+    struct dp_netdev_flow *flow;
+    ovs_u128 ufid;
+    int op;
+    struct match match;
+    struct nlattr *actions;
+    size_t actions_len;
+    int rxq;
+    uint32_t flow_mark;
+
+    struct ovs_list node;
+};
+
+struct dp_flow_offload {
+    struct ovs_mutex mutex;
+    struct ovs_list list;
+    pthread_cond_t cond;
+};
+
+static struct dp_flow_offload dp_flow_offload = {
+    .mutex = OVS_MUTEX_INITIALIZER,
+    .list  = OVS_LIST_INITIALIZER(&dp_flow_offload.list),
+};
+
+static struct ovsthread_once offload_thread_once
+    = OVSTHREAD_ONCE_INITIALIZER;
+
+enum {
+    DP_NETDEV_FLOW_OFFLOAD_OP_ADD,
+    DP_NETDEV_FLOW_OFFLOAD_OP_MOD,
+    DP_NETDEV_FLOW_OFFLOAD_OP_DEL,
+};
+
+static struct dp_flow_offload_item *
+dp_netdev_alloc_flow_offload(struct dp_netdev_pmd_thread *pmd,
+                                  struct dp_netdev_flow *flow,
+                                  odp_port_t in_port, const ovs_u128 *ufid,
+                                  int op)
+{
+    struct dp_flow_offload_item *offload;
+
+    offload = xzalloc(sizeof(*offload));
+    offload->pmd = pmd;
+    offload->flow = flow;
+    offload->in_port = in_port;
+    offload->ufid = *ufid;
+    offload->op = op;
+
+    ovs_refcount_ref(&pmd->ref_cnt);
+    ovs_refcount_ref(&flow->ref_cnt);
+
+    return offload;
+}
+
+static void
+dp_netdev_free_flow_offload(struct dp_flow_offload_item *offload)
+{
+    ovs_refcount_unref(&offload->pmd->ref_cnt);
+    ovs_refcount_unref(&offload->flow->ref_cnt);
+
+    free(offload->actions);
+    free(offload);
+}
+
+static void
+dp_netdev_append_flow_offload(struct dp_flow_offload_item *offload)
+{
+    ovs_mutex_lock(&dp_flow_offload.mutex);
+    ovs_list_push_back(&dp_flow_offload.list, &offload->node);
+    ovs_mutex_unlock(&dp_flow_offload.mutex);
+
+    pthread_cond_signal(&dp_flow_offload.cond);
+}
+
+static int
+dp_netdev_flow_offload_del(struct dp_flow_offload_item *offload)
+{
+    struct dp_netdev_flow *flow = offload->flow;
+    struct dp_netdev_port *port;
+    int ret = -1;
+
+    ovs_mutex_lock(&offload->pmd->flow_mutex);
+    port = dp_netdev_lookup_port(offload->pmd->dp, offload->in_port);
+    if (flow->has_mark && port) {
+        ret = netdev_flow_del(port->netdev, &offload->ufid, NULL);
+    }
+
+    dp_netdev_remove_flow_mark_map(flow->mark);
+    ovsrcu_quiesce_start();
+    flow->has_mark = false;
+    ovs_mutex_unlock(&offload->pmd->flow_mutex);
+
+    return ret;
+}
+
+static int
+dp_netdev_flow_offload_put(struct dp_flow_offload_item *offload)
+{
+    struct dp_netdev_port *port;
+    struct dp_netdev_flow *flow = offload->flow;
+    bool create = offload->op == DP_NETDEV_FLOW_OFFLOAD_OP_ADD;
+    struct offload_info info;
+    int ret;
+
+    port = dp_netdev_lookup_port(offload->pmd->dp, offload->in_port);
+    if (!port) {
+        return -1;
+    }
+
+    if (create) {
+        if (!dp_netdev_alloc_flow_mark(&info.flow_mark)) {
+            VLOG_ERR("failed to allocate flow mark!\n");
+            return -1;
+        }
+        offload->flow_mark = info.flow_mark;
+    } else {
+        info.flow_mark = offload->flow_mark;
+    }
+    info.rxq = offload->rxq;
+
+    ret = netdev_flow_put(port->netdev, &offload->match,
+                          offload->actions, offload->actions_len,
+                          &offload->ufid, &info, NULL);
+    if (ret) {
+        if (create) {
+            dp_netdev_remove_flow_mark_map(info.flow_mark);
+        }
+        return ret;
+    }
+
+    ovs_mutex_lock(&offload->pmd->flow_mutex);
+    if (create) {
+        flow->has_mark = true;
+        flow->mark = info.flow_mark;
+        if (!flow->dead) {
+            /*
+             * A flow could have been dead after we regain the lock,
+             * while the flow has offloaded to the netdev. When that
+             * happens, there should be an offload item in the offload
+             * list for the flow removal. To make sure the flow will
+             * be deleted successfully later, above 2 fields (has_mark
+             * and mark) have to be set properly.
+             */
+            dp_netdev_install_flow_mark_map(flow->mark, flow);
+            ovsrcu_quiesce_start();
+        }
+    }
+    ovs_mutex_unlock(&offload->pmd->flow_mutex);
+
+    return 0;
+}
+
+static void *
+dp_netdev_flow_offload_main(void *data OVS_UNUSED)
+{
+    struct dp_flow_offload_item *offload;
+    struct ovs_list *list;
+    const char *op;
+    int ret;
+
+    for (;;) {
+        ovs_mutex_lock(&dp_flow_offload.mutex);
+        if (ovs_list_is_empty(&dp_flow_offload.list)) {
+            pthread_cond_wait(&dp_flow_offload.cond,
+                              &dp_flow_offload.mutex.lock);
+        }
+        list = ovs_list_pop_front(&dp_flow_offload.list);
+        offload = CONTAINER_OF(list, struct dp_flow_offload_item, node);
+        ovs_mutex_unlock(&dp_flow_offload.mutex);
+
+        switch (offload->op) {
+        case DP_NETDEV_FLOW_OFFLOAD_OP_ADD:
+            op = "install";
+            ret = dp_netdev_flow_offload_put(offload);
+            break;
+        case DP_NETDEV_FLOW_OFFLOAD_OP_MOD:
+            op = "modify";
+            ret = dp_netdev_flow_offload_put(offload);
+            break;
+        case DP_NETDEV_FLOW_OFFLOAD_OP_DEL:
+            op = "delete";
+            ret = dp_netdev_flow_offload_del(offload);
+            break;
+        default:
+            OVS_NOT_REACHED();
+        }
+
+        VLOG_INFO("%s to %s netdev flow with mark %u\n",
+                  ret == 0 ? "succeed" : "failed",
+                  op, offload->flow_mark);
+        dp_netdev_free_flow_offload(offload);
+    }
+
+    return NULL;
+}
+
+static void
+queue_netdev_flow_del(struct dp_netdev_pmd_thread *pmd,
+                      struct dp_netdev_flow *flow,
+                      odp_port_t in_port, const ovs_u128 *ufid,
+                      int op)
+{
+    struct dp_flow_offload_item *offload;
+
+    if (ovsthread_once_start(&offload_thread_once)) {
+        pthread_cond_init(&dp_flow_offload.cond, NULL);
+        ovs_thread_create("dp_netdev_flow_offload",
+                          dp_netdev_flow_offload_main, NULL);
+        ovsthread_once_done(&offload_thread_once);
+    }
+
+    offload = dp_netdev_alloc_flow_offload(pmd, flow, in_port, ufid, op);
+    offload->flow_mark = flow->mark;
+    dp_netdev_append_flow_offload(offload);
+}
 
 static void
 dp_netdev_pmd_remove_flow(struct dp_netdev_pmd_thread *pmd,
@@ -1963,16 +2180,8 @@ dp_netdev_pmd_remove_flow(struct dp_netdev_pmd_thread *pmd,
     ovs_assert(cls != NULL);
     dpcls_remove(cls, &flow->cr);
     cmap_remove(&pmd->flow_table, node, dp_netdev_flow_hash(&flow->ufid));
-    if (flow->has_mark) {
-        struct dp_netdev_port *port;
-
-        port = dp_netdev_lookup_port(pmd->dp, in_port);
-        if (port) {
-            netdev_flow_del(port->netdev, &flow->ufid, NULL);
-        }
-        dp_netdev_remove_flow_mark_map(flow->mark);
-        flow->has_mark = false;
-    }
+    queue_netdev_flow_del(pmd, flow, in_port, &flow->ufid,
+                          DP_NETDEV_FLOW_OFFLOAD_OP_DEL);
     flow->dead = true;
 
     dp_netdev_flow_unref(flow);
@@ -2553,53 +2762,41 @@ out:
 }
 
 static void
-try_netdev_flow_put(struct dp_netdev_pmd_thread *pmd, odp_port_t in_port,
-                    struct dp_netdev_flow *flow, struct match *match,
-                    const ovs_u128 *ufid, const struct nlattr *actions,
-                    size_t actions_len, int rxq)
+queue_netdev_flow_put(struct dp_netdev_pmd_thread *pmd, odp_port_t in_port,
+                      struct dp_netdev_flow *flow, struct match *match,
+                      const ovs_u128 *ufid, const struct nlattr *actions,
+                      size_t actions_len, int rxq)
 {
-    struct offload_info info;
-    struct dp_netdev_port *port;
-    bool modification = flow->has_mark;
-    const char *op = modification ? "modify" : "install";
-    int ret;
+    struct dp_flow_offload_item *offload;
+    int op;
 
-    port = dp_netdev_lookup_port(pmd->dp, in_port);
-    if (!port) {
+    if (!netdev_is_flow_api_enabled()) {
         return;
     }
-    info.rxq = rxq;
-
-    if (modification) {
-        info.flow_mark = flow->mark;
-    } else {
-        if (!netdev_is_flow_api_enabled()) {
-            return;
-        }
-
-        if (!dp_netdev_alloc_flow_mark(&info.flow_mark)) {
-            VLOG_ERR("failed to allocate flow mark!\n");
-            return;
-        }
-    }
 
-    ret = netdev_flow_put(port->netdev, match,
-                          CONST_CAST(struct nlattr *, actions),
-                          actions_len, ufid, &info, NULL);
-    if (ret) {
-        VLOG_ERR("failed to %s netdev flow with mark %u\n",
-                 op, info.flow_mark);
-        return;
+    if (ovsthread_once_start(&offload_thread_once)) {
+        pthread_cond_init(&dp_flow_offload.cond, NULL);
+        ovs_thread_create("dp_netdev_flow_offload",
+                          dp_netdev_flow_offload_main, NULL);
+        ovsthread_once_done(&offload_thread_once);
     }
 
-    if (!modification) {
-        flow->has_mark = true;
-        flow->mark = info.flow_mark;
-        dp_netdev_install_flow_mark_map(info.flow_mark, flow);
+    if (flow->has_mark) {
+        op = DP_NETDEV_FLOW_OFFLOAD_OP_MOD;
+    } else {
+        op = DP_NETDEV_FLOW_OFFLOAD_OP_ADD;
+    }
+    offload = dp_netdev_alloc_flow_offload(pmd, flow, in_port, ufid, op);
+    offload->match = *match;
+    offload->rxq = rxq;
+    offload->actions = xmalloc(actions_len);
+    memcpy(offload->actions, actions, actions_len);
+    offload->actions_len = actions_len;
+    if (flow->has_mark) {
+        offload->flow_mark = flow->mark;
     }
 
-    VLOG_INFO("succeed to %s netdev flow with mark %u\n",
-              op, flow->mark);
+    dp_netdev_append_flow_offload(offload);
 }
 
 static struct dp_netdev_flow *
@@ -2654,8 +2851,8 @@ dp_netdev_flow_add(struct dp_netdev_pmd_thread *pmd,
     cmap_insert(&pmd->flow_table, CONST_CAST(struct cmap_node *, &flow->node),
                 dp_netdev_flow_hash(&flow->ufid));
 
-    try_netdev_flow_put(pmd, in_port, flow, match, ufid,
-                        actions, actions_len, rxq);
+    queue_netdev_flow_put(pmd, in_port, flow, match, ufid,
+                          actions, actions_len, rxq);
 
     if (OVS_UNLIKELY(!VLOG_DROP_DBG((&upcall_rl)))) {
         struct ds ds = DS_EMPTY_INITIALIZER;
@@ -2745,8 +2942,8 @@ flow_put_on_pmd(struct dp_netdev_pmd_thread *pmd,
             old_actions = dp_netdev_flow_get_actions(netdev_flow);
             ovsrcu_set(&netdev_flow->actions, new_actions);
 
-            try_netdev_flow_put(pmd, in_port, netdev_flow, match, ufid,
-                                put->actions, put->actions_len, -1);
+            queue_netdev_flow_put(pmd, in_port, netdev_flow, match, ufid,
+                                  put->actions, put->actions_len, -1);
 
             if (stats) {
                 get_dpif_flow_stats(netdev_flow, stats);
