diff --git a/lib/dpif-netdev.c b/lib/dpif-netdev.c
index e129fa3..899e0bd 100644
--- a/lib/dpif-netdev.c
+++ b/lib/dpif-netdev.c
@@ -1979,10 +1979,11 @@ is_last_flow_mark_reference(uint32_t mark)
     return true;
 }
 
-static void
+static int
 mark_to_flow_disassociate(struct dp_netdev_pmd_thread *pmd,
                           struct dp_netdev_flow *flow)
 {
+    int ret = 0;
     uint32_t mark = flow->mark;
     struct cmap_node *mark_node = CONST_CAST(struct cmap_node *,
                                              &flow->mark_node);
@@ -2002,7 +2003,7 @@ mark_to_flow_disassociate(struct dp_netdev_pmd_thread *pmd,
 
         port = dp_netdev_lookup_port(pmd->dp, in_port);
         if (port) {
-            netdev_flow_del(port->netdev, &flow->mega_ufid, NULL);
+            ret = netdev_flow_del(port->netdev, &flow->mega_ufid, NULL);
         }
 
         ovs_mutex_lock(&flow_mark.mutex);
@@ -2013,8 +2014,12 @@ mark_to_flow_disassociate(struct dp_netdev_pmd_thread *pmd,
         megaflow_to_mark_disassociate(&flow->mega_ufid);
     }
     dp_netdev_flow_unref(flow);
+
+    return ret;
 }
 
+static void queue_netdev_flow_del(struct dp_netdev_pmd_thread *pmd,
+                                  struct dp_netdev_flow *flow);
 static void
 flow_mark_flush(struct dp_netdev_pmd_thread *pmd)
 {
@@ -2022,7 +2027,7 @@ flow_mark_flush(struct dp_netdev_pmd_thread *pmd)
 
     CMAP_FOR_EACH (flow, mark_node, &flow_mark.mark_to_flow) {
         if (flow->pmd_id == pmd->core_id) {
-            mark_to_flow_disassociate(pmd, flow);
+            queue_netdev_flow_del(pmd, flow);
         }
     }
 }
@@ -2042,6 +2047,242 @@ mark_to_flow_find(const struct dp_netdev_pmd_thread *pmd,
     return NULL;
 }
 
+enum {
+    DP_NETDEV_FLOW_OFFLOAD_OP_ADD,
+    DP_NETDEV_FLOW_OFFLOAD_OP_MOD,
+    DP_NETDEV_FLOW_OFFLOAD_OP_DEL,
+};
+
+struct dp_flow_offload_item {
+    struct dp_netdev_pmd_thread *pmd;
+    struct dp_netdev_flow *flow;
+    int op;
+    struct match match;
+    struct nlattr *actions;
+    size_t actions_len;
+
+    struct ovs_list node;
+};
+
+struct dp_flow_offload {
+    struct ovs_mutex mutex;
+    struct ovs_list list;
+    pthread_cond_t cond;
+};
+
+static struct dp_flow_offload dp_flow_offload = {
+    .mutex = OVS_MUTEX_INITIALIZER,
+    .list  = OVS_LIST_INITIALIZER(&dp_flow_offload.list),
+};
+
+static struct ovsthread_once offload_thread_once
+    = OVSTHREAD_ONCE_INITIALIZER;
+
+static struct dp_flow_offload_item *
+dp_netdev_alloc_flow_offload(struct dp_netdev_pmd_thread *pmd,
+                             struct dp_netdev_flow *flow,
+                             int op)
+{
+    struct dp_flow_offload_item *offload;
+
+    offload = xzalloc(sizeof(*offload));
+    offload->pmd = pmd;
+    offload->flow = flow;
+    offload->op = op;
+
+    dp_netdev_flow_ref(flow);
+    dp_netdev_pmd_try_ref(pmd);
+
+    return offload;
+}
+
+static void
+dp_netdev_free_flow_offload(struct dp_flow_offload_item *offload)
+{
+    dp_netdev_pmd_unref(offload->pmd);
+    dp_netdev_flow_unref(offload->flow);
+
+    free(offload->actions);
+    free(offload);
+}
+
+static void
+dp_netdev_append_flow_offload(struct dp_flow_offload_item *offload)
+{
+    ovs_mutex_lock(&dp_flow_offload.mutex);
+    ovs_list_push_back(&dp_flow_offload.list, &offload->node);
+    ovs_mutex_unlock(&dp_flow_offload.mutex);
+
+    pthread_cond_signal(&dp_flow_offload.cond);
+}
+
+static int
+dp_netdev_flow_offload_del(struct dp_flow_offload_item *offload)
+{
+    return mark_to_flow_disassociate(offload->pmd, offload->flow);
+}
+
+static int
+dp_netdev_flow_offload_put(struct dp_flow_offload_item *offload)
+{
+    struct dp_netdev_port *port;
+    struct dp_netdev_flow *flow = offload->flow;
+    odp_port_t in_port = flow->flow.in_port.odp_port;
+    bool modification = offload->op == DP_NETDEV_FLOW_OFFLOAD_OP_MOD;
+    struct offload_info info;
+    uint32_t mark;
+    int ret;
+
+    port = dp_netdev_lookup_port(offload->pmd->dp, in_port);
+    if (!port) {
+        return -1;
+    }
+
+    VLOG_INFO(" ");
+    VLOG_INFO(":: about to offload:\n");
+    VLOG_INFO("   ufid: "UUID_FMT"\n",
+              UUID_ARGS((struct uuid *)&flow->ufid));
+    VLOG_INFO("   mask: "UUID_FMT"\n",
+              UUID_ARGS((struct uuid *)&flow->mega_ufid));
+
+    if (modification) {
+        mark = flow->mark;
+    } else {
+        /*
+         * If a mega flow has already been offloaded (from other PMD
+         * instances), do not offload it again.
+         */
+        mark = megaflow_to_mark_find(&flow->mega_ufid);
+        if (mark != INVALID_FLOW_MARK) {
+            VLOG_INFO("## got a previously installed mark %u\n", mark);
+            if (!mark_to_flow_find(offload->pmd, mark)) {
+                mark_to_flow_associate(mark, flow);
+            } else {
+                VLOG_INFO("## mark is already installed\n");
+            }
+            return 0;
+        }
+
+        mark = flow_mark_alloc();
+        if (mark == INVALID_FLOW_MARK) {
+            VLOG_ERR("failed to allocate flow mark!\n");
+        }
+    }
+
+    info.flow_mark = mark;
+    ret = netdev_flow_put(port->netdev, &offload->match,
+                          CONST_CAST(struct nlattr *, offload->actions),
+                          offload->actions_len, &flow->mega_ufid, &info,
+                          NULL);
+    if (ret) {
+        flow_mark_free(mark);
+        return -1;
+    }
+
+    if (!modification) {
+        megaflow_to_mark_associate(&flow->mega_ufid, mark);
+        mark_to_flow_associate(mark, flow);
+    }
+
+    return 0;
+}
+
+static void *
+dp_netdev_flow_offload_main(void *data OVS_UNUSED)
+{
+    struct dp_flow_offload_item *offload;
+    struct ovs_list *list;
+    const char *op;
+    int ret;
+
+    for (;;) {
+        ovs_mutex_lock(&dp_flow_offload.mutex);
+        if (ovs_list_is_empty(&dp_flow_offload.list)) {
+            pthread_cond_wait(&dp_flow_offload.cond,
+                              &dp_flow_offload.mutex.lock);
+        }
+        list = ovs_list_pop_front(&dp_flow_offload.list);
+        offload = CONTAINER_OF(list, struct dp_flow_offload_item, node);
+        ovs_mutex_unlock(&dp_flow_offload.mutex);
+
+        switch (offload->op) {
+        case DP_NETDEV_FLOW_OFFLOAD_OP_ADD:
+            op = "add";
+            ret = dp_netdev_flow_offload_put(offload);
+            break;
+        case DP_NETDEV_FLOW_OFFLOAD_OP_MOD:
+            op = "modify";
+            ret = dp_netdev_flow_offload_put(offload);
+            break;
+        case DP_NETDEV_FLOW_OFFLOAD_OP_DEL:
+            op = "delete";
+            ret = dp_netdev_flow_offload_del(offload);
+            break;
+        default:
+            OVS_NOT_REACHED();
+        }
+
+        VLOG_INFO("%s to %s netdev flow with mark %u\n",
+                  ret == 0 ? "succeed" : "failed",
+                  op, offload->flow->mark);
+        dp_netdev_free_flow_offload(offload);
+        ovsrcu_quiesce_start();
+    }
+
+    return NULL;
+}
+
+static void
+queue_netdev_flow_del(struct dp_netdev_pmd_thread *pmd,
+                      struct dp_netdev_flow *flow)
+{
+    struct dp_flow_offload_item *offload;
+
+    if (ovsthread_once_start(&offload_thread_once)) {
+        pthread_cond_init(&dp_flow_offload.cond, NULL);
+        ovs_thread_create("dp_netdev_flow_offload",
+                          dp_netdev_flow_offload_main, NULL);
+        ovsthread_once_done(&offload_thread_once);
+    }
+
+    offload = dp_netdev_alloc_flow_offload(pmd, flow,
+                                           DP_NETDEV_FLOW_OFFLOAD_OP_DEL);
+    dp_netdev_append_flow_offload(offload);
+}
+
+static void
+queue_netdev_flow_put(struct dp_netdev_pmd_thread *pmd,
+                      struct dp_netdev_flow *flow, struct match *match,
+                      const struct nlattr *actions, size_t actions_len)
+{
+    struct dp_flow_offload_item *offload;
+    int op;
+
+    if (!netdev_is_flow_api_enabled()) {
+        return;
+    }
+
+    if (ovsthread_once_start(&offload_thread_once)) {
+        pthread_cond_init(&dp_flow_offload.cond, NULL);
+        ovs_thread_create("dp_netdev_flow_offload",
+                          dp_netdev_flow_offload_main, NULL);
+        ovsthread_once_done(&offload_thread_once);
+    }
+
+    if (flow->mark != INVALID_FLOW_MARK) {
+        op = DP_NETDEV_FLOW_OFFLOAD_OP_MOD;
+    } else {
+        op = DP_NETDEV_FLOW_OFFLOAD_OP_ADD;
+    }
+    offload = dp_netdev_alloc_flow_offload(pmd, flow, op);
+    offload->match = *match;
+    offload->actions = xmalloc(actions_len);
+    memcpy(offload->actions, actions, actions_len);
+    offload->actions_len = actions_len;
+
+    dp_netdev_append_flow_offload(offload);
+}
+
 static void
 dp_netdev_pmd_remove_flow(struct dp_netdev_pmd_thread *pmd,
                           struct dp_netdev_flow *flow)
@@ -2056,7 +2297,7 @@ dp_netdev_pmd_remove_flow(struct dp_netdev_pmd_thread *pmd,
     dpcls_remove(cls, &flow->cr);
     cmap_remove(&pmd->flow_table, node, dp_netdev_flow_hash(&flow->ufid));
     if (flow->mark != INVALID_FLOW_MARK) {
-        mark_to_flow_disassociate(pmd, flow);
+        queue_netdev_flow_del(pmd, flow);
     }
     flow->dead = true;
 
@@ -2638,78 +2879,6 @@ out:
 }
 
 static void
-try_netdev_flow_put(struct dp_netdev_pmd_thread *pmd, odp_port_t in_port,
-                    struct dp_netdev_flow *flow, struct match *match,
-                    const ovs_u128 *ufid, const struct nlattr *actions,
-                    size_t actions_len)
-{
-    struct offload_info info;
-    struct dp_netdev_port *port;
-    bool modification = flow->mark != INVALID_FLOW_MARK;
-    const char *op = modification ? "modify" : "add";
-    uint32_t mark;
-    int ret;
-
-    port = dp_netdev_lookup_port(pmd->dp, in_port);
-    if (!port) {
-        return;
-    }
-
-    ovs_mutex_lock(&flow_mark.mutex);
-
-    VLOG_INFO(" ");
-    VLOG_INFO(":: about to offload:\n");
-    VLOG_INFO("   ufid: "UUID_FMT"\n",
-              UUID_ARGS((struct uuid *)ufid));
-    VLOG_INFO("   mask: "UUID_FMT"\n",
-              UUID_ARGS((struct uuid *)&flow->mega_ufid));
-
-    if (modification) {
-        mark = flow->mark;
-    } else {
-        if (!netdev_is_flow_api_enabled()) {
-            goto out;
-        }
-
-        /*
-         * If a mega flow has already been offloaded (from other PMD
-         * instances), do not offload it again.
-         */
-        mark = megaflow_to_mark_find(&flow->mega_ufid);
-        if (mark != INVALID_FLOW_MARK) {
-            VLOG_INFO("## got a previously installed mark %u\n", mark);
-            mark_to_flow_associate(mark, flow);
-            goto out;
-        }
-
-        mark = flow_mark_alloc();
-        if (mark == INVALID_FLOW_MARK) {
-            VLOG_ERR("failed to allocate flow mark!\n");
-            goto out;
-        }
-    }
-
-    info.flow_mark = mark;
-    ret = netdev_flow_put(port->netdev, match,
-                          CONST_CAST(struct nlattr *, actions),
-                          actions_len, &flow->mega_ufid, &info, NULL);
-    if (ret) {
-        VLOG_ERR("failed to %s netdev flow with mark %u\n", op, mark);
-        flow_mark_free(mark);
-        goto out;
-    }
-
-    if (!modification) {
-        megaflow_to_mark_associate(&flow->mega_ufid, mark);
-        mark_to_flow_associate(mark, flow);
-    }
-    VLOG_INFO("succeed to %s netdev flow with mark %u\n", op, mark);
-
-out:
-    ovs_mutex_unlock(&flow_mark.mutex);
-}
-
-static void
 dp_netdev_get_mega_ufid(const struct match *match, ovs_u128 *mega_ufid)
 {
     struct flow masked_flow;
@@ -2775,8 +2944,7 @@ dp_netdev_flow_add(struct dp_netdev_pmd_thread *pmd,
     cmap_insert(&pmd->flow_table, CONST_CAST(struct cmap_node *, &flow->node),
                 dp_netdev_flow_hash(&flow->ufid));
 
-    try_netdev_flow_put(pmd, in_port, flow, match, ufid,
-                        actions, actions_len);
+    queue_netdev_flow_put(pmd, flow, match, actions, actions_len);
 
     if (OVS_UNLIKELY(!VLOG_DROP_DBG((&upcall_rl)))) {
         struct ds ds = DS_EMPTY_INITIALIZER;
@@ -2858,7 +3026,6 @@ flow_put_on_pmd(struct dp_netdev_pmd_thread *pmd,
         if (put->flags & DPIF_FP_MODIFY) {
             struct dp_netdev_actions *new_actions;
             struct dp_netdev_actions *old_actions;
-            odp_port_t in_port = netdev_flow->flow.in_port.odp_port;
 
             new_actions = dp_netdev_actions_create(put->actions,
                                                    put->actions_len);
@@ -2866,8 +3033,8 @@ flow_put_on_pmd(struct dp_netdev_pmd_thread *pmd,
             old_actions = dp_netdev_flow_get_actions(netdev_flow);
             ovsrcu_set(&netdev_flow->actions, new_actions);
 
-            try_netdev_flow_put(pmd, in_port, netdev_flow, match, ufid,
-                                put->actions, put->actions_len);
+            queue_netdev_flow_put(pmd, netdev_flow, match,
+                                  put->actions, put->actions_len);
 
             if (stats) {
                 get_dpif_flow_stats(netdev_flow, stats);
