| Message ID | 1556626682-28858-2-git-send-email-david.marchand@redhat.com |
|---|---|
| State | Changes Requested |
| Headers | show |
| Series | Quicker pmd threads reloads | expand |
On 30.04.2019 15:17, David Marchand wrote: > No need for a latch here since we don't have to wait. > A simple boolean flag is enough. > > Fixes: e4cfed38b159 ("dpif-netdev: Add poll-mode-device thread.") > Signed-off-by: David Marchand <david.marchand@redhat.com> > --- > lib/dpif-netdev.c | 9 ++++----- > 1 file changed, 4 insertions(+), 5 deletions(-) > > diff --git a/lib/dpif-netdev.c b/lib/dpif-netdev.c > index f1422b2..30774ed 100644 > --- a/lib/dpif-netdev.c > +++ b/lib/dpif-netdev.c > @@ -681,10 +681,10 @@ struct dp_netdev_pmd_thread { > /* Current context of the PMD thread. */ > struct dp_netdev_pmd_thread_ctx ctx; > > - struct latch exit_latch; /* For terminating the pmd thread. */ > struct seq *reload_seq; > uint64_t last_reload_seq; > atomic_bool reload; /* Do we need to reload ports? */ > + atomic_bool exit; /* For terminating the pmd thread. */ > pthread_t thread; > unsigned core_id; /* CPU core id of this pmd thread. */ > int numa_id; /* numa node id of this pmd thread. */ > @@ -5479,7 +5479,7 @@ reload: > ovs_mutex_unlock(&pmd->perf_stats.stats_mutex); > > poll_cnt = pmd_load_queues_and_ports(pmd, &poll_list); > - exiting = latch_is_set(&pmd->exit_latch); > + atomic_read_relaxed(&pmd->exit, &exiting); I'm afraid that relaxed memory model is not suitable here. You need to change memory models for both 'reload' and 'exit' or put ack_rel thread fence between them. Otherwise reads/writes could be reordered resulting in missed exit and main thread hang on join. > /* Signal here to make sure the pmd finishes > * reloading the updated configuration. */ > dp_netdev_pmd_reload_done(pmd); > @@ -5898,7 +5898,7 @@ dp_netdev_configure_pmd(struct dp_netdev_pmd_thread *pmd, struct dp_netdev *dp, > pmd->n_output_batches = 0; > > ovs_refcount_init(&pmd->ref_cnt); > - latch_init(&pmd->exit_latch); > + atomic_init(&pmd->exit, false); > pmd->reload_seq = seq_create(); > pmd->last_reload_seq = seq_read(pmd->reload_seq); > atomic_init(&pmd->reload, false); > @@ -5945,7 +5945,6 @@ dp_netdev_destroy_pmd(struct dp_netdev_pmd_thread *pmd) > cmap_destroy(&pmd->classifiers); > cmap_destroy(&pmd->flow_table); > ovs_mutex_destroy(&pmd->flow_mutex); > - latch_destroy(&pmd->exit_latch); > seq_destroy(pmd->reload_seq); > xpthread_cond_destroy(&pmd->cond); > ovs_mutex_destroy(&pmd->cond_mutex); > @@ -5967,7 +5966,7 @@ dp_netdev_del_pmd(struct dp_netdev *dp, struct dp_netdev_pmd_thread *pmd) > pmd_free_static_tx_qid(pmd); > ovs_mutex_unlock(&dp->non_pmd_mutex); > } else { > - latch_set(&pmd->exit_latch); > + atomic_store_relaxed(&pmd->exit, true); > dp_netdev_reload_pmd__(pmd); > xpthread_join(pmd->thread, NULL); > } >
Hello Ilya, On Mon, May 6, 2019 at 5:37 PM Ilya Maximets <i.maximets@samsung.com> wrote: > On 30.04.2019 15:17, David Marchand wrote: > > No need for a latch here since we don't have to wait. > > A simple boolean flag is enough. > > > > Fixes: e4cfed38b159 ("dpif-netdev: Add poll-mode-device thread.") > > Signed-off-by: David Marchand <david.marchand@redhat.com> > > --- > > lib/dpif-netdev.c | 9 ++++----- > > 1 file changed, 4 insertions(+), 5 deletions(-) > > > > diff --git a/lib/dpif-netdev.c b/lib/dpif-netdev.c > > index f1422b2..30774ed 100644 > > --- a/lib/dpif-netdev.c > > +++ b/lib/dpif-netdev.c > > @@ -681,10 +681,10 @@ struct dp_netdev_pmd_thread { > > /* Current context of the PMD thread. */ > > struct dp_netdev_pmd_thread_ctx ctx; > > > > - struct latch exit_latch; /* For terminating the pmd thread. > */ > > struct seq *reload_seq; > > uint64_t last_reload_seq; > > atomic_bool reload; /* Do we need to reload ports? */ > > + atomic_bool exit; /* For terminating the pmd thread. > */ > > pthread_t thread; > > unsigned core_id; /* CPU core id of this pmd thread. > */ > > int numa_id; /* numa node id of this pmd thread. > */ > > @@ -5479,7 +5479,7 @@ reload: > > ovs_mutex_unlock(&pmd->perf_stats.stats_mutex); > > > > poll_cnt = pmd_load_queues_and_ports(pmd, &poll_list); > > - exiting = latch_is_set(&pmd->exit_latch); > > + atomic_read_relaxed(&pmd->exit, &exiting); > > I'm afraid that relaxed memory model is not suitable here. > You need to change memory models for both 'reload' and 'exit' > or put ack_rel thread fence between them. Otherwise reads/writes > could be reordered resulting in missed exit and main thread > hang on join. > Indeed, I can not use relaxed memory model on both atomics. I have been reading some articles and I am a bit puzzled :-). I don't understand why I would need to update both atomics memory model. On the pmd thread side: atomic_read_explicit(&pmd->reload, &reload, memory_model_acquire); atomic_read_relaxed(&pmd->exit, &exiting); On the control side: atomic_store_relaxed(&pmd->exit, true); atomic_store_explicit(&pmd->reload, true, memory_order_release); Would not it be enough to have those threads share the same view by synchronising on reload ?
On 14.05.2019 10:38, David Marchand wrote: > Hello Ilya, > On Mon, May 6, 2019 at 5:37 PM Ilya Maximets <i.maximets@samsung.com <mailto:i.maximets@samsung.com>> wrote: > > On 30.04.2019 15:17, David Marchand wrote: > > No need for a latch here since we don't have to wait. > > A simple boolean flag is enough. > > > > Fixes: e4cfed38b159 ("dpif-netdev: Add poll-mode-device thread.") > > Signed-off-by: David Marchand <david.marchand@redhat.com <mailto:david.marchand@redhat.com>> > > --- > > lib/dpif-netdev.c | 9 ++++----- > > 1 file changed, 4 insertions(+), 5 deletions(-) > > > > diff --git a/lib/dpif-netdev.c b/lib/dpif-netdev.c > > index f1422b2..30774ed 100644 > > --- a/lib/dpif-netdev.c > > +++ b/lib/dpif-netdev.c > > @@ -681,10 +681,10 @@ struct dp_netdev_pmd_thread { > > /* Current context of the PMD thread. */ > > struct dp_netdev_pmd_thread_ctx ctx; > > > > - struct latch exit_latch; /* For terminating the pmd thread. */ > > struct seq *reload_seq; > > uint64_t last_reload_seq; > > atomic_bool reload; /* Do we need to reload ports? */ > > + atomic_bool exit; /* For terminating the pmd thread. */ > > pthread_t thread; > > unsigned core_id; /* CPU core id of this pmd thread. */ > > int numa_id; /* numa node id of this pmd thread. */ > > @@ -5479,7 +5479,7 @@ reload: > > ovs_mutex_unlock(&pmd->perf_stats.stats_mutex); > > > > poll_cnt = pmd_load_queues_and_ports(pmd, &poll_list); > > - exiting = latch_is_set(&pmd->exit_latch); > > + atomic_read_relaxed(&pmd->exit, &exiting); > > I'm afraid that relaxed memory model is not suitable here. > You need to change memory models for both 'reload' and 'exit' > or put ack_rel thread fence between them. Otherwise reads/writes > could be reordered resulting in missed exit and main thread > hang on join. > > > Indeed, I can not use relaxed memory model on both atomics. > > I have been reading some articles and I am a bit puzzled :-). > I don't understand why I would need to update both atomics memory model. > > On the pmd thread side: > atomic_read_explicit(&pmd->reload, &reload, memory_model_acquire); > atomic_read_relaxed(&pmd->exit, &exiting); > > On the control side: > atomic_store_relaxed(&pmd->exit, true); > atomic_store_explicit(&pmd->reload, true, memory_order_release); > > Would not it be enough to have those threads share the same view by synchronising on reload ? Yes. You're right. Above example is valid. I re-checked the spec for release-acquire ordering and it seems that we could use 'reload' with rel-acq ordering as a synchronization point because it will force all memory writes (non-atomic and relaxed atomic) that happened-before the rel atomic store in main thread become visible side-effects in PMD thread after the acq atomic read. Probably, I was confused because of thinking that we need a backward synchronization (PMD --> main). But we don't. https://en.cppreference.com/w/c/atomic/memory_order#Release-Acquire_ordering Best regards, Ilya Maximets.
On Tue, May 14, 2019 at 11:53 AM Ilya Maximets <i.maximets@samsung.com> wrote: > > On 14.05.2019 10:38, David Marchand wrote: > > Hello Ilya, > > On Mon, May 6, 2019 at 5:37 PM Ilya Maximets <i.maximets@samsung.com > <mailto:i.maximets@samsung.com>> wrote: > > > > On 30.04.2019 15:17, David Marchand wrote: > > > No need for a latch here since we don't have to wait. > > > A simple boolean flag is enough. > > > > > > Fixes: e4cfed38b159 ("dpif-netdev: Add poll-mode-device thread.") > > > Signed-off-by: David Marchand <david.marchand@redhat.com <mailto: > david.marchand@redhat.com>> > > > --- > > > lib/dpif-netdev.c | 9 ++++----- > > > 1 file changed, 4 insertions(+), 5 deletions(-) > > > > > > diff --git a/lib/dpif-netdev.c b/lib/dpif-netdev.c > > > index f1422b2..30774ed 100644 > > > --- a/lib/dpif-netdev.c > > > +++ b/lib/dpif-netdev.c > > > @@ -681,10 +681,10 @@ struct dp_netdev_pmd_thread { > > > /* Current context of the PMD thread. */ > > > struct dp_netdev_pmd_thread_ctx ctx; > > > > > > - struct latch exit_latch; /* For terminating the pmd > thread. */ > > > struct seq *reload_seq; > > > uint64_t last_reload_seq; > > > atomic_bool reload; /* Do we need to reload > ports? */ > > > + atomic_bool exit; /* For terminating the pmd > thread. */ > > > pthread_t thread; > > > unsigned core_id; /* CPU core id of this pmd > thread. */ > > > int numa_id; /* numa node id of this pmd > thread. */ > > > @@ -5479,7 +5479,7 @@ reload: > > > ovs_mutex_unlock(&pmd->perf_stats.stats_mutex); > > > > > > poll_cnt = pmd_load_queues_and_ports(pmd, &poll_list); > > > - exiting = latch_is_set(&pmd->exit_latch); > > > + atomic_read_relaxed(&pmd->exit, &exiting); > > > > I'm afraid that relaxed memory model is not suitable here. > > You need to change memory models for both 'reload' and 'exit' > > or put ack_rel thread fence between them. Otherwise reads/writes > > could be reordered resulting in missed exit and main thread > > hang on join. > > > > > > Indeed, I can not use relaxed memory model on both atomics. > > > > I have been reading some articles and I am a bit puzzled :-). > > I don't understand why I would need to update both atomics memory model. > > > > On the pmd thread side: > > atomic_read_explicit(&pmd->reload, &reload, memory_model_acquire); > > atomic_read_relaxed(&pmd->exit, &exiting); > > > > On the control side: > > atomic_store_relaxed(&pmd->exit, true); > > atomic_store_explicit(&pmd->reload, true, memory_order_release); > > > > Would not it be enough to have those threads share the same view by > synchronising on reload ? > > Yes. You're right. Above example is valid. > I re-checked the spec for release-acquire ordering and it seems that > we could use 'reload' with rel-acq ordering as a synchronization point > because it will force all memory writes (non-atomic and relaxed atomic) > that happened-before the rel atomic store in main thread become visible > side-effects in PMD thread after the acq atomic read. > > Probably, I was confused because of thinking that we need a backward > synchronization (PMD --> main). But we don't. > > > https://en.cppreference.com/w/c/atomic/memory_order#Release-Acquire_ordering Thanks a lot. Then I will revisit my series with that in mind.
diff --git a/lib/dpif-netdev.c b/lib/dpif-netdev.c index f1422b2..30774ed 100644 --- a/lib/dpif-netdev.c +++ b/lib/dpif-netdev.c @@ -681,10 +681,10 @@ struct dp_netdev_pmd_thread { /* Current context of the PMD thread. */ struct dp_netdev_pmd_thread_ctx ctx; - struct latch exit_latch; /* For terminating the pmd thread. */ struct seq *reload_seq; uint64_t last_reload_seq; atomic_bool reload; /* Do we need to reload ports? */ + atomic_bool exit; /* For terminating the pmd thread. */ pthread_t thread; unsigned core_id; /* CPU core id of this pmd thread. */ int numa_id; /* numa node id of this pmd thread. */ @@ -5479,7 +5479,7 @@ reload: ovs_mutex_unlock(&pmd->perf_stats.stats_mutex); poll_cnt = pmd_load_queues_and_ports(pmd, &poll_list); - exiting = latch_is_set(&pmd->exit_latch); + atomic_read_relaxed(&pmd->exit, &exiting); /* Signal here to make sure the pmd finishes * reloading the updated configuration. */ dp_netdev_pmd_reload_done(pmd); @@ -5898,7 +5898,7 @@ dp_netdev_configure_pmd(struct dp_netdev_pmd_thread *pmd, struct dp_netdev *dp, pmd->n_output_batches = 0; ovs_refcount_init(&pmd->ref_cnt); - latch_init(&pmd->exit_latch); + atomic_init(&pmd->exit, false); pmd->reload_seq = seq_create(); pmd->last_reload_seq = seq_read(pmd->reload_seq); atomic_init(&pmd->reload, false); @@ -5945,7 +5945,6 @@ dp_netdev_destroy_pmd(struct dp_netdev_pmd_thread *pmd) cmap_destroy(&pmd->classifiers); cmap_destroy(&pmd->flow_table); ovs_mutex_destroy(&pmd->flow_mutex); - latch_destroy(&pmd->exit_latch); seq_destroy(pmd->reload_seq); xpthread_cond_destroy(&pmd->cond); ovs_mutex_destroy(&pmd->cond_mutex); @@ -5967,7 +5966,7 @@ dp_netdev_del_pmd(struct dp_netdev *dp, struct dp_netdev_pmd_thread *pmd) pmd_free_static_tx_qid(pmd); ovs_mutex_unlock(&dp->non_pmd_mutex); } else { - latch_set(&pmd->exit_latch); + atomic_store_relaxed(&pmd->exit, true); dp_netdev_reload_pmd__(pmd); xpthread_join(pmd->thread, NULL); }
No need for a latch here since we don't have to wait. A simple boolean flag is enough. Fixes: e4cfed38b159 ("dpif-netdev: Add poll-mode-device thread.") Signed-off-by: David Marchand <david.marchand@redhat.com> --- lib/dpif-netdev.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-)