@@ -20,6 +20,7 @@
#include <fcntl.h>
#include <errno.h>
#include <asm/unistd.h>
+#include <asm/barrier.h>
#include <linux/err.h>
#include <linux/kernel.h>
#include <linux/bpf.h>
@@ -2413,18 +2414,32 @@ int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr,
return 0;
}
+static __u64 bpf_perf_read_head(struct perf_event_mmap_page *header)
+{
+ __u64 data_head = READ_ONCE(header->data_head);
+
+ smp_rmb();
+ return data_head;
+}
+
+static void bpf_perf_write_tail(struct perf_event_mmap_page *header,
+ __u64 data_tail)
+{
+ smp_mb();
+ header->data_tail = data_tail;
+}
+
enum bpf_perf_event_ret
bpf_perf_event_read_simple(void *mem, unsigned long size,
unsigned long page_size, void **buf, size_t *buf_len,
bpf_perf_event_print_t fn, void *priv)
{
- volatile struct perf_event_mmap_page *header = mem;
+ struct perf_event_mmap_page *header = mem;
+ __u64 data_head = bpf_perf_read_head(header);
__u64 data_tail = header->data_tail;
- __u64 data_head = header->data_head;
int ret = LIBBPF_PERF_EVENT_ERROR;
void *base, *begin, *end;
- asm volatile("" ::: "memory"); /* in real code it should be smp_rmb() */
if (data_head == data_tail)
return LIBBPF_PERF_EVENT_CONT;
@@ -2467,8 +2482,6 @@ bpf_perf_event_read_simple(void *mem, unsigned long size,
data_tail += ehdr->size;
}
- __sync_synchronize(); /* smp_mb() */
- header->data_tail = data_tail;
-
+ bpf_perf_write_tail(header, data_tail);
return ret;
}
Add bpf_perf_read_head() and bpf_perf_write_tail() helpers to make it more clear in what context barriers are used here, and use smp_rmb() as well as smp_mb() barriers. Given libbpf is not restricted to x86-64 only, the compiler barrier needs to be replaced with smp_rmb(). Also the __sync_synchronize() emits mfence whereas faster lock + add can be used on x86-64 via smp_mb(). Fixes: d0cabbb021be ("tools: bpf: move the event reading loop to libbpf") Fixes: 39111695b1b8 ("samples: bpf: add bpf_perf_event_output example") Signed-off-by: Daniel Borkmann <daniel@iogearbox.net> --- tools/lib/bpf/libbpf.c | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-)