[v6,bpf-next,1/2] bpf: Add bpf_read_branch_records() helper

Branch records are a CPU feature that can be configured to record
certain branches that are taken during code execution. This data is
particularly interesting for profile guided optimizations. perf has had
branch record support for a while but the data collection can be a bit
coarse grained.

We (Facebook) have seen in experiments that associating metadata with
branch records can improve results (after postprocessing). We generally
use bpf_probe_read_*() to get metadata out of userspace. That's why bpf
support for branch records is useful.

Aside from this particular use case, having branch data available to bpf
progs can be useful to get stack traces out of userspace applications
that omit frame pointers.

Signed-off-by: Daniel Xu <dxu@dxuuu.xyz>
---
 include/uapi/linux/bpf.h | 25 +++++++++++++++++++++++-
 kernel/trace/bpf_trace.c | 41 ++++++++++++++++++++++++++++++++++++++++
 2 files changed, 65 insertions(+), 1 deletion(-)

On 1/27/20 12:35 AM, Daniel Xu wrote:
> Branch records are a CPU feature that can be configured to record
> certain branches that are taken during code execution. This data is
> particularly interesting for profile guided optimizations. perf has had
> branch record support for a while but the data collection can be a bit
> coarse grained.
> 
> We (Facebook) have seen in experiments that associating metadata with
> branch records can improve results (after postprocessing). We generally
> use bpf_probe_read_*() to get metadata out of userspace. That's why bpf
> support for branch records is useful.
> 
> Aside from this particular use case, having branch data available to bpf
> progs can be useful to get stack traces out of userspace applications
> that omit frame pointers.
> 
> Signed-off-by: Daniel Xu <dxu@dxuuu.xyz>
> ---
>   include/uapi/linux/bpf.h | 25 +++++++++++++++++++++++-
>   kernel/trace/bpf_trace.c | 41 ++++++++++++++++++++++++++++++++++++++++
>   2 files changed, 65 insertions(+), 1 deletion(-)
> 
> diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
> index f1d74a2bd234..332aa433d045 100644
> --- a/include/uapi/linux/bpf.h
> +++ b/include/uapi/linux/bpf.h
> @@ -2892,6 +2892,25 @@ union bpf_attr {
>    *		Obtain the 64bit jiffies
>    *	Return
>    *		The 64 bit jiffies
> + *
> + * int bpf_read_branch_records(struct bpf_perf_event_data *ctx, void *buf, u32 buf_size, u64 flags)

Small nit: s/buf_size/size/, so that it matches with your BPF_CALL below.

  +BPF_CALL_4(bpf_read_branch_records, struct bpf_perf_event_data_kern *, ctx,
  +	   void *, buf, u32, size, u64, flags)

> + *	Description
> + *		For an eBPF program attached to a perf event, retrieve the
> + *		branch records (struct perf_branch_entry) associated to *ctx*
> + *		and store it in	the buffer pointed by *buf* up to size
> + *		*buf_size* bytes.
> + *
> + *		The *flags* can be set to **BPF_F_GET_BRANCH_RECORDS_SIZE** to
> + *		instead	return the number of bytes required to store all the
> + *		branch entries. If this flag is set, *buf* may be NULL.
> + *	Return
> + *		On success, number of bytes written to *buf*. On error, a
> + *		negative value.

Maybe pull the 2nd paragraph from above in here so that it reflects the description
of the return value when flag is used also for this case in the 'Return' description.

> + *		**-EINVAL** if arguments invalid or **buf_size** not a multiple
> + *		of sizeof(struct perf_branch_entry).
> + *
> + *		**-ENOENT** if architecture does not support branch records.
>    */
>   #define __BPF_FUNC_MAPPER(FN)		\
>   	FN(unspec),			\
> @@ -3012,7 +3031,8 @@ union bpf_attr {
>   	FN(probe_read_kernel_str),	\
>   	FN(tcp_send_ack),		\
>   	FN(send_signal_thread),		\
> -	FN(jiffies64),
> +	FN(jiffies64),			\
> +	FN(read_branch_records),
>   
>   /* integer value in 'imm' field of BPF_CALL instruction selects which helper
>    * function eBPF program intends to call
> @@ -3091,6 +3111,9 @@ enum bpf_func_id {
>   /* BPF_FUNC_sk_storage_get flags */
>   #define BPF_SK_STORAGE_GET_F_CREATE	(1ULL << 0)
>   
> +/* BPF_FUNC_read_branch_records flags. */
> +#define BPF_F_GET_BRANCH_RECORDS_SIZE	(1ULL << 0)
> +
>   /* Mode for BPF_FUNC_skb_adjust_room helper. */
>   enum bpf_adj_room_mode {
>   	BPF_ADJ_ROOM_NET,
> diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c
> index 19e793aa441a..efd119de95b8 100644
> --- a/kernel/trace/bpf_trace.c
> +++ b/kernel/trace/bpf_trace.c
> @@ -1028,6 +1028,45 @@ static const struct bpf_func_proto bpf_perf_prog_read_value_proto = {
>            .arg3_type      = ARG_CONST_SIZE,
>   };
>   
> +BPF_CALL_4(bpf_read_branch_records, struct bpf_perf_event_data_kern *, ctx,
> +	   void *, buf, u32, size, u64, flags)
> +{
> +#ifndef CONFIG_X86
> +	return -ENOENT;
> +#else
> +	struct perf_branch_stack *br_stack = ctx->data->br_stack;
> +	u32 br_entry_size = sizeof(struct perf_branch_entry);

'static const u32 br_entry_size' if we use it as such below.

> +	u32 to_copy;
> +
> +	if (unlikely(flags & ~BPF_F_GET_BRANCH_RECORDS_SIZE))
> +		return -EINVAL;
> +
> +	if (unlikely(!br_stack))
> +		return -EINVAL;

Why the ifdef X86? In previous thread I meant to change it into since it's
implicit:

         if (unlikely(!br_stack))
                 return -ENOENT;

Or is there any other additional rationale?

> +	if (flags & BPF_F_GET_BRANCH_RECORDS_SIZE)
> +		return br_stack->nr * br_entry_size;
> +
> +	if (!buf || (size % br_entry_size != 0))
> +		return -EINVAL;
> +
> +	to_copy = min_t(u32, br_stack->nr * br_entry_size, size);
> +	memcpy(buf, br_stack->entries, to_copy);
> +
> +	return to_copy;
> +#endif
> +}
> +
> +static const struct bpf_func_proto bpf_read_branch_records_proto = {
> +	.func           = bpf_read_branch_records,
> +	.gpl_only       = true,
> +	.ret_type       = RET_INTEGER,
> +	.arg1_type      = ARG_PTR_TO_CTX,
> +	.arg2_type      = ARG_PTR_TO_MEM_OR_NULL,
> +	.arg3_type      = ARG_CONST_SIZE_OR_ZERO,
> +	.arg4_type      = ARG_ANYTHING,
> +};
> +
>   static const struct bpf_func_proto *
>   pe_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
>   {
> @@ -1040,6 +1079,8 @@ pe_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
>   		return &bpf_get_stack_proto_tp;
>   	case BPF_FUNC_perf_prog_read_value:
>   		return &bpf_perf_prog_read_value_proto;
> +	case BPF_FUNC_read_branch_records:
> +		return &bpf_read_branch_records_proto;
>   	default:
>   		return tracing_func_proto(func_id, prog);
>   	}
>

On Mon Jan 27, 2020 at 1:26 PM, Daniel Borkmann wrote:
[...]
> > diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
> > index f1d74a2bd234..332aa433d045 100644
> > --- a/include/uapi/linux/bpf.h
> > +++ b/include/uapi/linux/bpf.h
> > @@ -2892,6 +2892,25 @@ union bpf_attr {
> >    *		Obtain the 64bit jiffies
> >    *	Return
> >    *		The 64 bit jiffies
> > + *
> > + * int bpf_read_branch_records(struct bpf_perf_event_data *ctx, void *buf, u32 buf_size, u64 flags)
>
> 
> Small nit: s/buf_size/size/, so that it matches with your BPF_CALL
> below.

Ok

> 
> +BPF_CALL_4(bpf_read_branch_records, struct bpf_perf_event_data_kern *,
> ctx,
> + void *, buf, u32, size, u64, flags)
>
> 
> > + *	Description
> > + *		For an eBPF program attached to a perf event, retrieve the
> > + *		branch records (struct perf_branch_entry) associated to *ctx*
> > + *		and store it in	the buffer pointed by *buf* up to size
> > + *		*buf_size* bytes.
> > + *
> > + *		The *flags* can be set to **BPF_F_GET_BRANCH_RECORDS_SIZE** to
> > + *		instead	return the number of bytes required to store all the
> > + *		branch entries. If this flag is set, *buf* may be NULL.
> > + *	Return
> > + *		On success, number of bytes written to *buf*. On error, a
> > + *		negative value.
>
> 
> Maybe pull the 2nd paragraph from above in here so that it reflects the
> description
> of the return value when flag is used also for this case in the 'Return'
> description.

Ok.

[...]
> >   
> > +BPF_CALL_4(bpf_read_branch_records, struct bpf_perf_event_data_kern *, ctx,
> > +	   void *, buf, u32, size, u64, flags)
> > +{
> > +#ifndef CONFIG_X86
> > +	return -ENOENT;
> > +#else
> > +	struct perf_branch_stack *br_stack = ctx->data->br_stack;
> > +	u32 br_entry_size = sizeof(struct perf_branch_entry);
>
> 
> 'static const u32 br_entry_size' if we use it as such below.

Ok

>
> 
> > +	u32 to_copy;
> > +
> > +	if (unlikely(flags & ~BPF_F_GET_BRANCH_RECORDS_SIZE))
> > +		return -EINVAL;
> > +
> > +	if (unlikely(!br_stack))
> > +		return -EINVAL;
>
> 
> Why the ifdef X86? In previous thread I meant to change it into since
> it's
> implicit:
>
> 
> if (unlikely(!br_stack))
> return -ENOENT;
>
> 
> Or is there any other additional rationale?

Yeah, so br_stack can be null if the perf_event is misconfigured (branch
record not enabled). So we need to differentiate that from arch not
supporting it.

[...]


Thanks,
Daniel