diff mbox series

bpf: Add support to eBPF atomic operations

Message ID 20210909170636.365036-1-guillermo.e.martinez@oracle.com
State New
Headers show
Series bpf: Add support to eBPF atomic operations | expand

Commit Message

Guillermo E. Martinez Sept. 9, 2021, 5:06 p.m. UTC 
Hello people,

This patch add support for atomics operations in eBPF target
using the gcc built-in functions:

  __atomic_<operation>_fetch
  __atomic_fetch_<operation>

Please if you have comments, don't hesitate to let me know.

Kinds Regards,
Guillermo


eBPF add support for basic atomic operations, the following
gcc built-in functions are implemented for bpf target using
both: 32 and 64 bits data types:

 __atomic_fetch_add
 __atomic_fetch_sub
 __atomic_fetch_and
 __atomic_fetch_xor
 __atomic_fetch_or
 __atomic_exchange
 __atomic_compare_exchange_n

Also calls to __atomic_<operation>_fetch are fully supported.

New define instructions were added to bpf.md along with its
tests for gcc atomic built-in functions.

Those operations are fully compliant with:
  https://gcc.gnu.org/onlinedocs/gcc/_005f_005fatomic-Builtins.html
  https://www.kernel.org/doc/Documentation/networking/filter.rst

This support depends of two previous submissions in CGEN and
binutils-gdb projects:
   https://sourceware.org/pipermail/cgen/2021q3/002774.html
   https://sourceware.org/pipermail/binutils/2021-August/117798.html

gcc/
  * config/bpf/bpf.md: Add defines for atomic instructions.

gcc/testsuite/
   * gcc.target/bpf/atomic-compare-exchange.c: New test.
   * gcc.target/bpf/atomic-exchange.c: Likewise.
   * gcc.target/bpf/atomic-fetch-add.c: Likewise.
   * gcc.target/bpf/atomic-fetch-and.c: Likewise.
   * gcc.target/bpf/atomic-fetch-or.c: Likewise.
   * gcc.target/bpf/atomic-fetch-sub.c: Likewise.
   * gcc.target/bpf/atomic-fetch-xor.c: Likewise.
---
 gcc/config/bpf/bpf.h                          |   8 +-
 gcc/config/bpf/bpf.md                         | 122 ++++++++++++++++--
 gcc/config/bpf/constraints.md                 |   3 +
 .../gcc.target/bpf/atomic-compare-exchange.c  |  28 ++++
 .../gcc.target/bpf/atomic-exchange.c          |  19 +++
 .../gcc.target/bpf/atomic-fetch-add.c         |  26 ++++
 .../gcc.target/bpf/atomic-fetch-and.c         |  25 ++++
 .../gcc.target/bpf/atomic-fetch-or.c          |  25 ++++
 .../gcc.target/bpf/atomic-fetch-sub.c         |  25 ++++
 .../gcc.target/bpf/atomic-fetch-xor.c         |  25 ++++
 10 files changed, 292 insertions(+), 14 deletions(-)
 create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-compare-exchange.c
 create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-exchange.c
 create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-fetch-add.c
 create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-fetch-and.c
 create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-fetch-or.c
 create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-fetch-sub.c
 create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-fetch-xor.c
diff mbox series

Patch

diff --git a/gcc/config/bpf/bpf.h b/gcc/config/bpf/bpf.h
index 82be0c3e190..f8d0f7fb7dd 100644
--- a/gcc/config/bpf/bpf.h
+++ b/gcc/config/bpf/bpf.h
@@ -176,6 +176,7 @@ 
 enum reg_class
 {
   NO_REGS,		/* no registers in set.  */
+  R0,		        /* register r0.  */
   ALL_REGS,		/* all registers.  */
   LIM_REG_CLASSES	/* max value + 1.  */
 };
@@ -189,6 +190,7 @@  enum reg_class
 #define REG_CLASS_NAMES				\
 {						\
   "NO_REGS",					\
+  "R0",					\
   "ALL_REGS"					\
 }
 
@@ -202,14 +204,16 @@  enum reg_class
 #define REG_CLASS_CONTENTS			\
 {						\
    0x00000000, /* NO_REGS */			\
-   0x00000fff, /* ALL_REGS */		        \
+   0x00000001, /* R0 */		        \
+   0x00000fff, /* ALL_REGS */		  \
 }
 
 /* A C expression whose value is a register class containing hard
    register REGNO.  In general there is more that one such class;
    choose a class which is "minimal", meaning that no smaller class
    also contains the register.  */
-#define REGNO_REG_CLASS(REGNO) GENERAL_REGS
+#define REGNO_REG_CLASS(REGNO)          \
+  ((REGNO) == 0 ? R0 : GENERAL_REGS)
 
 /* A macro whose definition is the name of the class to which a
    valid base register must belong.  A base register is one used in
diff --git a/gcc/config/bpf/bpf.md b/gcc/config/bpf/bpf.md
index 03830cc250e..f5a4d2d02b4 100644
--- a/gcc/config/bpf/bpf.md
+++ b/gcc/config/bpf/bpf.md
@@ -25,6 +25,11 @@ 
 (define_c_enum "unspec" [
   UNSPEC_LDINDABS
   UNSPEC_XADD
+  UNSPEC_XAND
+  UNSPEC_XOR
+  UNSPEC_XXOR
+  UNSPEC_XCHG
+  UNSPEC_CMPXCHG
 ])
 
 ;;;; Constants
@@ -56,11 +61,10 @@ 
 ;; st		generic store instructions for immediates.
 ;; stx		generic store instructions.
 ;; jmp		jump instructions.
-;; xadd		atomic exchange-and-add instructions.
 ;; multi	multiword sequence (or user asm statements).
 
 (define_attr "type"
-  "unknown,alu,alu32,end,ld,lddw,ldx,st,stx,jmp,xadd,multi"
+  "unknown,alu,alu32,end,ld,lddw,ldx,st,stx,jmp,multi"
   (const_string "unknown"))
 
 ;; Length of instruction in bytes.
@@ -506,17 +510,111 @@ 
   "ldabs<ldop>\t%0"
   [(set_attr "type" "ld")])
 
-;;;; Atomic increments
+;;;; Atomic operations
 
 (define_mode_iterator AMO [SI DI])
 
-(define_insn "atomic_add<AMO:mode>"
-  [(set (match_operand:AMO 0 "memory_operand" "+m")
-        (unspec_volatile:AMO
-         [(plus:AMO (match_dup 0)
-                    (match_operand:AMO 1 "register_operand" "r"))
-          (match_operand:SI 2 "const_int_operand")] ;; Memory model.
-         UNSPEC_XADD))]
+(define_insn "atomic_fetch_add<AMO:mode>"
+  [(set (match_operand:AMO 0 "register_operand" "=r")
+  (unspec_volatile:AMO
+    [(match_operand:AMO 1 "memory_operand" "+o")
+     (match_operand:AMO 2 "nonmemory_operand" "0")
+     (match_operand:AMO 3 "const_int_operand")] ;; Memory model
+    UNSPEC_XADD))]
+  ""
+  "xadd<mop>\t%1,%0")
+
+(define_insn "atomic_fetch_and<AMO:mode>"
+  [(set (match_operand:AMO 0 "register_operand" "=r")
+  (unspec_volatile:AMO
+    [(match_operand:AMO 1 "memory_operand" "+o")
+     (match_operand:AMO 2 "nonmemory_operand" "0")
+     (match_operand:AMO 3 "const_int_operand")]
+    UNSPEC_XAND))]
+  ""
+  "xand<mop>\t%1,%0")
+
+(define_insn "atomic_fetch_or<AMO:mode>"
+  [(set (match_operand:AMO 0 "register_operand" "=r")
+  (unspec_volatile:AMO
+    [(match_operand:AMO 1 "memory_operand" "+o")
+     (match_operand:AMO 2 "nonmemory_operand" "0")
+     (match_operand:AMO 3 "const_int_operand")]
+    UNSPEC_XOR))]
+  ""
+  "xor<mop>\t%1,%0")
+
+(define_insn "atomic_fetch_xor<AMO:mode>"
+  [(set (match_operand:AMO 0 "register_operand" "=r")
+  (unspec_volatile:AMO
+    [(match_operand:AMO 1 "memory_operand" "+o")
+     (match_operand:AMO 2 "nonmemory_operand" "0")
+     (match_operand:AMO 3 "const_int_operand")]
+    UNSPEC_XXOR))]
+  ""
+  "xxor<mop>\t%1,%0")
+
+(define_insn "atomic_exchange<AMO:mode>"
+  [(set (match_operand:AMO 0 "register_operand" "=r")
+  (unspec_volatile:AMO
+    [(match_operand:AMO 1 "memory_operand" "+o")
+     (match_operand:AMO 2 "nonmemory_operand" "0")
+     (match_operand:AMO 3 "const_int_operand")]
+    UNSPEC_XCHG))]
+  ""
+  "xchg<mop>\t%1,%0")
+
+;; eBPF compare and exchange operation atomically compares the
+;; value addressed by memory operand(%0) with _R0_(%1), so
+;; there is a constraint to load the expected value in mentioned
+;; register. If they match it is replaced with desired value(%3).
+;; In either case, the value that was there before in %0 is
+;; zero-extended and loaded back to R0.
+
+(define_expand "atomic_compare_and_swap<AMO:mode>"
+  [(match_operand:SI 0 "register_operand")    ;; bool success
+   (match_operand:AMO 1 "register_operand")   ;; old value
+   (match_operand:AMO 2 "memory_operand")     ;; memory
+   (match_operand:AMO 3 "register_operand")   ;; expected
+   (match_operand:AMO 4 "register_operand")   ;; desired
+   (match_operand:SI 5 "const_int_operand")   ;; is_weak (unused)
+   (match_operand:SI 6 "const_int_operand")   ;; success model (unused)
+   (match_operand:SI 7 "const_int_operand")]  ;; failure model (unused)
   ""
-  "xadd<mop>\t%0,%1"
-  [(set_attr "type" "xadd")])
+{
+  emit_insn
+    (gen_atomic_compare_and_swap<AMO:mode>_1
+      (operands[1], operands[2], operands[3], operands[4], operands[6]));
+
+  /* Assume success operation, i.e memory operand
+     is matched with expected value.
+   */
+  emit_move_insn (operands[0], const1_rtx);
+  rtx_code_label *success_label = gen_label_rtx ();
+
+  /* At this point eBPF xcmp was executed, now we can ask
+   * for success exchange, and set suitable value for bool
+   * operand(%0)
+   */
+  emit_cmp_and_jump_insns (operands[1], operands[3], EQ, 0,
+                           GET_MODE (operands[1]), 1, success_label);
+  emit_move_insn (operands[0], const0_rtx);
+
+  if (success_label)
+    {
+       emit_label (success_label);
+       LABEL_NUSES (success_label) = 1;
+    }
+  DONE;
+})
+
+(define_insn "atomic_compare_and_swap<AMO:mode>_1"
+  [(set (match_operand:AMO 0 "register_operand" "=t") ;; must be r0
+  (unspec_volatile:AMO
+    [(match_operand:AMO 1 "memory_operand" "+o")  ;; memory
+     (match_operand:AMO 2 "register_operand" "0") ;; expected
+     (match_operand:AMO 3 "register_operand" "r") ;; desired
+     (match_operand:SI 4 "const_int_operand")]    ;; success (unused)
+    UNSPEC_CMPXCHG))]
+  ""
+  "xcmp<mop>\t%1,%3")
diff --git a/gcc/config/bpf/constraints.md b/gcc/config/bpf/constraints.md
index 66b7764d775..9606b6e150e 100644
--- a/gcc/config/bpf/constraints.md
+++ b/gcc/config/bpf/constraints.md
@@ -29,3 +29,6 @@ 
 (define_constraint "S"
   "A constant call address."
   (match_code "const,symbol_ref,label_ref,const_int"))
+
+(define_register_constraint "t" "R0"
+  "Register r0")
diff --git a/gcc/testsuite/gcc.target/bpf/atomic-compare-exchange.c b/gcc/testsuite/gcc.target/bpf/atomic-compare-exchange.c
new file mode 100644
index 00000000000..d522697ec9e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomic-compare-exchange.c
@@ -0,0 +1,28 @@ 
+/* { dg-do compile } */
+
+long val;
+long ptr;
+long expected;
+long desired;
+
+void
+foo ()
+{
+  int done;
+
+  done = __atomic_compare_exchange_n (&ptr, &expected, desired,
+                  0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+  done = __atomic_compare_exchange_n ((int *)&ptr, (int *)&expected,
+                  (int)desired, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+
+  done = __atomic_compare_exchange (&ptr, &expected, &desired,
+                  0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+  done = __atomic_compare_exchange ((int *)&ptr, (int *)&expected,
+                  (int *)&desired, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+
+  done = __sync_bool_compare_and_swap (&ptr, expected, desired);
+  done = __sync_bool_compare_and_swap ((int*)&ptr, expected, desired);
+}
+
+/* { dg-final { scan-assembler "xcmpdw\t.*" } } */
+/* { dg-final { scan-assembler "xcmpw\t.*" } } */
diff --git a/gcc/testsuite/gcc.target/bpf/atomic-exchange.c b/gcc/testsuite/gcc.target/bpf/atomic-exchange.c
new file mode 100644
index 00000000000..f4e60568f7f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomic-exchange.c
@@ -0,0 +1,19 @@ 
+/* { dg-do compile } */
+
+long val;
+long ptr;
+
+void
+foo ()
+{
+  long prev;
+
+  __atomic_exchange(&ptr, &val, &prev, __ATOMIC_RELAXED);
+  prev = __atomic_exchange_n(&ptr, val, __ATOMIC_RELAXED);
+
+  __atomic_exchange((int *)&ptr, (int *)&val, (int *)&prev, __ATOMIC_RELAXED);
+  prev = __atomic_exchange_n((int *)&ptr, (int)val, __ATOMIC_RELAXED);
+}
+
+/* { dg-final { scan-assembler "xchgdw\t.*" } } */
+/* { dg-final { scan-assembler "xchgw\t.*" } } */
diff --git a/gcc/testsuite/gcc.target/bpf/atomic-fetch-add.c b/gcc/testsuite/gcc.target/bpf/atomic-fetch-add.c
new file mode 100644
index 00000000000..2547396d2a1
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomic-fetch-add.c
@@ -0,0 +1,26 @@ 
+/* { dg-do compile } */
+
+long delta;
+long *val;
+
+void
+foo ()
+{
+  long k;
+
+  k = __atomic_fetch_add (val, delta, __ATOMIC_RELAXED);
+  k = __atomic_fetch_add ((int*)val, delta, __ATOMIC_RELAXED);
+
+  k = __atomic_add_fetch (val, delta, __ATOMIC_RELAXED);
+  k = __atomic_add_fetch ((int*)val, delta, __ATOMIC_RELAXED);
+
+  k = __sync_fetch_and_add (val, delta, __ATOMIC_RELAXED);
+  k = __sync_fetch_and_add ((int*)val, delta, __ATOMIC_RELAXED);
+
+  k = __sync_add_and_fetch (val, delta, __ATOMIC_RELAXED);
+  k = __sync_add_and_fetch ((int*)val, delta, __ATOMIC_RELAXED);
+
+}
+
+/* { dg-final { scan-assembler "xadddw\t.*" } } */
+/* { dg-final { scan-assembler "xaddw\t.*" } } */
diff --git a/gcc/testsuite/gcc.target/bpf/atomic-fetch-and.c b/gcc/testsuite/gcc.target/bpf/atomic-fetch-and.c
new file mode 100644
index 00000000000..6cc05a824f6
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomic-fetch-and.c
@@ -0,0 +1,25 @@ 
+/* { dg-do compile } */
+
+long mask;
+long *val;
+
+void
+foo ()
+{
+  long k;
+
+  k = __atomic_fetch_and (val, mask, __ATOMIC_RELAXED);
+  k = __atomic_fetch_and ((int*)val, mask, __ATOMIC_RELAXED);
+
+  k = __atomic_and_fetch (val, mask, __ATOMIC_RELAXED);
+  k = __atomic_and_fetch ((int*)val, mask, __ATOMIC_RELAXED);
+
+  k = __sync_fetch_and_and (val, mask, __ATOMIC_RELAXED);
+  k = __sync_fetch_and_and ((int*)val, mask, __ATOMIC_RELAXED);
+
+  k = __sync_and_and_fetch (val, mask, __ATOMIC_RELAXED);
+  k = __sync_and_and_fetch ((int*)val, mask, __ATOMIC_RELAXED);
+}
+
+/* { dg-final { scan-assembler "xanddw\t.*" } } */
+/* { dg-final { scan-assembler "xandw\t.*" } } */
diff --git a/gcc/testsuite/gcc.target/bpf/atomic-fetch-or.c b/gcc/testsuite/gcc.target/bpf/atomic-fetch-or.c
new file mode 100644
index 00000000000..af9a999e02a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomic-fetch-or.c
@@ -0,0 +1,25 @@ 
+/* { dg-do compile } */
+
+long bits;
+long *val;
+
+void
+foo ()
+{
+  long k;
+
+  k = __atomic_fetch_or (val, bits, __ATOMIC_RELAXED);
+  k = __atomic_fetch_or ((int *)val, bits, __ATOMIC_RELAXED);
+
+  k = __atomic_or_fetch (val, bits, __ATOMIC_RELAXED);
+  k = __atomic_or_fetch ((int*)val, bits, __ATOMIC_RELAXED);
+
+  k = __sync_fetch_and_or (val, bits, __ATOMIC_RELAXED);
+  k = __sync_fetch_and_or ((int*)val, bits, __ATOMIC_RELAXED);
+
+  k = __sync_or_and_fetch (val, bits, __ATOMIC_RELAXED);
+  k = __sync_or_and_fetch ((int*)val, bits, __ATOMIC_RELAXED);
+}
+
+/* { dg-final { scan-assembler "xordw\t.*" } } */
+/* { dg-final { scan-assembler "xorw\t.*" } } */
diff --git a/gcc/testsuite/gcc.target/bpf/atomic-fetch-sub.c b/gcc/testsuite/gcc.target/bpf/atomic-fetch-sub.c
new file mode 100644
index 00000000000..6e6a16caa3c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomic-fetch-sub.c
@@ -0,0 +1,25 @@ 
+/* { dg-do compile } */
+
+long delta;
+long *val;
+
+void
+foo ()
+{
+  long k;
+
+  k = __atomic_fetch_sub (val, delta, __ATOMIC_RELAXED);
+  k = __atomic_fetch_sub ((int*)val, delta, __ATOMIC_RELAXED);
+
+  k = __atomic_sub_fetch (val, delta, __ATOMIC_RELAXED);
+  k = __atomic_sub_fetch ((int*)val, delta, __ATOMIC_RELAXED);
+
+  k = __sync_fetch_and_sub (val, delta, __ATOMIC_RELAXED);
+  k = __sync_fetch_and_sub ((int*)val, delta, __ATOMIC_RELAXED);
+
+  k = __sync_sub_and_fetch (val, delta, __ATOMIC_RELAXED);
+  k = __sync_sub_and_fetch ((int*)val, delta, __ATOMIC_RELAXED);
+}
+
+/* { dg-final { scan-assembler "xadddw\t.*" } } */
+/* { dg-final { scan-assembler "xaddw\t.*" } } */
diff --git a/gcc/testsuite/gcc.target/bpf/atomic-fetch-xor.c b/gcc/testsuite/gcc.target/bpf/atomic-fetch-xor.c
new file mode 100644
index 00000000000..433600395a6
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bpf/atomic-fetch-xor.c
@@ -0,0 +1,25 @@ 
+/* { dg-do compile } */
+
+long bits;
+long *val;
+
+void
+foo ()
+{
+  long k;
+
+  k = __atomic_fetch_xor (val, bits, __ATOMIC_RELAXED);
+  k = __atomic_fetch_xor ((int *)val, bits, __ATOMIC_RELAXED);
+
+  k = __atomic_xor_fetch (val, bits, __ATOMIC_RELAXED);
+  k = __atomic_xor_fetch ((int*)val, bits, __ATOMIC_RELAXED);
+
+  k = __sync_fetch_and_xor (val, bits, __ATOMIC_RELAXED);
+  k = __sync_fetch_and_xor ((int*)val, bits, __ATOMIC_RELAXED);
+
+  k = __sync_xor_and_fetch (val, bits, __ATOMIC_RELAXED);
+  k = __sync_xor_and_fetch ((int*)val, bits, __ATOMIC_RELAXED);
+}
+
+/* { dg-final { scan-assembler "xxordw\t.*" } } */
+/* { dg-final { scan-assembler "xxorw\t.*" } } */