diff mbox series

[uci,4/6] tests: cram: add uci import testing on fuzzer corpus

Message ID 20201003074830.948-5-ynezz@true.cz
State Accepted
Delegated to: Petr Štetiar
Headers show
Series fixes and improvements | expand

Commit Message

Petr Štetiar Oct. 3, 2020, 7:48 a.m. UTC 
Use valgrind and uci cli compiled with undefined, address and leak
sanitizers.

Signed-off-by: Petr Štetiar <ynezz@true.cz>
---
 tests/cram/CMakeLists.txt        |  7 ++++++-
 tests/cram/test-san_uci_import.t | 15 +++++++++++++++
 tests/cram/test_uci_import.t     | 15 +++++++++++++++
 3 files changed, 36 insertions(+), 1 deletion(-)
 create mode 100644 tests/cram/test-san_uci_import.t
 create mode 100644 tests/cram/test_uci_import.t

Comments

Hauke Mehrtens Oct. 4, 2020, 3:10 p.m. UTC | #1 
On 10/3/20 9:48 AM, Petr Štetiar wrote:
> Use valgrind and uci cli compiled with undefined, address and leak
> sanitizers.
....
> diff --git a/tests/cram/test-san_uci_import.t b/tests/cram/test-san_uci_import.t
> new file mode 100644
> index 000000000000..7faed221f93e
> --- /dev/null
> +++ b/tests/cram/test-san_uci_import.t
> @@ -0,0 +1,15 @@
> +check that uci import is producing expected results:
> +
> +  $ [ -n "$BUILD_BIN_DIR" ] && export PATH="$BUILD_BIN_DIR:$PATH"
> +  $ export TEST_INPUTS="$TESTDIR/inputs"
> +  $ export FUZZ_CORPUS="$TESTDIR/../fuzz/corpus"
> +
> +  $ for file in $(LC_ALL=C find $FUZZ_CORPUS -type f | sort ); do
> +  >   uci-san import -f $file; \
> +  > done
> +  uci-san: I/O error
> +  uci-san: Parse error (invalid command) at line 0, byte 0
> +  uci-san: Parse error (invalid command) at line 1, byte 18

I do not get this error. Which test case should trigger it?

> +  uci-san: I/O error
> +  uci-san: I/O error
> +  [1].....
Petr Štetiar Oct. 5, 2020, 12:03 p.m. UTC | #2 
Hauke Mehrtens <hauke@hauke-m.de> [2020-10-04 17:10:59]:

> On 10/3/20 9:48 AM, Petr Štetiar wrote:
> > Use valgrind and uci cli compiled with undefined, address and leak
> > sanitizers.
> ....
> > diff --git a/tests/cram/test-san_uci_import.t b/tests/cram/test-san_uci_import.t
> > new file mode 100644
> > index 000000000000..7faed221f93e
> > --- /dev/null
> > +++ b/tests/cram/test-san_uci_import.t
> > @@ -0,0 +1,15 @@
> > +check that uci import is producing expected results:
> > +
> > +  $ [ -n "$BUILD_BIN_DIR" ] && export PATH="$BUILD_BIN_DIR:$PATH"
> > +  $ export TEST_INPUTS="$TESTDIR/inputs"
> > +  $ export FUZZ_CORPUS="$TESTDIR/../fuzz/corpus"
> > +
> > +  $ for file in $(LC_ALL=C find $FUZZ_CORPUS -type f | sort ); do
> > +  >   uci-san import -f $file; \
> > +  > done

BTW this above is a "standard" multiline shell construct (> prefix), so you
can just add simple "echo $file" and see what is going on, like this:

 diff --git a/tests/cram/test_uci_import.t b/tests/cram/test_uci_import.t
 index 8d5ab74ed4e7..e086ecf11ab1 100644
 --- a/tests/cram/test_uci_import.t
 +++ b/tests/cram/test_uci_import.t
 @@ -5,6 +5,7 @@ check that uci import is producing expected results:
    $ export FUZZ_CORPUS="$TESTDIR/../fuzz/corpus"
 
    $ for file in $(LC_ALL=C find $FUZZ_CORPUS -type f | sort ); do
 +  >   echo "testing $file"; \
    >   valgrind --quiet --leak-check=full uci import -f $file; \
    > done
    uci: I/O error

> > +  uci-san: I/O error
> > +  uci-san: Parse error (invalid command) at line 0, byte 0
> > +  uci-san: Parse error (invalid command) at line 1, byte 18
> 
> I do not get this error. Which test case should trigger it?

That's quite strange as I'm getting consistent results with gcc 8,9,10 and
clang 10 (the same version used on CI
https://gitlab.com/ynezz/openwrt-uci/-/jobs/771521825).  Both cram/shuni2 test
suites are run always for complete compiler matrix, fuzzer only under clang
compiler for obvious reasons.

Anyway, here is some clue, hope it helps:

  ynezz@ntbk:/opt/devel/openwrt/c/uci/build/tests/cram$ make test CTEST_OUTPUT_ON_FAILURE=1
  Running tests...
  Test project /opt/devel/openwrt/c/uci/build/tests/cram
      Start 1: cram
  1/1 Test #1: cram .............................***Failed    8.78 sec
  !
  --- /opt/devel/openwrt/c/uci/tests/cram/test_uci_import.t
  +++ /opt/devel/openwrt/c/uci/tests/cram/test_uci_import.t.err
  @@ -8,9 +8,21 @@
     >   echo "testing $file"; \
     >   valgrind --quiet --leak-check=full uci import -f $file; \
     > done
  +  testing /opt/devel/openwrt/c/uci/tests/cram/../fuzz/corpus/231ee80a172b8e1749b9d91867989d88e4faf7bb
  +  testing /opt/devel/openwrt/c/uci/tests/cram/../fuzz/corpus/26a6253fc1eb695b61a2fc7640ee4c03c19e438e
  +  testing /opt/devel/openwrt/c/uci/tests/cram/../fuzz/corpus/29a6e206439d792afba5e8e9c1fdf55e65a1145d
  +  testing /opt/devel/openwrt/c/uci/tests/cram/../fuzz/corpus/2e18ecc3a759dedc9357b1298e9269eccc5c5a6b
  +  uci: Parse error (package without name) at line 0, byte 68
  +  testing /opt/devel/openwrt/c/uci/tests/cram/../fuzz/corpus/51045ac5401085f5727c6d3c1cac5f8cc32a2927
  +  testing /opt/devel/openwrt/c/uci/tests/cram/../fuzz/corpus/845dcf3f15f3c28235e6be148a690e7f03b07f65
     uci: I/O error
  +  testing /opt/devel/openwrt/c/uci/tests/cram/../fuzz/corpus/bb589d0621e5472f470fa3425a234c74b1e202e8
     uci: Parse error (invalid command) at line 0, byte 0
  +  testing /opt/devel/openwrt/c/uci/tests/cram/../fuzz/corpus/ea387894a296772f96706df8b999a52d9334c746
  +  testing /opt/devel/openwrt/c/uci/tests/cram/../fuzz/corpus/id-000000,sig-06,src-000079,time-22005942,op-ext_AO,pos-8
     uci: Parse error (invalid command) at line 1, byte 18
  +  testing /opt/devel/openwrt/c/uci/tests/cram/../fuzz/corpus/id:000000,sig:11,src:000001,op:flip1,pos:24
     uci: I/O error
  +  testing /opt/devel/openwrt/c/uci/tests/cram/../fuzz/corpus/id:000008,sig:11,src:000022,op:arith8,pos:42,val:+26
     uci: I/O error
     [1]
  ..!

And it seems correct when run manually:

  ynezz@ntbk:/opt/devel/openwrt/c-projects/uci/build/tests/cram$ ../../uci import -f ../../../tests/fuzz/corpus/id-000000,sig-06,src-000079,time-22005942,op-ext_AO,pos-8
  ../../uci: Parse error (invalid command) at line 1, byte 18

Cheers,

Petr
Hauke Mehrtens Oct. 5, 2020, 6:21 p.m. UTC | #3 
On 10/5/20 2:03 PM, Petr Štetiar wrote:
> Hauke Mehrtens <hauke@hauke-m.de> [2020-10-04 17:10:59]:
> 
>> On 10/3/20 9:48 AM, Petr Štetiar wrote:
>>> Use valgrind and uci cli compiled with undefined, address and leak
>>> sanitizers.
>> ....
>>> diff --git a/tests/cram/test-san_uci_import.t b/tests/cram/test-san_uci_import.t
>>> new file mode 100644
>>> index 000000000000..7faed221f93e
>>> --- /dev/null
>>> +++ b/tests/cram/test-san_uci_import.t
>>> @@ -0,0 +1,15 @@
>>> +check that uci import is producing expected results:
>>> +
>>> +  $ [ -n "$BUILD_BIN_DIR" ] && export PATH="$BUILD_BIN_DIR:$PATH"
>>> +  $ export TEST_INPUTS="$TESTDIR/inputs"
>>> +  $ export FUZZ_CORPUS="$TESTDIR/../fuzz/corpus"
>>> +
>>> +  $ for file in $(LC_ALL=C find $FUZZ_CORPUS -type f | sort ); do
>>> +  >   uci-san import -f $file; \
>>> +  > done
> 
> BTW this above is a "standard" multiline shell construct (> prefix), so you
> can just add simple "echo $file" and see what is going on, like this:
> 
>  diff --git a/tests/cram/test_uci_import.t b/tests/cram/test_uci_import.t
>  index 8d5ab74ed4e7..e086ecf11ab1 100644
>  --- a/tests/cram/test_uci_import.t
>  +++ b/tests/cram/test_uci_import.t
>  @@ -5,6 +5,7 @@ check that uci import is producing expected results:
>     $ export FUZZ_CORPUS="$TESTDIR/../fuzz/corpus"
>  
>     $ for file in $(LC_ALL=C find $FUZZ_CORPUS -type f | sort ); do
>  +  >   echo "testing $file"; \
>     >   valgrind --quiet --leak-check=full uci import -f $file; \
>     > done
>     uci: I/O error
> 
>>> +  uci-san: I/O error
>>> +  uci-san: Parse error (invalid command) at line 0, byte 0
>>> +  uci-san: Parse error (invalid command) at line 1, byte 18
>>
>> I do not get this error. Which test case should trigger it?
> 
> That's quite strange as I'm getting consistent results with gcc 8,9,10 and
> clang 10 (the same version used on CI
> https://gitlab.com/ynezz/openwrt-uci/-/jobs/771521825).  Both cram/shuni2 test
> suites are run always for complete compiler matrix, fuzzer only under clang
> compiler for obvious reasons.

It looks like this file was not correctly added when I imported your
patch from patchwork:
tests/fuzz/corpus/id-000000,sig-06,src-000079,time-22005942,op-ext_AO,pos-8

I used the changes you committed to master and the tests are passing for me.

clang 7 is the default clang version in Debian stable. I do not want to
update so often. ;-)

Hauke
diff mbox series

Patch

diff --git a/tests/cram/CMakeLists.txt b/tests/cram/CMakeLists.txt
index 06c7c9419c7e..175c0671a558 100644
--- a/tests/cram/CMakeLists.txt
+++ b/tests/cram/CMakeLists.txt
@@ -1,6 +1,10 @@ 
 FIND_PACKAGE(PythonInterp 3 REQUIRED)
 FILE(GLOB test_cases "test_*.t")
 
+IF(CMAKE_C_COMPILER_ID STREQUAL "Clang")
+  FILE(GLOB test_cases_san "test-san_*.t")
+ENDIF()
+
 SET(PYTHON_VENV_DIR "${CMAKE_CURRENT_BINARY_DIR}/.venv")
 SET(PYTHON_VENV_PIP "${PYTHON_VENV_DIR}/bin/pip")
 SET(PYTHON_VENV_CRAM "${PYTHON_VENV_DIR}/bin/cram")
@@ -14,8 +18,9 @@  ADD_CUSTOM_TARGET(prepare-cram-venv ALL DEPENDS ${PYTHON_VENV_CRAM})
 
 ADD_TEST(
 	NAME cram
-	COMMAND ${PYTHON_VENV_CRAM} ${test_cases}
+	COMMAND ${PYTHON_VENV_CRAM} ${test_cases} ${test_cases_san}
 	WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
 )
 
 SET_PROPERTY(TEST cram APPEND PROPERTY ENVIRONMENT "UCI_LUA=$<TARGET_FILE:uci_lua>")
+SET_PROPERTY(TEST cram APPEND PROPERTY ENVIRONMENT "BUILD_BIN_DIR=$<TARGET_FILE_DIR:uci>")
diff --git a/tests/cram/test-san_uci_import.t b/tests/cram/test-san_uci_import.t
new file mode 100644
index 000000000000..7faed221f93e
--- /dev/null
+++ b/tests/cram/test-san_uci_import.t
@@ -0,0 +1,15 @@ 
+check that uci import is producing expected results:
+
+  $ [ -n "$BUILD_BIN_DIR" ] && export PATH="$BUILD_BIN_DIR:$PATH"
+  $ export TEST_INPUTS="$TESTDIR/inputs"
+  $ export FUZZ_CORPUS="$TESTDIR/../fuzz/corpus"
+
+  $ for file in $(LC_ALL=C find $FUZZ_CORPUS -type f | sort ); do
+  >   uci-san import -f $file; \
+  > done
+  uci-san: I/O error
+  uci-san: Parse error (invalid command) at line 0, byte 0
+  uci-san: Parse error (invalid command) at line 1, byte 18
+  uci-san: I/O error
+  uci-san: I/O error
+  [1]
diff --git a/tests/cram/test_uci_import.t b/tests/cram/test_uci_import.t
new file mode 100644
index 000000000000..8d5ab74ed4e7
--- /dev/null
+++ b/tests/cram/test_uci_import.t
@@ -0,0 +1,15 @@ 
+check that uci import is producing expected results:
+
+  $ [ -n "$BUILD_BIN_DIR" ] && export PATH="$BUILD_BIN_DIR:$PATH"
+  $ export TEST_INPUTS="$TESTDIR/inputs"
+  $ export FUZZ_CORPUS="$TESTDIR/../fuzz/corpus"
+
+  $ for file in $(LC_ALL=C find $FUZZ_CORPUS -type f | sort ); do
+  >   valgrind --quiet --leak-check=full uci import -f $file; \
+  > done
+  uci: I/O error
+  uci: Parse error (invalid command) at line 0, byte 0
+  uci: Parse error (invalid command) at line 1, byte 18
+  uci: I/O error
+  uci: I/O error
+  [1]