diff mbox series

[ACT] UBUNTU: SAUCE: ubuntu_ltp: improve test granularity

Message ID 20211202045806.23809-1-po-hsu.lin@canonical.com
State New
Headers show
Series [ACT] UBUNTU: SAUCE: ubuntu_ltp: improve test granularity | expand

Commit Message

Po-Hsu Lin Dec. 2, 2021, 4:58 a.m. UTC
Improve the test granularity by:
  * Running tests from different subsets one-by-one. The test case
    name will be in a format of category:sub-test , e.g. mm:mtest01
  * Remove the blacklist "skip" file, which was feed to runltp
    directly, integrate the list into test iteration loop
  * Remove the summary section

This change will make the test number change from 7 (with 7 sub-sets)
to around 280 cases. It will be more reliable and easier to hint.

Commit tested on dev jenkins with Focal kernel on a KVM instance. The
test result looks good.

Signed-off-by: Po-Hsu Lin <po-hsu.lin@canonical.com>
---
 ubuntu_ltp/control       | 17 ++++++++--
 ubuntu_ltp/skip          |  6 ----
 ubuntu_ltp/ubuntu_ltp.py | 73 +++++++++++++---------------------------
 3 files changed, 38 insertions(+), 58 deletions(-)
 delete mode 100644 ubuntu_ltp/skip

Comments

Po-Hsu Lin Dec. 16, 2021, 7:20 a.m. UTC | #1
With Francis's feedback, missing LTP_TIMEOUT_MUL added for fs_fill ppc64le

Applied and pushed.

Thanks
Sam
diff mbox series

Patch

diff --git a/ubuntu_ltp/control b/ubuntu_ltp/control
index bd5968f4..b35fc23b 100644
--- a/ubuntu_ltp/control
+++ b/ubuntu_ltp/control
@@ -14,11 +14,22 @@  result = job.run_test_detail(NAME, test_name='setup', tag='setup', timeout=40*60
 if result == 'ERROR':
     print("ERROR: test failed to build, skipping all the sub tests")
 else:
-    tests = [
+    categories = [
         'fs',         'mm',           'commands',
         'cpuhotplug', 'net.ipv6_lib', 'cve',         'crypto'
     ]
-    for test in tests:
-        results = job.run_test_detail(NAME, test_name=test, tag=test, timeout=60*60*2)
+    blacklist = ['oom01', 'oom02', 'oom03', 'oom04' ,'oom05', 'mkswap01_sh'] # oom* (lp:1847963), mkswap01_sh (lp:1830584)
+    for category in categories:
+        fn = '/opt/ltp/runtest/' + category
+        with open(fn , 'r') as f:
+            for line in f:
+                if line.strip() and not line.startswith('#'):
+                    if line.split()[0] in blacklist:
+                        continue
+                    with open ('/tmp/target' , 'w') as t:
+                        t.write(line)
+                    testcase = '{}:{}'.format(category, line.split()[0])
+                    timeout_threshold = 60*60
+                    job.run_test_detail(NAME, test_name=testcase, tag=testcase, timeout=timeout_threshold)
 
 # vi:set ts=4 sw=4 expandtab syntax=python:
diff --git a/ubuntu_ltp/skip b/ubuntu_ltp/skip
deleted file mode 100644
index 0624078e..00000000
--- a/ubuntu_ltp/skip
+++ /dev/null
@@ -1,6 +0,0 @@ 
-oom01
-oom02
-oom03
-oom04
-oom05
-mkswap01_sh mkswap01.sh
diff --git a/ubuntu_ltp/ubuntu_ltp.py b/ubuntu_ltp/ubuntu_ltp.py
index c6ee9178..16256c45 100644
--- a/ubuntu_ltp/ubuntu_ltp.py
+++ b/ubuntu_ltp/ubuntu_ltp.py
@@ -5,9 +5,7 @@  import os
 import platform
 import re
 import shutil
-import time
 from autotest.client                        import test, utils
-from autotest.client.shared     import error
 
 class ubuntu_ltp(test.test):
     version = 1
@@ -87,8 +85,6 @@  class ubuntu_ltp(test.test):
         utils.make(nprocs)
         utils.make('install')
 
-        cmd = 'cat %s > /tmp/skip' % os.path.join(self.bindir, 'skip')
-        utils.system_output(cmd)
 
     # run_once
     #
@@ -97,50 +93,29 @@  class ubuntu_ltp(test.test):
     def run_once(self, test_name):
         if test_name == 'setup':
             return
-        fn = '/tmp/syscalls-' + time.strftime("%h%d-%H%M%S")
-        log_failed = fn + '.failed'
-        log_output = fn + '.output'
-
-        fn = '/opt/ltp/runtest/%s' % (test_name)
-
-        print("Setting LTP_TIMEOUT_MUL exceptions...")
-        print("Setting LTP_TIMEOUT_MUL=3 for cve-2018-1000204 / ioctl_sg01 (lp:1899413, lp:1936886, lp:1949934)")
-        timeout_cases = {'zram01': '5', 'ioctl_sg01': '3'}
-        if self.arch in ['ppc64le']:
-            print("Running on PowerPC, set timeout multiplier LTP_TIMEOUT_MUL=3 for fs_fill (lp:1878763)")
-            timeout_cases['fs_fill'] = '3'
-
-        with open(fn , 'r') as f:
-            for line in f:
-                if line.strip() and not line.startswith('#'):
-                    # Reset the timeout multiplier
-                    os.environ["LTP_TIMEOUT_MUL"] = '1'
-                    with open ('/tmp/target' , 'w') as t:
-                        t.write(line)
-
-                    for _case in timeout_cases:
-                        if _case in line:
-                            os.environ["LTP_TIMEOUT_MUL"] = timeout_cases[_case]
-                            break
-
-                    cmd = '/opt/ltp/runltp -f /tmp/target -S /tmp/skip -C %s -q -l %s -o %s -T /dev/null' % (log_failed, log_output, log_output)
-                    utils.run(cmd, ignore_status=True, verbose=False)
-                    # /dev/loop# creation will be taken care by the runltp
-
-
-        num_failed = sum(1 for line in open(log_failed))
-        print("== Test Suite Summary ==")
-        print("{} test cases failed".format(num_failed))
-
-        if num_failed > 0:
-            cmd = "awk '{print$1}' " + log_failed + " | sort | uniq | tr '\n' ' '"
-            failed_list = utils.system_output(cmd, retain_output=False, verbose=False)
-            print("Failed test cases : %s" % failed_list)
-
-        cmd = 'cat ' + log_output
-        utils.system_output(cmd, retain_output=True, verbose=False)
-
-        if num_failed > 0:
-            raise error.TestError('Test failed for ' + test_name)
+
+        test_case = test_name.split(':')[1]
+
+        if test_case == 'zram01':
+            print("Setting LTP_TIMEOUT_MUL=5 for zram01 test (lp:1897556)")
+            os.environ["LTP_TIMEOUT_MUL"] = '5'
+        elif test_case in ['cve-2018-1000204 ', 'ioctl_sg01']:
+            print("Setting LTP_TIMEOUT_MUL=3 for cve-2018-1000204 / ioctl_sg01 (lp:1899413, lp:1936886, lp:1949934)")
+            os.environ["LTP_TIMEOUT_MUL"] = '3'
+        elif test_case == 'fs_fill' and self.arch == 'ppc64le':
+            print("Setting LTP_TIMEOUT_MUL=3 for fs_fill on PowerPC")
+
+        cmd = '/opt/ltp/runltp -f /tmp/target -q -C /dev/null -l /dev/null -T /dev/null'
+        print(utils.system_output(cmd, verbose=False))
+        # /dev/loop# creation will be taken care by the runltp
+
+    def cleanup(self, test_name):
+        if test_name == 'setup':
+            return
+
+        # Restore the timeout multiplier
+        if 'LTP_TIMEOUT_MUL' in os.environ:
+            print("Restore timeout multiplier LTP_TIMEOUT_MUL back to default")
+            del os.environ["LTP_TIMEOUT_MUL"]
 
 # vi:set ts=4 sw=4 expandtab syntax=python: