diff mbox series

[v2,23/30] fuzzy_sync: Convert to runtime

Message ID 20220503174718.21205-24-chrubis@suse.cz
State Superseded
Headers show
Series Introduce runtime and conver tests | expand

Commit Message

Cyril Hrubis May 3, 2022, 5:47 p.m. UTC
The runtime for fuzzy sync tests are computed as timeout * exec_time_p.
The default timeout was 300 and the default was 0.5.

CC: Richard Palethorpe <rpalethorpe@suse.com>
Signed-off-by: Cyril Hrubis <chrubis@suse.cz>
---
 include/tst_fuzzy_sync.h                      | 19 ++++---------------
 testcases/kernel/crypto/af_alg07.c            |  1 +
 testcases/kernel/mem/thp/thp04.c              |  1 +
 testcases/kernel/pty/pty03.c                  |  2 +-
 testcases/kernel/pty/pty05.c                  |  1 +
 testcases/kernel/pty/pty06.c                  |  1 +
 testcases/kernel/pty/pty07.c                  |  1 +
 testcases/kernel/sound/snd_seq01.c            |  2 +-
 testcases/kernel/sound/snd_timer01.c          |  1 +
 testcases/kernel/syscalls/bind/bind06.c       |  2 +-
 testcases/kernel/syscalls/inotify/inotify09.c |  1 +
 .../kernel/syscalls/ipc/shmctl/shmctl05.c     |  2 +-
 testcases/kernel/syscalls/sendmsg/sendmsg03.c |  1 +
 .../kernel/syscalls/setsockopt/setsockopt06.c |  2 +-
 .../kernel/syscalls/setsockopt/setsockopt07.c |  1 +
 .../syscalls/timerfd/timerfd_settime02.c      |  1 +
 testcases/kernel/syscalls/writev/writev03.c   |  2 +-
 17 files changed, 20 insertions(+), 21 deletions(-)

Comments

Petr Vorel May 3, 2022, 9:30 p.m. UTC | #1
Hi Cyril,

> The runtime for fuzzy sync tests are computed as timeout * exec_time_p.
> The default timeout was 300 and the default was 0.5.

> CC: Richard Palethorpe <rpalethorpe@suse.com>
> Signed-off-by: Cyril Hrubis <chrubis@suse.cz>
> ---
>  include/tst_fuzzy_sync.h                      | 19 ++++---------------
>  testcases/kernel/crypto/af_alg07.c            |  1 +
>  testcases/kernel/mem/thp/thp04.c              |  1 +
>  testcases/kernel/pty/pty03.c                  |  2 +-
>  testcases/kernel/pty/pty05.c                  |  1 +
>  testcases/kernel/pty/pty06.c                  |  1 +
>  testcases/kernel/pty/pty07.c                  |  1 +
>  testcases/kernel/sound/snd_seq01.c            |  2 +-
>  testcases/kernel/sound/snd_timer01.c          |  1 +
>  testcases/kernel/syscalls/bind/bind06.c       |  2 +-
>  testcases/kernel/syscalls/inotify/inotify09.c |  1 +
>  .../kernel/syscalls/ipc/shmctl/shmctl05.c     |  2 +-
>  testcases/kernel/syscalls/sendmsg/sendmsg03.c |  1 +
>  .../kernel/syscalls/setsockopt/setsockopt06.c |  2 +-
>  .../kernel/syscalls/setsockopt/setsockopt07.c |  1 +
>  .../syscalls/timerfd/timerfd_settime02.c      |  1 +
>  testcases/kernel/syscalls/writev/writev03.c   |  2 +-
>  17 files changed, 20 insertions(+), 21 deletions(-)

FYI you haven't converted fuzzy sync tests.
I suppose these errors must be fix in first commit.

./tst_fuzzy_sync01
tst_test.c:1522: TINFO: Timeout per run is 0h 00m 30s
tst_test.c:1483: TBROK: Runtime not set!

./tst_fuzzy_sync02
tst_test.c:1522: TINFO: Timeout per run is 0h 00m 30s
tst_test.c:1483: TBROK: Runtime not set!

./tst_fuzzy_sync03
tst_test.c:1522: TINFO: Timeout per run is 0h 00m 30s
tst_test.c:1483: TBROK: Runtime not set!
tst_fuzzy_sync03.c:45: TFAIL: Worker performed wrong number of iterations: 0 != 65535


Only tst_fuzzy_sync03.c is run in CI:
https://github.com/pevik/ltp/runs/6280658557?check_suite_focus=true
/__w/ltp/ltp/lib/tst_test.c:1483: TBROK: Runtime not set!
/__w/ltp/ltp/lib/newlib_tests/tst_fuzzy_sync03.c:45: TFAIL: Worker performed wrong number of iterations: 0 != 65535

maybe after fix the other two might be possible to run in CI as well.

Kind regards,
Petr
Richard Palethorpe May 5, 2022, 8:47 a.m. UTC | #2
Hello Cyril,

Cyril Hrubis <chrubis@suse.cz> writes:

> The runtime for fuzzy sync tests are computed as timeout * exec_time_p.
> The default timeout was 300 and the default was 0.5.
>
> CC: Richard Palethorpe <rpalethorpe@suse.com>
> Signed-off-by: Cyril Hrubis <chrubis@suse.cz>
> ---
>  include/tst_fuzzy_sync.h                      | 19
> ++++---------------

The fzsync lib/newlib_tests validation tests probably need more than the
default 5 seconds.

Otherwise LGTM
Richard Palethorpe May 5, 2022, 8:51 a.m. UTC | #3
Hi,

Petr Vorel <pvorel@suse.cz> writes:

> Hi Cyril,
>
>> The runtime for fuzzy sync tests are computed as timeout * exec_time_p.
>> The default timeout was 300 and the default was 0.5.
>
>> CC: Richard Palethorpe <rpalethorpe@suse.com>
>> Signed-off-by: Cyril Hrubis <chrubis@suse.cz>
>> ---
>>  include/tst_fuzzy_sync.h                      | 19 ++++---------------
>>  testcases/kernel/crypto/af_alg07.c            |  1 +
>>  testcases/kernel/mem/thp/thp04.c              |  1 +
>>  testcases/kernel/pty/pty03.c                  |  2 +-
>>  testcases/kernel/pty/pty05.c                  |  1 +
>>  testcases/kernel/pty/pty06.c                  |  1 +
>>  testcases/kernel/pty/pty07.c                  |  1 +
>>  testcases/kernel/sound/snd_seq01.c            |  2 +-
>>  testcases/kernel/sound/snd_timer01.c          |  1 +
>>  testcases/kernel/syscalls/bind/bind06.c       |  2 +-
>>  testcases/kernel/syscalls/inotify/inotify09.c |  1 +
>>  .../kernel/syscalls/ipc/shmctl/shmctl05.c     |  2 +-
>>  testcases/kernel/syscalls/sendmsg/sendmsg03.c |  1 +
>>  .../kernel/syscalls/setsockopt/setsockopt06.c |  2 +-
>>  .../kernel/syscalls/setsockopt/setsockopt07.c |  1 +
>>  .../syscalls/timerfd/timerfd_settime02.c      |  1 +
>>  testcases/kernel/syscalls/writev/writev03.c   |  2 +-
>>  17 files changed, 20 insertions(+), 21 deletions(-)
>
> FYI you haven't converted fuzzy sync tests.
> I suppose these errors must be fix in first commit.
>
> ./tst_fuzzy_sync01
> tst_test.c:1522: TINFO: Timeout per run is 0h 00m 30s
> tst_test.c:1483: TBROK: Runtime not set!

Ah, I see the missing runtime actually causes an error. I would have
expected it to use a default 5 seconds?
Cyril Hrubis May 5, 2022, 10:05 a.m. UTC | #4
Hi!
> Ah, I see the missing runtime actually causes an error. I would have
> expected it to use a default 5 seconds?

Yes I think that having the runtime explicitly written down is better
than some percentage of some value somewhere.

I.e. for most of the fuzzy sync tests what should be done is to
callibrate how long does the test need on an average hardware and put
that number in.
diff mbox series

Patch

diff --git a/include/tst_fuzzy_sync.h b/include/tst_fuzzy_sync.h
index 4f09ed416..bef424002 100644
--- a/include/tst_fuzzy_sync.h
+++ b/include/tst_fuzzy_sync.h
@@ -160,15 +160,6 @@  struct tst_fzsync_pair {
 	int b_cntr;
 	/** Internal; Used by tst_fzsync_pair_exit() and fzsync_pair_wait() */
 	int exit;
-	/**
-	 * The maximum desired execution time as a proportion of the timeout
-	 *
-	 * A value x so that 0 < x < 1 which decides how long the test should
-	 * be run for (assuming the loop limit is not exceeded first).
-	 *
-	 * Defaults to 0.5 (~150 seconds with default timeout).
-	 */
-	float exec_time_p;
 	/** Internal; The test time remaining on tst_fzsync_pair_reset() */
 	float exec_time_start;
 	/**
@@ -214,7 +205,6 @@  static inline void tst_fzsync_pair_init(struct tst_fzsync_pair *pair)
 	CHK(avg_alpha, 0, 1, 0.25);
 	CHK(min_samples, 20, INT_MAX, 1024);
 	CHK(max_dev_ratio, 0, 1, 0.1);
-	CHK(exec_time_p, 0, 1, 0.5);
 	CHK(exec_loops, 20, INT_MAX, 3000000);
 
 	if (tst_ncpus_available() <= 1)
@@ -291,7 +281,7 @@  static inline void tst_fzsync_pair_reset(struct tst_fzsync_pair *pair,
 	if (run_b)
 		SAFE_PTHREAD_CREATE(&pair->thread_b, 0, run_b, 0);
 
-	pair->exec_time_start = (float)tst_timeout_remaining();
+	pair->exec_time_start = (float)tst_remaining_runtime();
 }
 
 /**
@@ -644,10 +634,9 @@  static inline void tst_fzsync_wait_b(struct tst_fzsync_pair *pair)
  */
 static inline int tst_fzsync_run_a(struct tst_fzsync_pair *pair)
 {
-	float rem_p = 1 - tst_timeout_remaining() / pair->exec_time_start;
+	float rem_p = 1 - tst_remaining_runtime() / pair->exec_time_start;
 
-	if ((pair->exec_time_p * SAMPLING_SLICE < rem_p)
-		&& (pair->sampling > 0)) {
+	if ((SAMPLING_SLICE < rem_p) && (pair->sampling > 0)) {
 		tst_res(TINFO, "Stopped sampling at %d (out of %d) samples, "
 			"sampling time reached 50%% of the total time limit",
 			pair->exec_loop, pair->min_samples);
@@ -655,7 +644,7 @@  static inline int tst_fzsync_run_a(struct tst_fzsync_pair *pair)
 		tst_fzsync_pair_info(pair);
 	}
 
-	if (pair->exec_time_p < rem_p) {
+	if (rem_p >= 1) {
 		tst_res(TINFO,
 			"Exceeded execution time, requesting exit");
 		tst_atomic_store(1, &pair->exit);
diff --git a/testcases/kernel/crypto/af_alg07.c b/testcases/kernel/crypto/af_alg07.c
index ef13ad764..4754c501e 100644
--- a/testcases/kernel/crypto/af_alg07.c
+++ b/testcases/kernel/crypto/af_alg07.c
@@ -125,6 +125,7 @@  static struct tst_test test = {
 	.cleanup = cleanup,
 	.min_kver = "4.10.0",
 	.min_cpus = 2,
+	.max_iteration_runtime = 150,
 	.taint_check = TST_TAINT_W | TST_TAINT_D,
 	.tags = (const struct tst_tag[]) {
 		{"linux-git", "ff7b11aa481f"},
diff --git a/testcases/kernel/mem/thp/thp04.c b/testcases/kernel/mem/thp/thp04.c
index 985394dc3..77f49e777 100644
--- a/testcases/kernel/mem/thp/thp04.c
+++ b/testcases/kernel/mem/thp/thp04.c
@@ -163,6 +163,7 @@  static struct tst_test test = {
 	.test_all = run,
 	.setup = setup,
 	.cleanup = cleanup,
+	.max_iteration_runtime = 150,
 	.tags = (const struct tst_tag[]) {
 		{"linux-git", "a8f97366452e"},
 		{"linux-git", "8310d48b125d"},
diff --git a/testcases/kernel/pty/pty03.c b/testcases/kernel/pty/pty03.c
index 71bcb2eb6..f5680e7c5 100644
--- a/testcases/kernel/pty/pty03.c
+++ b/testcases/kernel/pty/pty03.c
@@ -135,7 +135,6 @@  static void do_test(unsigned int n)
 static void setup(void)
 {
 	fzp.min_samples = 20;
-	fzp.exec_time_p = 0.1;
 
 	tst_fzsync_pair_init(&fzp);
 }
@@ -151,6 +150,7 @@  static struct tst_test test = {
 	.setup = setup,
 	.cleanup = cleanup,
 	.needs_root = 1,
+	.max_iteration_runtime = 30,
 	.tags = (const struct tst_tag[]) {
 		{"linux-git", "0ace17d568241"},
 		{"CVE", "2020-14416"},
diff --git a/testcases/kernel/pty/pty05.c b/testcases/kernel/pty/pty05.c
index afef051c8..72a2f2c36 100644
--- a/testcases/kernel/pty/pty05.c
+++ b/testcases/kernel/pty/pty05.c
@@ -97,6 +97,7 @@  static struct tst_test test = {
 	.setup = setup,
 	.cleanup = cleanup,
 	.taint_check = TST_TAINT_W | TST_TAINT_D,
+	.max_iteration_runtime = 150,
 	.tags = (const struct tst_tag[]) {
 		{"linux-git", "82f2341c94d27"},
 		{"CVE", "2017-2636"},
diff --git a/testcases/kernel/pty/pty06.c b/testcases/kernel/pty/pty06.c
index cc8eb1f85..37edbc981 100644
--- a/testcases/kernel/pty/pty06.c
+++ b/testcases/kernel/pty/pty06.c
@@ -97,6 +97,7 @@  static struct tst_test test = {
 	.cleanup = cleanup,
 	.needs_root = 1,
 	.taint_check = TST_TAINT_W | TST_TAINT_D,
+	.max_iteration_runtime = 150,
 	.tags = (const struct tst_tag[]) {
 	    { "linux-git", "ca4463bf8438"},
 	    {}
diff --git a/testcases/kernel/pty/pty07.c b/testcases/kernel/pty/pty07.c
index c63b71c89..0190b581d 100644
--- a/testcases/kernel/pty/pty07.c
+++ b/testcases/kernel/pty/pty07.c
@@ -110,6 +110,7 @@  static struct tst_test test = {
 	.cleanup = cleanup,
 	.needs_root = 1,
 	.taint_check = TST_TAINT_W | TST_TAINT_D,
+	.max_iteration_runtime = 150,
 	.tags = (const struct tst_tag[]) {
 		{ "linux-git", "6cd1ed50efd8"},
 		{}
diff --git a/testcases/kernel/sound/snd_seq01.c b/testcases/kernel/sound/snd_seq01.c
index c56752230..3ce9550c7 100644
--- a/testcases/kernel/sound/snd_seq01.c
+++ b/testcases/kernel/sound/snd_seq01.c
@@ -123,7 +123,7 @@  static struct tst_test test = {
 	.tcnt = ARRAY_SIZE(testfunc_list),
 	.setup = setup,
 	.cleanup = cleanup,
-	.timeout = 120,
+	.max_iteration_runtime = 60,
 	.taint_check = TST_TAINT_W | TST_TAINT_D,
 	.tags = (const struct tst_tag[]) {
 		{"linux-git", "d15d662e89fc"},
diff --git a/testcases/kernel/sound/snd_timer01.c b/testcases/kernel/sound/snd_timer01.c
index 51591c18e..4c46c966f 100644
--- a/testcases/kernel/sound/snd_timer01.c
+++ b/testcases/kernel/sound/snd_timer01.c
@@ -139,6 +139,7 @@  static struct tst_test test = {
 	.setup = setup,
 	.cleanup = cleanup,
 	.taint_check = TST_TAINT_W | TST_TAINT_D,
+	.max_iteration_runtime = 150,
 	.tags = (const struct tst_tag[]) {
 		{"linux-git", "d11662f4f798"},
 		{"linux-git", "ba3021b2c79b"},
diff --git a/testcases/kernel/syscalls/bind/bind06.c b/testcases/kernel/syscalls/bind/bind06.c
index 1dfc534a7..a2c1d67c1 100644
--- a/testcases/kernel/syscalls/bind/bind06.c
+++ b/testcases/kernel/syscalls/bind/bind06.c
@@ -102,7 +102,7 @@  static struct tst_test test = {
 	.test_all = run,
 	.setup = setup,
 	.cleanup = cleanup,
-	.timeout = 600,
+	.max_iteration_runtime = 300,
 	.taint_check = TST_TAINT_W | TST_TAINT_D,
 	.needs_kconfigs = (const char *[]) {
 		"CONFIG_USER_NS=y",
diff --git a/testcases/kernel/syscalls/inotify/inotify09.c b/testcases/kernel/syscalls/inotify/inotify09.c
index fdfc9c078..8364a9e6a 100644
--- a/testcases/kernel/syscalls/inotify/inotify09.c
+++ b/testcases/kernel/syscalls/inotify/inotify09.c
@@ -94,6 +94,7 @@  static struct tst_test test = {
 	.setup = setup,
 	.cleanup = cleanup,
 	.test_all = verify_inotify,
+	.max_iteration_runtime = 150,
 	.tags = (const struct tst_tag[]) {
 		{"linux-git", "d90a10e2444b"},
 		{}
diff --git a/testcases/kernel/syscalls/ipc/shmctl/shmctl05.c b/testcases/kernel/syscalls/ipc/shmctl/shmctl05.c
index 228174c9b..2a7dd3c0f 100644
--- a/testcases/kernel/syscalls/ipc/shmctl/shmctl05.c
+++ b/testcases/kernel/syscalls/ipc/shmctl/shmctl05.c
@@ -106,7 +106,7 @@  static void cleanup(void)
 }
 
 static struct tst_test test = {
-	.timeout = 20,
+	.max_iteration_runtime = 10,
 	.setup = setup,
 	.test_all = do_test,
 	.cleanup = cleanup,
diff --git a/testcases/kernel/syscalls/sendmsg/sendmsg03.c b/testcases/kernel/syscalls/sendmsg/sendmsg03.c
index c7d72f686..fb9182dec 100644
--- a/testcases/kernel/syscalls/sendmsg/sendmsg03.c
+++ b/testcases/kernel/syscalls/sendmsg/sendmsg03.c
@@ -105,6 +105,7 @@  static struct tst_test test = {
 	.setup = setup,
 	.cleanup = cleanup,
 	.taint_check = TST_TAINT_W | TST_TAINT_D,
+	.max_iteration_runtime = 150,
 	.tags = (const struct tst_tag[]) {
 		{"linux-git", "8f659a03a0ba"},
 		{"CVE", "2017-17712"},
diff --git a/testcases/kernel/syscalls/setsockopt/setsockopt06.c b/testcases/kernel/syscalls/setsockopt/setsockopt06.c
index c6fc2fba9..dafb9665f 100644
--- a/testcases/kernel/syscalls/setsockopt/setsockopt06.c
+++ b/testcases/kernel/syscalls/setsockopt/setsockopt06.c
@@ -46,7 +46,6 @@  static void setup(void)
 	SAFE_FILE_PRINTF("/proc/self/gid_map", "0 %d 1", real_gid);
 
 	fzsync_pair.exec_loops = 100000;
-	fzsync_pair.exec_time_p = 0.9;
 	tst_fzsync_pair_init(&fzsync_pair);
 }
 
@@ -123,6 +122,7 @@  static struct tst_test test = {
 	.test_all = run,
 	.setup = setup,
 	.cleanup = cleanup,
+	.max_iteration_runtime = 270,
 	.taint_check = TST_TAINT_W | TST_TAINT_D,
 	.needs_kconfigs = (const char *[]) {
 		"CONFIG_USER_NS=y",
diff --git a/testcases/kernel/syscalls/setsockopt/setsockopt07.c b/testcases/kernel/syscalls/setsockopt/setsockopt07.c
index 147ae3899..cde0ea2d4 100644
--- a/testcases/kernel/syscalls/setsockopt/setsockopt07.c
+++ b/testcases/kernel/syscalls/setsockopt/setsockopt07.c
@@ -136,6 +136,7 @@  static struct tst_test test = {
 	.test_all = run,
 	.setup = setup,
 	.cleanup = cleanup,
+	.max_iteration_runtime = 150,
 	.needs_kconfigs = (const char *[]) {
 		"CONFIG_USER_NS=y",
 		"CONFIG_NET_NS=y",
diff --git a/testcases/kernel/syscalls/timerfd/timerfd_settime02.c b/testcases/kernel/syscalls/timerfd/timerfd_settime02.c
index bd92ee964..327a1b8e5 100644
--- a/testcases/kernel/syscalls/timerfd/timerfd_settime02.c
+++ b/testcases/kernel/syscalls/timerfd/timerfd_settime02.c
@@ -112,6 +112,7 @@  static struct tst_test test = {
 	.cleanup = cleanup,
 	.min_kver = "2.6.25",
 	.taint_check = TST_TAINT_W | TST_TAINT_D,
+	.max_iteration_runtime = 150,
 	.tags = (const struct tst_tag[]) {
 		{"linux-git", "1e38da300e1e"},
 		{"CVE", "2017-10661"},
diff --git a/testcases/kernel/syscalls/writev/writev03.c b/testcases/kernel/syscalls/writev/writev03.c
index d0c64743b..467e2de6c 100644
--- a/testcases/kernel/syscalls/writev/writev03.c
+++ b/testcases/kernel/syscalls/writev/writev03.c
@@ -49,7 +49,6 @@  static void setup(void)
 	mapfd = SAFE_OPEN(MAPFILE, O_CREAT|O_RDWR|O_TRUNC, 0644);
 	SAFE_WRITE(1, mapfd, buf, BUF_SIZE);
 
-	fzsync_pair.exec_time_p = 0.25;
 	tst_fzsync_pair_init(&fzsync_pair);
 }
 
@@ -146,6 +145,7 @@  static struct tst_test test = {
 	.min_cpus = 2,
 	.setup = setup,
 	.cleanup = cleanup,
+	.max_iteration_runtime = 75,
 	.tags = (const struct tst_tag[]) {
 		{"linux-git", "d4690f1e1cda"},
 		{}