diff mbox series

[v5,13/24] tests/avocado: replay_linux.py remove the timeout expected guards

Message ID 20240318154621.2361161-14-npiggin@gmail.com
State New
Headers show
Series replay: fixes and new test cases | expand

Commit Message

Nicholas Piggin March 18, 2024, 3:46 p.m. UTC
replay_linux tests with virtio on aarch64 gciv3 and x86-64 q35 machines
seems to be more reliable now, so timeouts are no longer expected.
pc_i440fx, gciv2, and non-virtio still have problems, so mark them as
flaky: they are not just long-running, but can hang indefinitely.

These tests take about 400 seconds each, so add the SPEED=slow guard.

Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
 tests/avocado/replay_linux.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

Comments

Alex Bennée March 19, 2024, 5:57 p.m. UTC | #1
Nicholas Piggin <npiggin@gmail.com> writes:

> replay_linux tests with virtio on aarch64 gciv3 and x86-64 q35 machines
> seems to be more reliable now, so timeouts are no longer expected.
> pc_i440fx, gciv2, and non-virtio still have problems, so mark them as
> flaky: they are not just long-running, but can hang indefinitely.
>
> These tests take about 400 seconds each, so add the SPEED=slow guard.
>
> Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
> ---
>  tests/avocado/replay_linux.py | 10 +++++++---
>  1 file changed, 7 insertions(+), 3 deletions(-)
>
> diff --git a/tests/avocado/replay_linux.py b/tests/avocado/replay_linux.py
> index b3b91ddd9a..b3b74a367c 100644
> --- a/tests/avocado/replay_linux.py
> +++ b/tests/avocado/replay_linux.py
> @@ -118,7 +118,7 @@ def run_replay_dump(self, replay_path):
>          except subprocess.CalledProcessError:
>              self.fail('replay-dump.py failed')
>  
> -@skipUnless(os.getenv('AVOCADO_TIMEOUT_EXPECTED'), 'Test might timeout')
> +@skipUnless(os.getenv('SPEED') == 'slow', 'runtime limited')
>  class ReplayLinuxX8664(ReplayLinux):
>      """
>      :avocado: tags=arch:x86_64
> @@ -127,19 +127,21 @@ class ReplayLinuxX8664(ReplayLinux):
>  
>      chksum = 'e3c1b309d9203604922d6e255c2c5d098a309c2d46215d8fc026954f3c5c27a0'
>  
> +    @skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable')
>      def test_pc_i440fx(self):
>          """
>          :avocado: tags=machine:pc
>          """
>          self.run_rr(shift=1)
>  
> +    @skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable')
>      def test_pc_q35(self):
>          """
>          :avocado: tags=machine:q35
>          """
>          self.run_rr(shift=3)
>  
> -@skipUnless(os.getenv('AVOCADO_TIMEOUT_EXPECTED'), 'Test might timeout')
> +@skipUnless(os.getenv('SPEED') == 'slow', 'runtime limited')
>  class ReplayLinuxX8664Virtio(ReplayLinux):
>      """
>      :avocado: tags=arch:x86_64
> @@ -153,6 +155,7 @@ class ReplayLinuxX8664Virtio(ReplayLinux):
>  
>      chksum = 'e3c1b309d9203604922d6e255c2c5d098a309c2d46215d8fc026954f3c5c27a0'
>  
> +    @skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable')
>      def test_pc_i440fx(self):
>          """
>          :avocado: tags=machine:pc
> @@ -165,7 +168,7 @@ def test_pc_q35(self):
>          """
>          self.run_rr(shift=3)
>  
> -@skipUnless(os.getenv('AVOCADO_TIMEOUT_EXPECTED'), 'Test might timeout')
> +@skipUnless(os.getenv('SPEED') == 'slow', 'runtime limited')
>  class ReplayLinuxAarch64(ReplayLinux):
>      """
>      :avocado: tags=accel:tcg
> @@ -187,6 +190,7 @@ def get_common_args(self):
>                  '-device', 'virtio-rng-pci,rng=rng0',
>                  '-object', 'rng-builtin,id=rng0')
>  
> +    @skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is
> unstable')

This needs to apply to both I think:

(5/7) ./tests/avocado/replay_linux.py:ReplayLinuxAarch64.test_virt_gicv2: SKIP: Test is unstable
 (6/7)
 ./tests/avocado/replay_linux.py:ReplayLinuxAarch64.test_virt_gicv3:
 INTERRUPTED: Test interrupted by SIGTERM\nRunner error occurred:
 Timeout reached\nOriginal status: ERROR\n{'name':
 '6-./tests/avocado/replay_linux.py:ReplayLinuxAarch64.test_virt_gicv3',
 'logdir':
 '/home/alex/avocado/job-results/job-2024-03-19T16.50-686495d/test-results/...
 (1800.17 s)

With that:

Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
Nicholas Piggin March 20, 2024, 4:34 a.m. UTC | #2
On Wed Mar 20, 2024 at 3:57 AM AEST, Alex Bennée wrote:
> Nicholas Piggin <npiggin@gmail.com> writes:
>
> > replay_linux tests with virtio on aarch64 gciv3 and x86-64 q35 machines
> > seems to be more reliable now, so timeouts are no longer expected.
> > pc_i440fx, gciv2, and non-virtio still have problems, so mark them as
> > flaky: they are not just long-running, but can hang indefinitely.
> >
> > These tests take about 400 seconds each, so add the SPEED=slow guard.
> >
> > Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
> > ---
> >  tests/avocado/replay_linux.py | 10 +++++++---
> >  1 file changed, 7 insertions(+), 3 deletions(-)
> >
> > diff --git a/tests/avocado/replay_linux.py b/tests/avocado/replay_linux.py
> > index b3b91ddd9a..b3b74a367c 100644
> > --- a/tests/avocado/replay_linux.py
> > +++ b/tests/avocado/replay_linux.py
> > @@ -118,7 +118,7 @@ def run_replay_dump(self, replay_path):
> >          except subprocess.CalledProcessError:
> >              self.fail('replay-dump.py failed')
> >  
> > -@skipUnless(os.getenv('AVOCADO_TIMEOUT_EXPECTED'), 'Test might timeout')
> > +@skipUnless(os.getenv('SPEED') == 'slow', 'runtime limited')
> >  class ReplayLinuxX8664(ReplayLinux):
> >      """
> >      :avocado: tags=arch:x86_64
> > @@ -127,19 +127,21 @@ class ReplayLinuxX8664(ReplayLinux):
> >  
> >      chksum = 'e3c1b309d9203604922d6e255c2c5d098a309c2d46215d8fc026954f3c5c27a0'
> >  
> > +    @skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable')
> >      def test_pc_i440fx(self):
> >          """
> >          :avocado: tags=machine:pc
> >          """
> >          self.run_rr(shift=1)
> >  
> > +    @skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable')
> >      def test_pc_q35(self):
> >          """
> >          :avocado: tags=machine:q35
> >          """
> >          self.run_rr(shift=3)
> >  
> > -@skipUnless(os.getenv('AVOCADO_TIMEOUT_EXPECTED'), 'Test might timeout')
> > +@skipUnless(os.getenv('SPEED') == 'slow', 'runtime limited')
> >  class ReplayLinuxX8664Virtio(ReplayLinux):
> >      """
> >      :avocado: tags=arch:x86_64
> > @@ -153,6 +155,7 @@ class ReplayLinuxX8664Virtio(ReplayLinux):
> >  
> >      chksum = 'e3c1b309d9203604922d6e255c2c5d098a309c2d46215d8fc026954f3c5c27a0'
> >  
> > +    @skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable')
> >      def test_pc_i440fx(self):
> >          """
> >          :avocado: tags=machine:pc
> > @@ -165,7 +168,7 @@ def test_pc_q35(self):
> >          """
> >          self.run_rr(shift=3)
> >  
> > -@skipUnless(os.getenv('AVOCADO_TIMEOUT_EXPECTED'), 'Test might timeout')
> > +@skipUnless(os.getenv('SPEED') == 'slow', 'runtime limited')
> >  class ReplayLinuxAarch64(ReplayLinux):
> >      """
> >      :avocado: tags=accel:tcg
> > @@ -187,6 +190,7 @@ def get_common_args(self):
> >                  '-device', 'virtio-rng-pci,rng=rng0',
> >                  '-object', 'rng-builtin,id=rng0')
> >  
> > +    @skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is
> > unstable')
>
> This needs to apply to both I think:
>
> (5/7) ./tests/avocado/replay_linux.py:ReplayLinuxAarch64.test_virt_gicv2: SKIP: Test is unstable
>  (6/7)
>  ./tests/avocado/replay_linux.py:ReplayLinuxAarch64.test_virt_gicv3:
>  INTERRUPTED: Test interrupted by SIGTERM\nRunner error occurred:
>  Timeout reached\nOriginal status: ERROR\n{'name':
>  '6-./tests/avocado/replay_linux.py:ReplayLinuxAarch64.test_virt_gicv3',
>  'logdir':
>  '/home/alex/avocado/job-results/job-2024-03-19T16.50-686495d/test-results/...
>  (1800.17 s)

Oh, aarch64 is hanging for you too? It was passing for me... But I'll
guard it out for now and maybe we can get back to it later.

Thanks,
Nick

>
> With that:
>
> Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
diff mbox series

Patch

diff --git a/tests/avocado/replay_linux.py b/tests/avocado/replay_linux.py
index b3b91ddd9a..b3b74a367c 100644
--- a/tests/avocado/replay_linux.py
+++ b/tests/avocado/replay_linux.py
@@ -118,7 +118,7 @@  def run_replay_dump(self, replay_path):
         except subprocess.CalledProcessError:
             self.fail('replay-dump.py failed')
 
-@skipUnless(os.getenv('AVOCADO_TIMEOUT_EXPECTED'), 'Test might timeout')
+@skipUnless(os.getenv('SPEED') == 'slow', 'runtime limited')
 class ReplayLinuxX8664(ReplayLinux):
     """
     :avocado: tags=arch:x86_64
@@ -127,19 +127,21 @@  class ReplayLinuxX8664(ReplayLinux):
 
     chksum = 'e3c1b309d9203604922d6e255c2c5d098a309c2d46215d8fc026954f3c5c27a0'
 
+    @skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable')
     def test_pc_i440fx(self):
         """
         :avocado: tags=machine:pc
         """
         self.run_rr(shift=1)
 
+    @skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable')
     def test_pc_q35(self):
         """
         :avocado: tags=machine:q35
         """
         self.run_rr(shift=3)
 
-@skipUnless(os.getenv('AVOCADO_TIMEOUT_EXPECTED'), 'Test might timeout')
+@skipUnless(os.getenv('SPEED') == 'slow', 'runtime limited')
 class ReplayLinuxX8664Virtio(ReplayLinux):
     """
     :avocado: tags=arch:x86_64
@@ -153,6 +155,7 @@  class ReplayLinuxX8664Virtio(ReplayLinux):
 
     chksum = 'e3c1b309d9203604922d6e255c2c5d098a309c2d46215d8fc026954f3c5c27a0'
 
+    @skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable')
     def test_pc_i440fx(self):
         """
         :avocado: tags=machine:pc
@@ -165,7 +168,7 @@  def test_pc_q35(self):
         """
         self.run_rr(shift=3)
 
-@skipUnless(os.getenv('AVOCADO_TIMEOUT_EXPECTED'), 'Test might timeout')
+@skipUnless(os.getenv('SPEED') == 'slow', 'runtime limited')
 class ReplayLinuxAarch64(ReplayLinux):
     """
     :avocado: tags=accel:tcg
@@ -187,6 +190,7 @@  def get_common_args(self):
                 '-device', 'virtio-rng-pci,rng=rng0',
                 '-object', 'rng-builtin,id=rng0')
 
+    @skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable')
     def test_virt_gicv2(self):
         """
         :avocado: tags=machine:gic-version=2