#!/usr/bin/env python3 # Copyright (C) 2021 The Qt Company Ltd. # SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 import sys import os import re import glob import subprocess from subprocess import STDOUT, PIPE from tempfile import TemporaryDirectory, mkstemp MY_NAME = os.path.basename(__file__) my_dir = os.path.dirname(__file__) testrunner = os.path.join(my_dir, "..", "..", "..", "..", "util", "testrunner", "qt-testrunner.py") mock_test = os.path.join(my_dir, "qt_mock_test.py") xml_log_template = os.path.join(my_dir, "qt_mock_test-log.xml") with open(xml_log_template) as f: XML_TEMPLATE = f.read() import unittest def setUpModule(): global TEMPDIR TEMPDIR = TemporaryDirectory(prefix="tst_qt_testrunner-") global EMPTY_FILE EMPTY_FILE = os.path.join(TEMPDIR.name, "EMPTY") with open(EMPTY_FILE, "w") as f: pass filename = os.path.join(TEMPDIR.name, "file_1") print("setUpModule(): setting up temporary directory and env var" " QT_MOCK_TEST_STATE_FILE=" + filename + " and" " QT_MOCK_TEST_XML_TEMPLATE_FILE=" + xml_log_template) os.environ["QT_MOCK_TEST_STATE_FILE"] = filename os.environ["QT_MOCK_TEST_XML_TEMPLATE_FILE"] = xml_log_template os.environ["QT_TESTRUNNER_TESTING"] = "1" def tearDownModule(): print("\ntearDownModule(): Cleaning up temporary directory:", TEMPDIR.name) del os.environ["QT_MOCK_TEST_STATE_FILE"] TEMPDIR.cleanup() # Helper to run a command and always capture output def run(args : list, **kwargs): if args[0].endswith(".py"): # Make sure we run python executables with the same python version. # It also helps on Windows, that .py files are not directly executable. args = [ sys.executable, *args ] if DEBUG: print("Running: ", args, flush=True) proc = subprocess.run(args, stdout=PIPE, stderr=STDOUT, **kwargs) if DEBUG and proc.stdout: print(proc.stdout.decode(), flush=True) return proc # Helper to run qt-testrunner.py with proper testing arguments. # Always append --log-dir=TEMPDIR unless specifically told not to. def run_testrunner(xml_filename=None, log_dir=None, testrunner_args=None, wrapper_script=None, wrapper_args=None, qttest_args=None, env=None): args = [ testrunner ] if xml_filename: args += [ "--parse-xml-testlog", xml_filename ] if log_dir == None: args += [ "--log-dir", TEMPDIR.name ] elif log_dir != "": args += [ "--log-dir", log_dir ] if testrunner_args: args += testrunner_args if wrapper_script: args += [ wrapper_script ] if wrapper_args: args += wrapper_args args += [ mock_test ] if qttest_args: args += qttest_args return run(args, env=env) # Write the XML_TEMPLATE to filename, replacing the templated results. def write_xml_log(filename, failure=None, inject_message=None): data = XML_TEMPLATE if failure is None: failure = [] elif isinstance(failure, str): failure = [ failure ] # Replace what was asked to fail with "fail" for x in failure: data = data.replace("{{" + x + "_result}}", "fail") # Replace the rest with "pass" data = re.sub(r"{{[^}]+}}", "pass", data) # Inject possible tags inside the first if inject_message: i = data.index("") data = data[:i] + inject_message + data[i:] with open(filename, "w") as f: f.write(data) # Test that qt_mock_test.py behaves well. This is necessary to properly # test qt-testrunner. class Test_qt_mock_test(unittest.TestCase): def setUp(self): state_file = os.environ["QT_MOCK_TEST_STATE_FILE"] if os.path.exists(state_file): os.remove(state_file) def assertProcessCrashed(self, proc): if DEBUG: print("process returncode is:", proc.returncode) self.assertTrue(proc.returncode < 0 or proc.returncode >= 128) def test_always_pass(self): proc = run([mock_test, "always_pass"]) self.assertEqual(proc.returncode, 0) def test_always_fail(self): proc = run([mock_test, "always_fail"]) self.assertEqual(proc.returncode, 1) def test_fail_then_pass_2(self): proc = run([mock_test, "fail_then_pass:2"]) self.assertEqual(proc.returncode, 1) proc = run([mock_test, "fail_then_pass:2"]) self.assertEqual(proc.returncode, 1) proc = run([mock_test, "fail_then_pass:2"]) self.assertEqual(proc.returncode, 0) def test_fail_then_pass_1(self): proc = run([mock_test, "fail_then_pass:1"]) self.assertEqual(proc.returncode, 1) proc = run([mock_test, "fail_then_pass:1"]) self.assertEqual(proc.returncode, 0) def test_fail_then_pass_many_tests(self): proc = run([mock_test, "fail_then_pass:1"]) self.assertEqual(proc.returncode, 1) proc = run([mock_test, "fail_then_pass:2"]) self.assertEqual(proc.returncode, 1) proc = run([mock_test, "fail_then_pass:1"]) self.assertEqual(proc.returncode, 0) proc = run([mock_test, "fail_then_pass:2"]) self.assertEqual(proc.returncode, 1) proc = run([mock_test, "fail_then_pass:2"]) self.assertEqual(proc.returncode, 0) def test_fail_then_crash(self): proc = run([mock_test, "fail_then_crash"]) self.assertEqual(proc.returncode, 1) proc = run([mock_test, "fail_then_crash"]) self.assertProcessCrashed(proc) def test_xml_file_is_written(self): filename = os.path.join(TEMPDIR.name, "testlog.xml") proc = run([mock_test, "-o", filename+",xml"]) self.assertEqual(proc.returncode, 0) self.assertTrue(os.path.exists(filename)) self.assertGreater(os.path.getsize(filename), 0) os.remove(filename) # Test it will write an empty XML file if template is empty def test_empty_xml_file_is_written(self): my_env = { **os.environ, "QT_MOCK_TEST_XML_TEMPLATE_FILE": EMPTY_FILE } filename = os.path.join(TEMPDIR.name, "testlog.xml") proc = run([mock_test, "-o", filename+",xml"], env=my_env) self.assertEqual(proc.returncode, 0) self.assertTrue(os.path.exists(filename)) self.assertEqual(os.path.getsize(filename), 0) os.remove(filename) def test_crash_cleanly(self): proc = run([mock_test], env={ **os.environ, "QT_MOCK_TEST_CRASH_CLEANLY":"1" }) if DEBUG: print("returncode:", proc.returncode) self.assertProcessCrashed(proc) # Find in @path, files that start with @testname and end with @pattern, # where @pattern is a glob-like string. def find_test_logs(testname=None, path=None, pattern="-*[0-9].xml"): if testname is None: testname = os.path.basename(mock_test) if path is None: path = TEMPDIR.name pattern = os.path.join(path, testname + pattern) logfiles = glob.glob(pattern) if DEBUG: print(f"Test ({testname}) logfiles found: ", logfiles) return logfiles # Test regular invocations of qt-testrunner. class Test_testrunner(unittest.TestCase): def setUp(self): state_file = os.environ["QT_MOCK_TEST_STATE_FILE"] if os.path.exists(state_file): os.remove(state_file) # The mock_test honors only the XML output arguments, the rest are ignored. old_logfiles = find_test_logs(pattern="*.xml") for fname in old_logfiles: os.remove(os.path.join(TEMPDIR.name, fname)) self.env = dict(os.environ) self.testrunner_args = [] def prepare_env(self, run_list=None): if run_list is not None: self.env['QT_MOCK_TEST_RUN_LIST'] = ",".join(run_list) def run2(self): return run_testrunner(testrunner_args=self.testrunner_args, env=self.env) def test_simple_invocation(self): # All tests pass. proc = self.run2() self.assertEqual(proc.returncode, 0) def test_always_pass(self): self.prepare_env(run_list=["always_pass"]) proc = self.run2() self.assertEqual(proc.returncode, 0) def test_output_files_are_generated(self): proc = self.run2() xml_output_files = find_test_logs() self.assertEqual(len(xml_output_files), 1) def test_always_fail(self): self.prepare_env(run_list=["always_fail"]) proc = self.run2() # TODO verify that re-runs==max_repeats self.assertEqual(proc.returncode, 2) def test_flaky_pass_1(self): self.prepare_env(run_list=["always_pass,fail_then_pass:1"]) proc = self.run2() self.assertEqual(proc.returncode, 0) def test_flaky_pass_5(self): self.prepare_env(run_list=["always_pass,fail_then_pass:1,fail_then_pass:5"]) proc = self.run2() self.assertEqual(proc.returncode, 0) def test_flaky_fail(self): self.prepare_env(run_list=["always_pass,fail_then_pass:6"]) proc = self.run2() self.assertEqual(proc.returncode, 2) def test_flaky_pass_fail(self): self.prepare_env(run_list=["always_pass,fail_then_pass:1,fail_then_pass:6"]) proc = self.run2() # TODO verify that one func was re-run and passed but the other failed. self.assertEqual(proc.returncode, 2) def test_initTestCase_fail_crash(self): self.prepare_env(run_list=["initTestCase,always_pass"]) proc = self.run2() self.assertEqual(proc.returncode, 3) def test_fail_then_crash(self): self.prepare_env(run_list=["fail_then_crash"]) proc = self.run2() self.assertEqual(proc.returncode, 3) # By testing --no-extra-args, we ensure qt-testrunner works for # tst_selftests and the other NON_XML_GENERATING_TESTS. def test_no_extra_args_pass(self): self.testrunner_args += ["--no-extra-args"] proc = self.run2() self.assertEqual(proc.returncode, 0) def test_no_extra_args_fail(self): self.prepare_env(run_list=["always_fail"]) self.testrunner_args += ["--no-extra-args"] proc = self.run2() self.assertEqual(proc.returncode, 3) def test_no_extra_args_reruns_only_once_1(self): self.prepare_env(run_list=["fail_then_pass:1"]) self.testrunner_args += ["--no-extra-args"] proc = self.run2() # The 1st rerun PASSed. self.assertEqual(proc.returncode, 0) def test_no_extra_args_reruns_only_once_2(self): self.prepare_env(run_list=["fail_then_pass:2"]) self.testrunner_args += ["--no-extra-args"] proc = self.run2() # We never re-run more than once, so the exit code shows FAIL. self.assertEqual(proc.returncode, 3) # If no XML file is found by qt-testrunner, it is usually considered a # CRASH and the whole test is re-run. Even when the return code is zero. # It is a PASS only if the test is not capable of XML output (see no_extra_args above). def test_no_xml_log_written_pass_crash(self): del self.env["QT_MOCK_TEST_XML_TEMPLATE_FILE"] self.prepare_env(run_list=["always_pass"]) proc = self.run2() self.assertEqual(proc.returncode, 3) # On the 2nd iteration of the full test, both of the tests pass. # Still it's a CRASH because no XML file was found. def test_no_xml_log_written_fail_then_pass_crash(self): del self.env["QT_MOCK_TEST_XML_TEMPLATE_FILE"] self.prepare_env(run_list=["always_pass,fail_then_pass:1"]) proc = self.run2() # TODO verify that the whole test has run twice. self.assertEqual(proc.returncode, 3) # Even after 2 iterations of the full test we still get failures but no XML file, # and this is considered a CRASH. def test_no_xml_log_written_crash(self): del self.env["QT_MOCK_TEST_XML_TEMPLATE_FILE"] self.prepare_env(run_list=["fail_then_pass:2"]) proc = self.run2() self.assertEqual(proc.returncode, 3) def test_empty_xml_crash_1(self): self.env["QT_MOCK_TEST_XML_TEMPLATE_FILE"] = EMPTY_FILE self.prepare_env(run_list=["always_pass"]) proc = self.run2() self.assertEqual(proc.returncode, 3) def test_empty_xml_crash_2(self): self.env["QT_MOCK_TEST_XML_TEMPLATE_FILE"] = EMPTY_FILE self.prepare_env(run_list=["always_fail"]) proc = self.run2() self.assertEqual(proc.returncode, 3) # test qFatal should be a crash in all cases. def test_qfatal_crash_1(self): fatal_xml_message = """ """ logfile = os.path.join(TEMPDIR.name, os.path.basename(mock_test) + ".xml") write_xml_log(logfile, failure=None, inject_message=fatal_xml_message) del self.env["QT_MOCK_TEST_XML_TEMPLATE_FILE"] self.env["QT_TESTRUNNER_DEBUG_NO_UNIQUE_OUTPUT_FILENAME"] = "1" self.prepare_env(run_list=["always_pass"]) proc = self.run2() self.assertEqual(proc.returncode, 3) def test_qfatal_crash_2(self): fatal_xml_message = """ """ logfile = os.path.join(TEMPDIR.name, os.path.basename(mock_test) + ".xml") write_xml_log(logfile, failure="always_fail", inject_message=fatal_xml_message) del self.env["QT_MOCK_TEST_XML_TEMPLATE_FILE"] self.env["QT_TESTRUNNER_DEBUG_NO_UNIQUE_OUTPUT_FILENAME"] = "1" self.prepare_env(run_list=["always_pass,always_fail"]) proc = self.run2() self.assertEqual(proc.returncode, 3) def test_qwarn_is_ignored_1(self): qwarn_xml_message = """ """ logfile = os.path.join(TEMPDIR.name, os.path.basename(mock_test) + ".xml") write_xml_log(logfile, failure=None, inject_message=qwarn_xml_message) del self.env["QT_MOCK_TEST_XML_TEMPLATE_FILE"] self.env["QT_TESTRUNNER_DEBUG_NO_UNIQUE_OUTPUT_FILENAME"] = "1" self.prepare_env(run_list=["always_pass"]) proc = self.run2() self.assertEqual(proc.returncode, 0) def test_qwarn_is_ignored_2(self): fatal_xml_message = """ """ logfile = os.path.join(TEMPDIR.name, os.path.basename(mock_test) + ".xml") write_xml_log(logfile, failure=None, inject_message=fatal_xml_message) del self.env["QT_MOCK_TEST_XML_TEMPLATE_FILE"] self.env["QT_TESTRUNNER_DEBUG_NO_UNIQUE_OUTPUT_FILENAME"] = "1" self.prepare_env(run_list=["always_pass"]) proc = self.run2() self.assertEqual(proc.returncode, 3) # If a test returns success but XML contains failures, it's a CRASH. def test_wrong_xml_log_written_1_crash(self): logfile = os.path.join(TEMPDIR.name, os.path.basename(mock_test) + ".xml") write_xml_log(logfile, failure="always_fail") del self.env["QT_MOCK_TEST_XML_TEMPLATE_FILE"] self.env["QT_TESTRUNNER_DEBUG_NO_UNIQUE_OUTPUT_FILENAME"] = "1" self.prepare_env(run_list=["always_pass"]) proc = self.run2() self.assertEqual(proc.returncode, 3) # If a test returns failure but XML contains only pass, it's a CRASH. def test_wrong_xml_log_written_2_crash(self): logfile = os.path.join(TEMPDIR.name, os.path.basename(mock_test) + ".xml") write_xml_log(logfile) del self.env["QT_MOCK_TEST_XML_TEMPLATE_FILE"] self.env["QT_TESTRUNNER_DEBUG_NO_UNIQUE_OUTPUT_FILENAME"] = "1" self.prepare_env(run_list=["always_fail"]) proc = self.run2() self.assertEqual(proc.returncode, 3) def create_wrapper(self, filename, content=None): if not content: content='exec "$@"' filename = os.path.join(TEMPDIR.name, filename) with open(filename, "w") as f: f.write(f'#!/bin/sh\n{content}\n') self.wrapper_script = f.name os.chmod(self.wrapper_script, 0o700) # Test that it re-runs the full executable in case of crash, even if the # XML file is valid and shows one specific test failing. def test_crash_reruns_full_QTQAINFRA_5226(self): self.env["QT_MOCK_TEST_RUN_LIST"] = "always_fail" # Tell qt_mock_test to crash after writing a proper XML file. self.env["QT_MOCK_TEST_CRASH_CLEANLY"] = "1" proc = self.run2() # Verify qt-testrunner exited with 3 which means CRASH. self.assertEqual(proc.returncode, 3) # Verify that a full executable re-run happened that re-runs 2 times, # instead of individual functions that re-run 5 times. xml_output_files = find_test_logs() self.assertEqual(len(xml_output_files), 2) # Test that qt-testrunner detects the correct executable name even if we # use a special wrapper script, and that it uses that in the XML log filename. @unittest.skipUnless(os.name == "posix", "Wrapper script needs POSIX shell") def test_wrapper(self): self.create_wrapper("coin_vxworks_qemu_runner.sh") proc = run_testrunner(wrapper_script=self.wrapper_script, env=self.env) self.assertEqual(proc.returncode, 0) xml_output_files = find_test_logs() self.assertEqual(len(xml_output_files), 1) # The "androidtestrunner" wrapper is special. It expects the QTest arguments after "--". # So our mock androidtestrunner wrapper ignores everything before "--" # and executes our hardcoded mock_test with the arguments that follow. def create_mock_anroidtestrunner_wrapper(self): self.create_wrapper("androidtestrunner", content= 'while [ "$1" != "--" ]; do shift; done; shift; exec {} "$@"'.format(mock_test)) @unittest.skipUnless(os.name == "posix", "Wrapper script needs POSIX shell") def test_androidtestrunner_with_aab(self): self.create_mock_anroidtestrunner_wrapper() # Copied from our CI logs. The only relevant option is --aab. androidtestrunner_args= [ '--path', '/home/qt/work/qt/qtdeclarative_standalone_tests/tests/auto/quickcontrols/qquickpopup/android-build-tst_qquickpopup', '--adb', '/opt/android/sdk/platform-tools/adb', '--skip-install-root', '--ndk-stack', '/opt/android/android-ndk-r27c/ndk-stack', '--manifest', '/home/qt/work/qt/qtdeclarative_standalone_tests/tests/auto/quickcontrols/qquickpopup/android-build-tst_qquickpopup/app/AndroidManifest.xml', '--make', '"/opt/cmake-3.30.5/bin/cmake" --build /home/qt/work/qt/qtdeclarative_standalone_tests --target tst_qquickpopup_make_aab', '--aab', '/home/qt/work/qt/qtdeclarative_standalone_tests/tests/auto/quickcontrols/qquickpopup/android-build-tst_qquickpopup/tst_qquickpopup.aab', '--bundletool', '/opt/bundletool/bundletool', '--timeout', '1425' ] # In COIN CI, TESTRUNNER="qt-testrunner.py --". That's why we append "--". proc = run_testrunner(testrunner_args=["--"], wrapper_script=self.wrapper_script, wrapper_args=androidtestrunner_args, env=self.env) self.assertEqual(proc.returncode, 0) xml_output_files = find_test_logs("tst_qquickpopup") self.assertEqual(len(xml_output_files), 1) # similar to above but with "--apk" @unittest.skipUnless(os.name == "posix", "Wrapper script needs POSIX shell") def test_androidtestrunner_with_apk(self): self.create_mock_anroidtestrunner_wrapper() androidtestrunner_args= ['--blah', '--apk', '/whatever/waza.apk', 'blue'] proc = run_testrunner(testrunner_args=["--"], wrapper_script=self.wrapper_script, wrapper_args=androidtestrunner_args, env=self.env) self.assertEqual(proc.returncode, 0) xml_output_files = find_test_logs("waza") self.assertEqual(len(xml_output_files), 1) # similar to above but with neither "--apk" nor "--aab". qt-testrunner throws error. @unittest.skipUnless(os.name == "posix", "Wrapper script needs POSIX shell") def test_androidtestrunner_fail_to_detect_filename(self): self.create_mock_anroidtestrunner_wrapper() androidtestrunner_args= ['--blah', '--argh', '/whatever/waza.apk', 'waza.aab'] proc = run_testrunner(testrunner_args=["--"], wrapper_script=self.wrapper_script, wrapper_args=androidtestrunner_args, env=self.env) self.assertEqual(proc.returncode, 1) xml_output_files = find_test_logs("waza") self.assertEqual(len(xml_output_files), 0) # Test qt-testrunner script with an existing XML log file: # qt-testrunner.py qt_mock_test.py --parse-xml-testlog file.xml # qt-testrunner should repeat the testcases that are logged as # failures and fail or pass depending on how the testcases behave. # Different XML files are generated for the following test cases. # + No failure logged. qt-testrunner should exit(0) # + The "always_pass" test has failed. qt-testrunner should exit(0). # + The "always_fail" test has failed. qt-testrunner should exit(2). # + The "always_crash" test has failed. qt-testrunner should exit(3) # since the re-run will crash. # + The "fail_then_pass:2" test failed. qt-testrunner should exit(0). # + The "fail_then_pass:5" test failed. qt-testrunner should exit(2). # + The "initTestCase" failed which is listed as NO_RERUN thus # qt-testrunner should exit(3). class Test_testrunner_with_xml_logfile(unittest.TestCase): # Runs before every single test function, creating a unique temp file. def setUp(self): (_handle, self.xml_file) = mkstemp( suffix=".xml", prefix="qt_mock_test-log-", dir=TEMPDIR.name) os.close(_handle) if os.path.exists(os.environ["QT_MOCK_TEST_STATE_FILE"]): os.remove(os.environ["QT_MOCK_TEST_STATE_FILE"]) def tearDown(self): os.remove(self.xml_file) del self.xml_file # Run testrunner specifically for the tests here, with --parse-xml-testlog. def run3(self, testrunner_args=None): return run_testrunner(self.xml_file, testrunner_args=testrunner_args) def test_no_failure(self): write_xml_log(self.xml_file, failure=None) proc = self.run3() self.assertEqual(proc.returncode, 0) def test_always_pass_failed(self): write_xml_log(self.xml_file, failure="always_pass") proc = self.run3() self.assertEqual(proc.returncode, 0) def test_always_pass_failed_max_repeats_0(self): write_xml_log(self.xml_file, failure="always_pass") proc = self.run3(testrunner_args=["--max-repeats", "0"]) self.assertEqual(proc.returncode, 2) def test_always_fail_failed(self): write_xml_log(self.xml_file, failure="always_fail") proc = self.run3() self.assertEqual(proc.returncode, 2) # Assert that one of the re-runs was in verbose mode matches = re.findall("VERBOSE RUN", proc.stdout.decode()) self.assertEqual(len(matches), 1) # Assert that the environment was altered too self.assertIn("QT_LOGGING_RULES", proc.stdout.decode()) def test_always_crash_crashed(self): write_xml_log(self.xml_file, failure="always_crash") proc = self.run3() self.assertEqual(proc.returncode, 3) def test_fail_then_pass_2_failed(self): write_xml_log(self.xml_file, failure="fail_then_pass:2") proc = self.run3() self.assertEqual(proc.returncode, 0) def test_fail_then_pass_5_failed(self): write_xml_log(self.xml_file, failure="fail_then_pass:5") proc = self.run3() self.assertEqual(proc.returncode, 2) def test_with_two_failures(self): write_xml_log(self.xml_file, failure=["always_pass", "fail_then_pass:2"]) proc = self.run3() self.assertEqual(proc.returncode, 0) # Check that test output is properly interleaved with qt-testrunner's logging. matches = re.findall(r"(PASS|FAIL!).*\n.*Test process exited with code", proc.stdout.decode()) self.assertEqual(len(matches), 4) def test_initTestCase_fail_crash(self): write_xml_log(self.xml_file, failure="initTestCase") proc = self.run3() self.assertEqual(proc.returncode, 3) if __name__ == "__main__": DEBUG = False if "--debug" in sys.argv: sys.argv.remove("--debug") DEBUG = True # We set failfast=True as we do not want the test suite to continue if the # tests of qt_mock_test failed. The next ones depend on it. unittest.main(failfast=True)