diff options
Diffstat (limited to 'tests/auto/util/testrunner/qt_mock_test.py')
| -rwxr-xr-x | tests/auto/util/testrunner/qt_mock_test.py | 221 |
1 files changed, 221 insertions, 0 deletions
diff --git a/tests/auto/util/testrunner/qt_mock_test.py b/tests/auto/util/testrunner/qt_mock_test.py new file mode 100755 index 00000000000..af8fdf24509 --- /dev/null +++ b/tests/auto/util/testrunner/qt_mock_test.py @@ -0,0 +1,221 @@ +#!/usr/bin/env python3 +# Copyright (C) 2021 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 + + +# This is an artificial test, mimicking the Qt tests, for example tst_whatever. +# Its purpose is to assist in testing qt-testrunner.py. +# +# Mode A: +# +# If invoked with a test function argument, it runs that test function. +# +# Usage: +# +# $0 always_pass +# $0 always_fail +# $0 always_crash +# $0 fail_then_pass:N # where N is the number of failing runs before passing +# +# Needs environment variable: +# + QT_MOCK_TEST_STATE_FILE :: points to a unique filename, to be written +# for keeping the state of the fail_then_pass:N tests. +# +# Mode B: +# +# If invoked without any argument, it runs the tests listed in the +# variable QT_MOCK_TEST_RUN_LIST. If variable is empty it just runs +# the always_pass test. It also understands qtestlib's `-o outfile.xml,xml` +# option for writing a mock testlog in a file. Requires environment variables: +# + QT_MOCK_TEST_STATE_FILE :: See above +# + QT_MOCK_TEST_XML_TEMPLATE_FILE :: may point to the template XML file +# located in the same source directory. Without this variable, the +# option `-o outfile.xml,xml` will be ignored. +# + QT_MOCK_TEST_RUN_LIST :: may contain a comma-separated list of test +# that should run. +# + QT_MOCK_TEST_CRASH_CLEANLY :: if set to 1, then the executable will +# crash (exit with a high exit code) +# after successfully running the given tests and writing the XML logfile. + + + +import sys +import os +import traceback +from tst_qt_testrunner import write_xml_log + + +MY_NAME = os.path.basename(sys.argv[0]) +STATE_FILE = None +XML_TEMPLATE = None +XML_OUTPUT_FILE = None +CRASH_CLEANLY = False + + +def crash(): + sys.exit(131) + +def put_failure(test_name): + with open(STATE_FILE, "a") as f: + f.write(test_name + "\n") +def get_failures(test_name): + n = 0 + try: + with open(STATE_FILE) as f: + for line in f: + if line.strip() == test_name: + n += 1 + except FileNotFoundError: + return 0 + return n + +# Only care about the XML log output file. +def parse_output_argument(a): + global XML_OUTPUT_FILE + if a.endswith(",xml"): + XML_OUTPUT_FILE = a[:-4] + +# Strip qtestlib specific arguments. +# Only care about the "-o ...,xml" argument. +def clean_cmdline(): + args = [] + prev_arg = None + skip_next_arg = True # Skip argv[0] + for a in sys.argv: + if skip_next_arg: + if prev_arg == "-o": + parse_output_argument(a) + prev_arg = None + skip_next_arg = False + continue + if a in ("-o", "-maxwarnings"): + skip_next_arg = True + prev_arg = a + continue + if a in ("-v1", "-v2", "-vs"): + print("VERBOSE RUN") + if "QT_LOGGING_RULES" in os.environ: + print("Environment has QT_LOGGING_RULES:", + os.environ["QT_LOGGING_RULES"]) + continue + args.append(a) + return args + + +def log_test(testcase, result, + testsuite=MY_NAME.rpartition(".")[0]): + print("%-7s: %s::%s()" % (result, testsuite, testcase)) + +def log_xml(fail_list): + if XML_OUTPUT_FILE and XML_TEMPLATE is not None: + if XML_TEMPLATE == "": + # If the template is an empty file, then write an empty output file + with open(XML_OUTPUT_FILE, "w"): + pass + else: + write_xml_log(XML_OUTPUT_FILE, failure=fail_list) + +# Return the exit code +def run_test(testname): + if testname == "initTestCase": + exit_code = 1 # specifically test that initTestCase fails + elif testname == "always_pass": + exit_code = 0 + elif testname == "always_fail": + exit_code = 1 + elif testname == "always_crash": + exit_code = 131 + elif testname == "fail_then_crash": + previous_fails = get_failures(testname) + if previous_fails == 0: + put_failure(testname) + exit_code = 1 + else: + exit_code = 131 + elif testname.startswith("fail_then_pass"): + wanted_fails = int(testname.partition(":")[2]) + previous_fails = get_failures(testname) + if previous_fails < wanted_fails: + put_failure(testname) + exit_code = 1 + else: + exit_code = 0 + else: + assert False, "Unknown argument: %s" % testname + + if exit_code == 0: + log_test(testname, "PASS") + elif exit_code == 1: + log_test(testname, "FAIL!") + else: + log_test(testname, "CRASH!") + + return exit_code + +def no_args_run(): + try: + run_list = os.environ["QT_MOCK_TEST_RUN_LIST"].split(",") + except KeyError: + run_list = ["always_pass"] + + total_result = True + fail_list = [] + for test in run_list: + test_exit_code = run_test(test) + if test_exit_code not in (0, 1): + crash() + if test_exit_code != 0: + fail_list.append(test) + total_result = total_result and (test_exit_code == 0) + + log_xml(fail_list) + + if CRASH_CLEANLY: + # Crash despite all going well and writing all output files cleanly. + crash() + + if total_result: + sys.exit(0) + else: + sys.exit(1) + + +def main(): + global STATE_FILE + # Will fail if env var is not set. + STATE_FILE = os.environ["QT_MOCK_TEST_STATE_FILE"] + + global XML_TEMPLATE + if "QT_MOCK_TEST_XML_TEMPLATE_FILE" in os.environ: + with open(os.environ["QT_MOCK_TEST_XML_TEMPLATE_FILE"]) as f: + XML_TEMPLATE = f.read() + + global CRASH_CLEANLY + if ("QT_MOCK_TEST_CRASH_CLEANLY" in os.environ + and os.environ["QT_MOCK_TEST_CRASH_CLEANLY"] == "1" + ): + CRASH_CLEANLY = True + + args = clean_cmdline() + + if len(args) == 0: + no_args_run() + assert False, "Unreachable!" + else: # run single test function + exit_code = run_test(args[0]) + # Write "fail" in the XML log only if the specific run has failed. + if exit_code != 0: + log_xml([args[0]]) + else: + log_xml([]) + sys.exit(exit_code) + + +# TODO write XPASS test that does exit(1) + +if __name__ == "__main__": + try: + main() + except Exception as e: + traceback.print_exc() + exit(128) # Something went wrong with this script |