Merge remote-tracking branch 'origin/5.12' into dev

Change-Id: I61b563bce41b43dda2b03ea46a456228435bc294
author: Friedemann Kleint <Friedemann.Kleint@qt.io> 2019-01-22 14:55:21 +0100
committer: Friedemann Kleint <Friedemann.Kleint@qt.io> 2019-01-23 09:40:15 +0100
commit: 34e6ac44ef6d322760fcda02136b25a097b0fe15 (patch)
tree: 50f3494be415740932576437782b1716493b4f2b
parent: f5ee713b60b541ad9d9cac11fa65b4b1091e25ac (diff)
parent: cebc32a588358851dd58dc0fc892e986dd263efb (diff)
6 files changed, 386 insertions, 2 deletions
diff --git a/build_scripts/utils.py b/build_scripts/utils.py
index 9500b3409..ffe404860 100644
--- a/build_scripts/utils.py
+++ b/build_scripts/utils.py
@@ -1133,7 +1133,7 @@ def run_instruction(instruction, error, initial_env=None):
 def acceptCITestConfiguration(hostOS, hostOSVer, targetArch, compiler):
     # Disable unsupported CI configs for now
     # NOTE: String must match with QT CI's storagestruct thrift
-    if hostOSVer in ["WinRT_10"]:
+    if hostOSVer in ["WinRT_10", "WebAssembly"]:
         print("Disabled " + hostOSVer + " from Coin configuration")
         return False
    # With 5.11 CI will create two sets of release binaries, one with msvc 2015 and one with msvc 2017
diff --git a/coin_build_instructions.py b/coin_build_instructions.py
index db021470f..5c12415dd 100644
--- a/coin_build_instructions.py
+++ b/coin_build_instructions.py
@@ -128,6 +128,11 @@ def run_build_instructions():
     if not acceptCITestConfiguration(CI_HOST_OS, CI_HOST_OS_VER, CI_TARGET_ARCH, CI_COMPILER):
         exit()
 
+    # Remove some environment variables that impact cmake
+    for env_var in ['CC', 'CXX']:
+        if os.environ.get(env_var):
+            del os.environ[env_var]
+
     # Uses default python, hopefully we have python2 installed on all hosts
     # Skip building using Python 2 on Windows, because of different MSVC C runtimes (VS2008 vs VS2015+)
     if CI_HOST_OS != "Windows":
diff --git a/coin_test_instructions.py b/coin_test_instructions.py
index 4121bb558..7fbe540b4 100644
--- a/coin_test_instructions.py
+++ b/coin_test_instructions.py
@@ -87,6 +87,11 @@ def run_test_instructions():
     if not acceptCITestConfiguration(CI_HOST_OS, CI_HOST_OS_VER, CI_TARGET_ARCH, CI_COMPILER):
         exit()
 
+    # Remove some environment variables that impact cmake
+    for env_var in ['CC', 'CXX']:
+        if os.environ.get(env_var):
+            del os.environ[env_var]
+
     os.chdir(CI_ENV_AGENT_DIR)
     testRun = 0
     # We didn't build for Python 2 in win
diff --git a/debug_windows.py b/debug_windows.py
new file mode 100644
index 000000000..704228c1f
--- /dev/null
+++ b/debug_windows.py
@@ -0,0 +1,357 @@
+#############################################################################
+##
+## Copyright (C) 2018 The Qt Company Ltd.
+## Contact: https://www.qt.io/licensing/
+##
+## This file is part of Qt for Python.
+##
+## $QT_BEGIN_LICENSE:LGPL$
+## Commercial License Usage
+## Licensees holding valid commercial Qt licenses may use this file in
+## accordance with the commercial license agreement provided with the
+## Software or, alternatively, in accordance with the terms contained in
+## a written agreement between you and The Qt Company. For licensing terms
+## and conditions see https://www.qt.io/terms-conditions. For further
+## information use the contact form at https://www.qt.io/contact-us.
+##
+## GNU Lesser General Public License Usage
+## Alternatively, this file may be used under the terms of the GNU Lesser
+## General Public License version 3 as published by the Free Software
+## Foundation and appearing in the file LICENSE.LGPL3 included in the
+## packaging of this file. Please review the following information to
+## ensure the GNU Lesser General Public License version 3 requirements
+## will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
+##
+## GNU General Public License Usage
+## Alternatively, this file may be used under the terms of the GNU
+## General Public License version 2.0 or (at your option) the GNU General
+## Public license version 3 or any later version approved by the KDE Free
+## Qt Foundation. The licenses are as published by the Free Software
+## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
+## included in the packaging of this file. Please review the following
+## information to ensure the GNU General Public License requirements will
+## be met: https://www.gnu.org/licenses/gpl-2.0.html and
+## https://www.gnu.org/licenses/gpl-3.0.html.
+##
+## $QT_END_LICENSE$
+##
+###############
+
+"""
+This is a troubleshooting script that assists finding out which DLLs or
+which symbols in a DLL are missing when executing a PySide2 python
+script.
+It can also be used with any other non Python executable.
+
+Usage: python debug_windows.py
+       When no arguments are given the script will try to import
+       PySide2.QtCore.
+
+Usage: python debug_windows.py python -c "import PySide2.QtWebEngine"
+       python debug_windows.py my_executable.exe arg1 arg2 --arg3=4
+       Any arguments given after the script name will be considered
+       as the target executable and the arguments passed to that
+       executable.
+
+The script requires administrator privileges.
+
+The script uses cdb.exe and gflags.exe under the hood, which are
+installed together with the Windows Kit found at:
+https://developer.microsoft.com/en-us/windows/downloads/windows-10-sdk
+
+"""
+
+from __future__ import print_function
+
+import sys
+import re
+import subprocess
+import ctypes
+import logging
+import argparse
+from os import path
+from textwrap import dedent
+
+is_win = sys.platform == "win32"
+is_py_3 = sys.version_info[0] == 3
+if is_win:
+    if is_py_3:
+        import winreg
+    else:
+        import _winreg as winreg
+        import exceptions
+
+
+def get_parser_args():
+    desc_msg = "Run an executable under cdb with loader snaps set."
+    help_msg = "Pass the executable and the arguments passed to it as a list."
+    parser = argparse.ArgumentParser(description=desc_msg)
+    parser.add_argument('args', nargs='*', help=help_msg)
+    # Prepend -- so that python options like '-c' are ignored by
+    # argparse.
+    massaged_args = ['--'] + sys.argv[1:]
+    return parser.parse_args(massaged_args)
+
+
+parser_args = get_parser_args()
+verbose_log_file_name = path.join(path.dirname(path.abspath(__file__)),
+                                  'log_debug_windows.txt')
+
+
+def is_admin():
+    try:
+        return ctypes.windll.shell32.IsUserAnAdmin()
+    except Exception as e:
+        log.error("is_admin: Exception error: {}".format(e))
+        return False
+
+
+def get_verbose_logger():
+    handler = logging.FileHandler(verbose_log_file_name, mode='w')
+    main_logger = logging.getLogger('main')
+    main_logger.setLevel(logging.INFO)
+    main_logger.addHandler(handler)
+    return main_logger
+
+
+def get_non_verbose_logger():
+    handler = logging.StreamHandler()
+    main_logger = logging.getLogger('main.non_verbose')
+    main_logger.setLevel(logging.INFO)
+    main_logger.addHandler(handler)
+    return main_logger
+
+
+big_log = get_verbose_logger()
+log = get_non_verbose_logger()
+
+
+def sub_keys(key):
+    i = 0
+    while True:
+        try:
+            sub_key = winreg.EnumKey(key, i)
+            yield sub_key
+            i += 1
+        except WindowsError as e:
+            log.error(e)
+            break
+
+
+def sub_values(key):
+    i = 0
+    while True:
+        try:
+            v = winreg.EnumValue(key, i)
+            yield v
+            i += 1
+        except WindowsError as e:
+            log.error(e)
+            break
+
+
+def get_installed_windows_kits():
+    roots_key = r"SOFTWARE\Microsoft\Windows Kits\Installed Roots"
+    log.info("Searching for Windows kits in registry path: "
+             "{}".format(roots_key))
+    roots = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, roots_key, 0,
+                           winreg.KEY_READ)
+    kits = []
+    pattern = re.compile(r'KitsRoot(\d+)')
+
+    for (name, value, value_type) in sub_values(roots):
+        if value_type == winreg.REG_SZ and name.startswith('KitsRoot'):
+            match = pattern.search(name)
+            if match:
+                version = match.group(1)
+                kits.append({'version': version, 'value': value})
+
+    if not kits:
+        log.error(dedent("""
+            No windows kits found in the registry.
+            Consider downloading and installing the latest kit, either from
+            https://docs.microsoft.com/en-us/windows-hardware/drivers/debugger/debugger-download-tools
+            or from
+            https://developer.microsoft.com/en-us/windows/downloads/windows-10-sdk
+        """))
+        exit(1)
+    return kits
+
+
+def get_appropriate_kit(kits):
+    # Fixme, figure out if there is a more special way to choose a kit
+    # and not just latest version.
+    log.info("Found Windows kits are: {}".format(kits))
+    chosen_kit = {'version': "0", 'value': None}
+    for kit in kits:
+        if (kit['version'] > chosen_kit['version'] and
+                # version 8.1 is actually '81', so consider everything
+                # above version 20, as '2.0', etc.
+                kit['version'] < "20"):
+            chosen_kit = kit
+    first_kit = kits[0]
+    return first_kit
+
+
+def get_cdb_and_gflags_path(kits):
+    first_kit = get_appropriate_kit(kits)
+    first_path_path = first_kit['value']
+    log.info('Using kit found at {}'.format(first_path_path))
+    bits = 'x64' if (sys.maxsize > 2 ** 32) else 'x32'
+    debuggers_path = path.join(first_path_path, 'Debuggers', bits)
+    cdb_path = path.join(debuggers_path, 'cdb.exe')
+
+    if not path.exists(cdb_path):
+        log.error("Couldn't find cdb.exe at: {}.".format(cdb_path))
+        exit(1)
+    else:
+        log.info("Found cdb.exe at: {}.".format(cdb_path))
+
+    gflags_path = path.join(debuggers_path, 'gflags.exe')
+
+    if not path.exists(gflags_path):
+        log.error('Couldn\'t find gflags.exe at: {}.'.format(gflags_path))
+        exit(1)
+    else:
+        log.info('Found gflags.exe at: {}.'.format(cdb_path))
+
+    return cdb_path, gflags_path
+
+
+def toggle_loader_snaps(executable_name, gflags_path, enable=True):
+    arg = '+sls' if enable else '-sls'
+    gflags_args = [gflags_path, '-i', executable_name, arg]
+    try:
+        log.info('Invoking gflags: {}'.format(gflags_args))
+        output = subprocess.check_output(gflags_args, stderr=subprocess.STDOUT,
+                                         universal_newlines=True)
+        log.info(output)
+    except exceptions.WindowsError as e:
+        log.error("\nRunning {} exited with exception: "
+                  "\n{}".format(gflags_args, e))
+        exit(1)
+    except subprocess.CalledProcessError as e:
+        log.error("\nRunning {} exited with: {} and stdout was: "
+                  "{}".format(gflags_args, e.returncode, e.output))
+        exit(1)
+
+
+def find_error_like_snippets(content):
+    snippets = []
+    lines = content.splitlines()
+    context_lines = 4
+
+    def error_predicate(l):
+        # A list of mostly false positives are filtered out.
+        # For deeper inspection, the full log exists.
+        errors = {'errorhandling',
+                  'windowserrorreporting',
+                  'core-winrt-error',
+                  'RtlSetLastWin32Error',
+                  'RaiseInvalid16BitExeError',
+                  'BaseWriteErrorElevationRequiredEvent',
+                  'for DLL "Unknown"',
+                  'LdrpGetProcedureAddress',
+                  'X509_STORE_CTX_get_error',
+                  'ERR_clear_error',
+                  'ERR_peek_last_error',
+                  'ERR_error_string',
+                  'ERR_get_error',
+                  ('ERROR: Module load completed but symbols could '
+                   'not be loaded')}
+        return (re.search('error', l, re.IGNORECASE)
+                and all(e not in errors for e in errors))
+
+    for i in range(1, len(lines)):
+        line = lines[i]
+        if error_predicate(line):
+            snippets.append(lines[i - context_lines:i + context_lines + 1])
+
+    return snippets
+
+
+def print_error_snippets(snippets):
+    if len(snippets) > 0:
+        log.info("\nThe following possible errors were found:\n")
+
+        for i in range(1, len(snippets)):
+            log.info("Snippet {}:".format(i))
+            for line in snippets[i]:
+                log.info(line)
+        log.info("")
+
+
+def call_command_under_cdb_with_gflags(executable_path, args):
+    executable_name = path.basename(executable_path)
+    invocation = [executable_path] + args
+
+    kits = get_installed_windows_kits()
+    cdb_path, gflags_path = get_cdb_and_gflags_path(kits)
+
+    toggle_loader_snaps(executable_name, gflags_path, enable=True)
+
+    log.info("Debugging the following command invocation: "
+             "{}".format(invocation))
+
+    cdb_args = [cdb_path] + invocation
+
+    log.info('Invoking cdb: {}'.format(cdb_args))
+
+    p = subprocess.Popen(cdb_args,
+                         stdout=subprocess.PIPE,
+                         stderr=subprocess.PIPE,
+                         stdin=subprocess.PIPE,
+                         shell=False)
+
+    # Symbol fix, start process, print all thread stack traces, exit.
+    cdb_commands = ['.symfix', 'g', '!uniqstack', 'q']
+    cdb_commands_text = '\n'.join(cdb_commands)
+    out, err = p.communicate(input=cdb_commands_text.encode('utf-8'))
+
+    out_decoded = out.decode('utf-8')
+    big_log.info('stdout: {}'.format(out_decoded))
+    if err:
+        big_log.info('stderr: {}'.format(err.decode('utf-8')))
+
+    log.info('Finished execution of process under cdb.')
+
+    toggle_loader_snaps(executable_name, gflags_path, enable=False)
+
+    snippets = find_error_like_snippets(out_decoded)
+    print_error_snippets(snippets)
+
+    log.info("Finished processing.\n !!! Full log can be found at: "
+             "{}".format(verbose_log_file_name))
+
+
+def test_run_import_qt_core_under_cdb_with_gflags():
+    # The weird characters are there for faster grepping of the output
+    # because there is a lot of content in the full log.
+    # The 2+2 is just ensure that Python itself works.
+    python_code = """
+print(">>>>>>>>>>>>>>>>>>>>>>> Test computation of 2+2 is: {}".format(2+2))
+import PySide2.QtCore
+print(">>>>>>>>>>>>>>>>>>>>>>> QtCore object instance: {}".format(PySide2.QtCore))
+"""
+    call_command_under_cdb_with_gflags(sys.executable, ["-c", python_code])
+
+
+def handle_args():
+    if not parser_args.args:
+        test_run_import_qt_core_under_cdb_with_gflags()
+    else:
+        call_command_under_cdb_with_gflags(parser_args.args[0],
+                                           parser_args.args[1:])
+
+
+if __name__ == '__main__':
+    if not is_win:
+        log.error("This script only works on Windows.")
+        exit(1)
+
+    if is_admin():
+        handle_args()
+    else:
+        log.error("Please rerun the script with administrator privileges. "
+                  "It is required for gflags.exe to work. ")
+        exit(1)
diff --git a/sources/pyside2/PySide2/QtLocation/typesystem_location.xml b/sources/pyside2/PySide2/QtLocation/typesystem_location.xml
index 794c1f3ef..d1885756e 100644
--- a/sources/pyside2/PySide2/QtLocation/typesystem_location.xml
+++ b/sources/pyside2/PySide2/QtLocation/typesystem_location.xml
@@ -54,6 +54,9 @@
   </object-type>
   <value-type name="QGeoRouteSegment"/>
   <object-type name="QGeoServiceProvider">
+      <!-- Temporary workaround until qtlocation:8fbab5c949bf0fe1ec5e1e452a9c475681d7edc4 has landed -->
+      <modify-function signature="loaderError()const" remove="all"/>
+      <modify-function signature="loaderErrorString()const" remove="all"/>
       <enum-type name="Error"/>
       <enum-type name="RoutingFeature" flags="RoutingFeatures"/>
       <enum-type name="GeocodingFeature" flags="GeocodingFeatures"/>
diff --git a/sources/pyside2/PySide2/support/generate_pyi.py b/sources/pyside2/PySide2/support/generate_pyi.py
index d754bce76..f1913d03f 100644
--- a/sources/pyside2/PySide2/support/generate_pyi.py
+++ b/sources/pyside2/PySide2/support/generate_pyi.py
@@ -52,6 +52,7 @@ import io
 import re
 import subprocess
 import argparse
+import glob
 from contextlib import contextmanager
 from textwrap import dedent
 
@@ -68,6 +69,9 @@ sourcepath = os.path.splitext(__file__)[0] + ".py"
 USE_PEP563 = sys.version_info[:2] >= (3, 7)
 
 indent = " " * 4
+is_py3 = sys.version_info[0] == 3
+is_ci = os.environ.get("QTEST_ENVIRONMENT", "") == "ci"
+
 
 class Writer(object):
     def __init__(self, outfile):
@@ -248,7 +252,7 @@ def generate_pyi(import_name, outpath, options):
             else:
                 wr.print(line)
     logger.info("Generated: {outfilepath}".format(**locals()))
-    if sys.version_info[0] == 3:
+    if is_py3:
         # Python 3: We can check the file directly if the syntax is ok.
         subprocess.check_output([sys.executable, outfilepath])
     return 1
@@ -295,6 +299,16 @@ def generate_all_pyi(outpath, options):
     lockdir = os.path.join(outpath, "generate_pyi.lockfile")
     with single_process(lockdir) as locked:
         if locked:
+            if is_ci:
+                # When COIN is running, we sometimes get racing conditions with
+                # the windows manifest tool which wants access to a module that
+                # we already have imported. But when we wait until all binaries
+                # are created, that cannot happen, because we are then the last
+                # process, and the tool has already been run.
+                bin_pattern = "Qt*.pyd" if sys.platform == "win32" else "Qt*.so"
+                search = os.path.join(PySide2.__path__[0], bin_pattern)
+                if len(glob.glob(search)) < len(PySide2.__all__):
+                    return
             for mod_name in PySide2.__all__:
                 import_name = "PySide2." + mod_name
                 step = generate_pyi(import_name, outpath, options)
author	Friedemann Kleint <Friedemann.Kleint@qt.io>	2019-01-22 14:55:21 +0100
committer	Friedemann Kleint <Friedemann.Kleint@qt.io>	2019-01-23 09:40:15 +0100
commit	34e6ac44ef6d322760fcda02136b25a097b0fe15 (patch)
tree	50f3494be415740932576437782b1716493b4f2b
parent	f5ee713b60b541ad9d9cac11fa65b4b1091e25ac (diff)
parent	cebc32a588358851dd58dc0fc892e986dd263efb (diff)