4 files changed, 335 insertions, 36 deletions
diff --git a/sources/pyside-tools/android_deploy.py b/sources/pyside-tools/android_deploy.py
index f10834c12..fbc613069 100644
--- a/sources/pyside-tools/android_deploy.py
+++ b/sources/pyside-tools/android_deploy.py
@@ -237,14 +237,13 @@ if __name__ == "__main__":
                         help=f"Path to shiboken{MAJOR_VERSION} Android Wheel",
                         required=not config_option_exists())
 
-    #TODO: --ndk-path and --sdk-path will be removed when automatic download of sdk and ndk is added
     parser.add_argument("--ndk-path", type=lambda p: Path(p).resolve(),
                         help=("Path to Android Ndk. If omitted, the default from buildozer is used")
-                        , required=True)
+                        )
 
     parser.add_argument("--sdk-path", type=lambda p: Path(p).resolve(),
                         help=("Path to Android Sdk. If omitted, the default from buildozer is used")
-                        , required=True)
+                        )
 
     parser.add_argument("--extra-ignore-dirs", type=str, help=HELP_EXTRA_IGNORE_DIRS)
 
diff --git a/sources/pyside-tools/deploy_lib/config.py b/sources/pyside-tools/deploy_lib/config.py
index af019a093..8af341bad 100644
--- a/sources/pyside-tools/deploy_lib/config.py
+++ b/sources/pyside-tools/deploy_lib/config.py
@@ -12,6 +12,9 @@ from .commands import run_qmlimportscanner
 # Some QML plugins like QtCore are excluded from this list as they don't contribute much to
 # executable size. Excluding them saves the extra processing of checking for them in files
 EXCLUDED_QML_PLUGINS = {"QtQuick", "QtQuick3D", "QtCharts", "QtWebEngine", "QtTest", "QtSensors"}
+# TODO: Move this to android module. Fix circular import.
+ANDROID_NDK_VERSION = "25c"
+ANDROID_DEPLOY_CACHE = Path.home() / ".pyside6_android_deploy"
 
 
 class BaseConfig:
@@ -128,13 +131,30 @@ class Config(BaseConfig):
                 self.ndk_path = android_data.ndk_path
             else:
                 ndk_path_temp = self.get_value("buildozer", "ndk_path")
-                self.ndk_path = Path(ndk_path_temp) if ndk_path_temp else None
+                if ndk_path_temp:
+                    self.ndk_path = Path(ndk_path_temp)
+                else:
+                    self.ndk_path = (ANDROID_DEPLOY_CACHE / "android-ndk"
+                                     / f"android-ndk-r{ANDROID_NDK_VERSION}")
+                    if not self.ndk_path.exists():
+                        logging.info("[DEPLOY] Use default NDK from buildoer")
+
+            if self.ndk_path:
+                print(f"Using Android NDK: {str(self.ndk_path)}")
 
             if android_data.sdk_path:
                 self.sdk_path = android_data.sdk_path
             else:
                 sdk_path_temp = self.get_value("buildozer", "sdk_path")
-                self.sdk_path = Path(sdk_path_temp) if sdk_path_temp else None
+                if sdk_path_temp:
+                    self.sdk_path = Path(sdk_path_temp)
+                else:
+                    self.sdk_path = ANDROID_DEPLOY_CACHE / "android-sdk"
+                    if not self.sdk_path.exists():
+                        logging.info("[DEPLOY] Use default SDK from buildozer")
+
+            if self.sdk_path:
+                print(f"Using Android SDK: {str(self.sdk_path)}")
 
             recipe_dir_temp = self.get_value("buildozer", "recipe_dir")
             self.recipe_dir = Path(recipe_dir_temp) if recipe_dir_temp else None
diff --git a/tools/cross_compile_android/android_utilities.py b/tools/cross_compile_android/android_utilities.py
new file mode 100644
index 000000000..e93631173
--- /dev/null
+++ b/tools/cross_compile_android/android_utilities.py
@@ -0,0 +1,256 @@
+# Copyright (C) 2023 The Qt Company Ltd.
+# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
+
+import logging
+import shutil
+import os
+import stat
+import sys
+import subprocess
+
+from urllib import request
+from pathlib import Path
+from typing import List
+from packaging import version
+from tqdm import tqdm
+
+# the tag number does not matter much since we update the sdk later
+DEFAULT_SDK_TAG = 6514223
+ANDROID_NDK_VERSION = "25c"
+
+
+def run_command(command: List[str], cwd: str = None, ignore_fail: bool = False,
+                dry_run: bool = False, accept_prompts: bool = False, show_stdout: bool = False,
+                capture_stdout: bool = False):
+
+    if capture_stdout and not show_stdout:
+        raise RuntimeError("capture_stdout should always be used together with show_stdout")
+
+    if dry_run:
+        print(" ".join(command))
+        return
+
+    input = None
+    if accept_prompts:
+        input = str.encode("y")
+
+    if show_stdout:
+        stdout = None
+    else:
+        stdout = subprocess.DEVNULL
+
+    result = subprocess.run(command, cwd=cwd, input=input, stdout=stdout,
+                            capture_output=capture_stdout)
+
+    if result.returncode != 0 and not ignore_fail:
+        sys.exit(result.returncode)
+
+    if capture_stdout and not result.returncode:
+        return result.stdout.decode("utf-8")
+
+    return None
+
+
+class DownloadProgressBar(tqdm):
+    def update_to(self, b=1, bsize=1, tsize=None):
+        if tsize is not None:
+            self.total = tsize
+        self.update(b * bsize - self.n)
+
+
+class SdkManager:
+    def __init__(self, android_sdk_dir: Path, dry_run: bool = False):
+        self._sdk_manager = android_sdk_dir / "tools" / "bin" / "sdkmanager"
+
+        if not self._sdk_manager.exists():
+            raise RuntimeError(f"Unable to find SdkManager in {str(self._sdk_manager)}")
+
+        if not os.access(self._sdk_manager, os.X_OK):
+            current_permissions = stat.S_IMODE(os.lstat(self._sdk_manager).st_mode)
+            os.chmod(self._sdk_manager, current_permissions | stat.S_IEXEC)
+
+        self._android_sdk_dir = android_sdk_dir
+        self._dry_run = dry_run
+
+    def list_packages(self):
+        command = [self._sdk_manager, f"--sdk_root={self._android_sdk_dir}", "--list"]
+        return run_command(command=command, dry_run=self._dry_run, show_stdout=True,
+                           capture_stdout=True)
+
+    def install(self, *args, accept_license: bool = False, show_stdout=False):
+        command = [str(self._sdk_manager), f"--sdk_root={self._android_sdk_dir}", *args]
+        run_command(command=command, dry_run=self._dry_run,
+                    accept_prompts=accept_license, show_stdout=show_stdout)
+
+
+def _unpack(zip_file: Path, destination: Path):
+    """
+    Unpacks the zip_file into destination preserving all permissions
+
+    TODO: Try to use zipfile module. Currently we cannot use zipfile module here because
+    extractAll() does not preserve permissions.
+
+    In case `unzip` is not available, the user is requested to install it manually
+    """
+    unzip = shutil.which("unzip")
+    if not unzip:
+        raise RuntimeError("Unable to find program unzip. Use `sudo apt-get install unzip`"
+                           "to install it")
+
+    command = [unzip, zip_file, "-d", destination]
+    run_command(command=command, show_stdout=True)
+
+
+def _download(url: str, destination: Path):
+    """
+    Download url to destination
+    """
+    headers, download_path = None, None
+    # https://github.com/tqdm/tqdm#hooks-and-callbacks
+    with DownloadProgressBar(unit='B', unit_scale=True, miniters=1, desc=url.split('/')[-1]) as t:
+        download_path, headers = request.urlretrieve(url=url, filename=destination,
+                                                     reporthook=t.update_to)
+    assert headers["Content-Type"] == "application/zip"
+    assert Path(download_path).resolve() == destination
+
+
+def download_android_ndk(ndk_path: Path):
+    """
+    Downloads the given ndk_version into ndk_path
+    """
+    ndk_path = ndk_path / "android-ndk"
+    ndk_zip_path = ndk_path / f"android-ndk-r{ANDROID_NDK_VERSION}-linux.zip"
+    ndk_version_path = ndk_path / f"android-ndk-r{ANDROID_NDK_VERSION}"
+
+    if ndk_version_path.exists():
+        print(f"NDK path found in {str(ndk_version_path)}")
+    else:
+        ndk_path.mkdir(parents=True, exist_ok=True)
+        url = f"https://dl.google.com/android/repository/android-ndk-r{ANDROID_NDK_VERSION}-linux.zip"
+
+        print(f"Downloading Android Ndk version r{ANDROID_NDK_VERSION}")
+        _download(url=url, destination=ndk_zip_path)
+
+        print("Unpacking Android Ndk")
+        _unpack(zip_file=(ndk_path /
+                f"android-ndk-r{ANDROID_NDK_VERSION}-linux.zip"),
+                destination=ndk_path)
+
+    return ndk_version_path
+
+
+def download_android_commandlinetools(android_sdk_dir: Path):
+    """
+    Downloads Android commandline tools into cltools_path.
+    """
+    android_sdk_dir = android_sdk_dir / "android-sdk"
+    url = ("https://dl.google.com/android/repository/"
+           f"commandlinetools-linux-{DEFAULT_SDK_TAG}_latest.zip")
+    cltools_zip_path = android_sdk_dir / f"commandlinetools-linux-{DEFAULT_SDK_TAG}_latest.zip"
+    cltools_path = android_sdk_dir / "tools"
+
+    if cltools_path.exists():
+        print(f"Command-line tools found in {str(cltools_path)}")
+    else:
+        android_sdk_dir.mkdir(parents=True, exist_ok=True)
+
+        print("Download Android Command Line Tools: "
+              f"commandlinetools-linux-{DEFAULT_SDK_TAG}_latest.zip")
+        _download(url=url, destination=cltools_zip_path)
+
+        print("Unpacking Android Command Line Tools")
+        _unpack(zip_file=cltools_zip_path, destination=android_sdk_dir)
+
+    return android_sdk_dir
+
+
+def android_list_build_tools_versions(sdk_manager: SdkManager):
+    """
+    List all the build-tools versions available for download
+    """
+    available_packages = sdk_manager.list_packages()
+    build_tools_versions = []
+    lines = available_packages.split('\n')
+
+    for line in lines:
+        if not line.strip().startswith('build-tools;'):
+            continue
+        package_name = line.strip().split(' ')[0]
+        if package_name.count(';') != 1:
+            raise RuntimeError(f"Unable to parse build-tools version: {package_name}")
+        ver = package_name.split(';')[1]
+
+        build_tools_versions.append(version.Version(ver))
+
+    return build_tools_versions
+
+
+def find_installed_buildtools_version(build_tools_dir: Path):
+    """
+    It is possible that the user has multiple build-tools installed. The newer  version is generally
+    used. This function find the newest among the installed build-tools
+    """
+    versions = [version.Version(bt_dir.name) for bt_dir in build_tools_dir.iterdir()
+                if bt_dir.is_dir()]
+    return max(versions)
+
+
+def find_latest_buildtools_version(sdk_manager: SdkManager):
+    """
+    Uses sdk manager to find the latest build-tools version
+    """
+    available_build_tools_v = android_list_build_tools_versions(sdk_manager=sdk_manager)
+
+    if not available_build_tools_v:
+        raise RuntimeError('Unable to find any build tools available for download')
+
+    return max(available_build_tools_v)
+
+
+def install_android_packages(android_sdk_dir: Path, android_api: str, dry_run: bool = False,
+                             accept_license: bool = False, skip_update: bool = False):
+    """
+    Use the sdk manager to install build-tools, platform-tools and platform API
+    """
+    tools_dir = android_sdk_dir / "tools"
+    if not tools_dir.exists():
+        raise RuntimeError("Unable to find Android command-line tools in "
+                           f"{str(tools_dir)}")
+
+    # incase of --verbose flag
+    show_output = (logging.getLogger().getEffectiveLevel() == logging.INFO)
+
+    sdk_manager = SdkManager(android_sdk_dir=android_sdk_dir, dry_run=dry_run)
+
+    # install/upgrade platform-tools
+    if not (android_sdk_dir / "platform-tools").exists():
+        print("Installing/Updating Android platform-tools")
+        sdk_manager.install("platform-tools", accept_license=accept_license,
+                            show_stdout=show_output)
+        # The --update command is only relevant for platform tools
+        if not skip_update:
+            sdk_manager.install("--update", show_stdout=show_output)
+
+    # install/upgrade build-tools
+    buildtools_dir = android_sdk_dir / "build-tools"
+
+    if not buildtools_dir.exists():
+        latest_build_tools_v = find_latest_buildtools_version(sdk_manager=sdk_manager)
+        print(f"Installing Android build-tools version {latest_build_tools_v}")
+        sdk_manager.install(f"build-tools;{latest_build_tools_v}", show_stdout=show_output)
+    else:
+        if not skip_update:
+            latest_build_tools_v = find_latest_buildtools_version(sdk_manager=sdk_manager)
+            installed_build_tools_v = find_installed_buildtools_version(buildtools_dir)
+            if latest_build_tools_v > installed_build_tools_v:
+                print(f"Updating Android build-tools version to {latest_build_tools_v}")
+                sdk_manager.install(f"build-tools;{latest_build_tools_v}", show_stdout=show_output)
+                installed_build_tools_v = latest_build_tools_v
+
+    # install the platform API
+    platform_api_dir = android_sdk_dir / "platforms" / f"android-{android_api}"
+    if not platform_api_dir.exists():
+        print(f"Installing Android platform API {android_api}")
+        sdk_manager.install(f"platforms;android-{android_api}", show_stdout=show_output)
+
+    print("Android packages installation done")
diff --git a/tools/cross_compile_android/main.py b/tools/cross_compile_android/main.py
index 073fe74e8..97f4030ac 100644
--- a/tools/cross_compile_android/main.py
+++ b/tools/cross_compile_android/main.py
@@ -16,9 +16,31 @@ from git import Repo, RemoteProgress
 from tqdm import tqdm
 from jinja2 import Environment, FileSystemLoader
 
+from android_utilities import (run_command, download_android_commandlinetools,
+                                download_android_ndk, install_android_packages)
+
 # Note: Does not work with PyEnv. Your Host Python should contain openssl.
 PYTHON_VERSION = "3.10"
 
+APIC_HELP = ('''
+Points to the installation path of Python for the specific Android
+platform. If the path given does not exist, then Python for Android
+is cross compiled for the specific platform and installed into this
+path as <path>/Python-'plat_name'/_install.
+
+If this path is not given, then Python for Android is cross-compiled
+into a temportary directory, which is deleted when the Qt for Python
+Android wheels are created.
+''')
+
+SKIP_UPDATE_HELP = ("skip the updation of SDK packages build-tools, platform-tools to"
+                    " latest version")
+
+ACCEPT_LICENSE_HELP = ('''
+Accepts license automatically for Android SDK installation. Otherwise,
+accept the license manually through command line.
+''')
+
 
 @dataclass
 class PlatformData:
@@ -48,20 +70,10 @@ class CloneProgress(RemoteProgress):
         self.pbar.refresh()
 
 
-def run_command(command: List[str], cwd: str = None, ignore_fail: bool = False,
-                dry_run: bool = False):
-    if dry_run:
-        print(" ".join(command))
-        return
-    ex = subprocess.call(command, cwd=cwd)
-    if ex != 0 and not ignore_fail:
-        sys.exit(ex)
-
-
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
-        description="This tool cross builds cpython for android and uses that Python to cross build"
-                    "android Qt for Python wheels",
+        description="This tool cross builds CPython for Android and uses that Python to cross build"
+                    "Android Qt for Python wheels",
         formatter_class=argparse.RawTextHelpFormatter,
     )
 
@@ -72,11 +84,9 @@ if __name__ == "__main__":
     parser.add_argument("-v", "--verbose", help="run in verbose mode", action="store_const",
                         dest="loglevel", const=logging.INFO)
     parser.add_argument("--api-level", type=str, default="31", help="Android API level to use")
-    parser.add_argument("--ndk-path", type=str, required=True,
-                        help="Path to Android NDK (Preferred 25b)")
+    parser.add_argument("--ndk-path", type=str, help="Path to Android NDK (Preferred 25b)")
     # sdk path is needed to compile all the Qt Java Acitivity files into Qt6AndroidBindings.jar
-    parser.add_argument("--sdk-path", type=str, required=True,
-                        help="Path to Android SDK")
+    parser.add_argument("--sdk-path", type=str, help="Path to Android SDK")
     parser.add_argument("--qt-install-path", type=str, required=not occp_exists(),
                         help="Qt installation path eg: /home/Qt/6.5.0")
 
@@ -85,19 +95,16 @@ if __name__ == "__main__":
 
     parser.add_argument("-apic", "--android-python-install-path", type=str, default=None,
                         required=occp_exists(),
-                        help='''
-                        Points to the installation path of Python for the specific Android
-                        platform. If the path given does not exist, then Python for android
-                        is cross compiled for the specific platform and installed into this
-                        path as <path>/Python-'plat_name'/_install.
-
-                        If this path is not given, then Python for android is cross-compiled
-                        into a temportary directory, which is deleted when the Qt for Python
-                        android wheels are created.
-                        ''')
+                        help=APIC_HELP)
 
     parser.add_argument("--dry-run", action="store_true", help="show the commands to be run")
 
+    parser.add_argument("--skip-update", action="store_true",
+                        help=SKIP_UPDATE_HELP)
+
+    parser.add_argument("--auto-accept-license", action="store_true",
+                        help=ACCEPT_LICENSE_HELP)
+
     args = parser.parse_args()
 
     logging.basicConfig(level=args.loglevel)
@@ -114,6 +121,24 @@ if __name__ == "__main__":
     gcc_march = None
     plat_bits = None
     dry_run = args.dry_run
+    plat_name = args.plat_name
+    api_level = args.api_level
+    skip_update = args.skip_update
+    auto_accept_license = args.auto_accept_license
+
+    # auto download Android NDK and SDK
+    pyside6_deploy_cache = Path.home() / ".pyside6_android_deploy"
+
+    if not ndk_path:
+        # Download android ndk
+        ndk_path = download_android_ndk(pyside6_deploy_cache)
+
+    if not sdk_path:
+        # download and unzip command-line tools
+        sdk_path = download_android_commandlinetools(pyside6_deploy_cache)
+        # install and update required android packages
+        install_android_packages(android_sdk_dir=sdk_path, android_api=api_level, dry_run=dry_run,
+                                 accept_license=auto_accept_license, skip_update=skip_update)
 
     # python path is valid, if Python for android installation exists in python_path
     valid_python_path = True
@@ -130,8 +155,6 @@ if __name__ == "__main__":
                 break
 
     templates_path = Path(__file__).parent / "templates"
-    plat_name = args.plat_name
-    api_level = args.api_level
 
     # for armv7a the API level dependent binaries like clang are named
     # armv7a-linux-androideabi27-clang, as opposed to other platforms which
@@ -187,7 +210,7 @@ if __name__ == "__main__":
 
             # run the cross compile script
             logging.info(f"Running Python cross-compile for platform {platform_data.plat_name}")
-            run_command(["./cross_compile.sh"], cwd=cpython_dir, dry_run=dry_run)
+            run_command(["./cross_compile.sh"], cwd=cpython_dir, dry_run=dry_run, show_stdout=True)
 
             python_path = (f"{android_py_install_path_prefix}/Python-{platform_data.plat_name}-linux-android/"
                            "_install")
@@ -196,7 +219,8 @@ if __name__ == "__main__":
             # libpython3.x.so, to match with python_for_android's Python library. Otherwise,
             # the Qfp binaries won't be able to link to Python
             run_command(["patchelf", "--set-soname", f"libpython{PYTHON_VERSION}.so",
-                         f"libpython{PYTHON_VERSION}.so.1.0"], cwd=Path(python_path) / "lib")
+                        f"libpython{PYTHON_VERSION}.so.1.0"], cwd=Path(python_path) / "lib",
+                        dry_run=dry_run)
 
             logging.info(
                 f"Cross compile Python for Android platform {platform_data.plat_name}. "
@@ -242,4 +266,4 @@ if __name__ == "__main__":
                             (f"--qt-target-path={qt_install_path}/"
                              f"android_{platform_data.qt_plat_name}"),
                             "--no-qt-tools", "--skip-docs", "--unity"]
-        run_command(qfp_ccompile_cmd, cwd=pyside_setup_dir, dry_run=dry_run)
+        run_command(qfp_ccompile_cmd, cwd=pyside_setup_dir, dry_run=dry_run, show_stdout=True)