changeset 43513:081a77df7bc6

packaging: consolidate CLI functionality into packaging.py

Consolidating functionality for invoking code in the hgpackaging
package through a single CLI entry point will make things simpler
when we add more complexity to that package. For example, it will
allow us to run things out of a virtualenv with third party
packages.

This commit consolidates functionality from the Inno and WiX
build.py scripts into a new packaging.py script. That script
simply creates a virtualenv and runs the CLI functionality in
it.

The new virtualenv is populated with jinja2 because I felt
it easier to incorporate requirements file processing in this
commit and we will soon use jinja2 in an upcoming commit.

The unified CLI functionality will also make it easier to
script other packaging workflows going forward. e.g. RPM, Debian,
and macOS packaging.

Differential Revision: https://phab.mercurial-scm.org/D7156

--- a/contrib/automation/hgautomation/windows.py	Wed Oct 23 18:30:22 2019 -0700
+++ b/contrib/automation/hgautomation/windows.py	Wed Oct 23 18:37:36 2019 -0700
@@ -71,7 +71,7 @@
 BUILD_INNO = r'''
 Set-Location C:\hgdev\src
 $python = "C:\hgdev\python27-{arch}\python.exe"
-C:\hgdev\python37-x64\python.exe contrib\packaging\inno\build.py --python $python
+C:\hgdev\python37-x64\python.exe contrib\packaging\packaging.py inno --python $python
 if ($LASTEXITCODE -ne 0) {{
     throw "process exited non-0: $LASTEXITCODE"
 }}
@@ -88,7 +88,7 @@
 BUILD_WIX = r'''
 Set-Location C:\hgdev\src
 $python = "C:\hgdev\python27-{arch}\python.exe"
-C:\hgdev\python37-x64\python.exe contrib\packaging\wix\build.py --python $python {extra_args}
+C:\hgdev\python37-x64\python.exe contrib\packaging\packaging.py wix --python $python {extra_args}
 if ($LASTEXITCODE -ne 0) {{
     throw "process exited non-0: $LASTEXITCODE"
 }}

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/contrib/packaging/hgpackaging/cli.py	Wed Oct 23 18:37:36 2019 -0700
@@ -0,0 +1,153 @@
+# cli.py - Command line interface for automation
+#
+# Copyright 2019 Gregory Szorc <[e-mail]>
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+# no-check-code because Python 3 native.
+
+import argparse
+import os
+import pathlib
+
+from . import (
+    inno,
+    wix,
+)
+
+HERE = pathlib.Path(os.path.abspath(os.path.dirname(__file__)))
+SOURCE_DIR = HERE.parent.parent.parent
+
+
+def build_inno(python=None, iscc=None, version=None):
+    if not os.path.isabs(python):
+        raise Exception("--python arg must be an absolute path")
+
+    if iscc:
+        iscc = pathlib.Path(iscc)
+    else:
+        iscc = (
+            pathlib.Path(os.environ["ProgramFiles(x86)"])
+            / "Inno Setup 5"
+            / "ISCC.exe"
+        )
+
+    build_dir = SOURCE_DIR / "build"
+
+    inno.build(
+        SOURCE_DIR, build_dir, pathlib.Path(python), iscc, version=version,
+    )
+
+
+def build_wix(
+    name=None,
+    python=None,
+    version=None,
+    sign_sn=None,
+    sign_cert=None,
+    sign_password=None,
+    sign_timestamp_url=None,
+    extra_packages_script=None,
+    extra_wxs=None,
+    extra_features=None,
+):
+    fn = wix.build_installer
+    kwargs = {
+        "source_dir": SOURCE_DIR,
+        "python_exe": pathlib.Path(python),
+        "version": version,
+    }
+
+    if not os.path.isabs(python):
+        raise Exception("--python arg must be an absolute path")
+
+    if extra_packages_script:
+        kwargs["extra_packages_script"] = extra_packages_script
+    if extra_wxs:
+        kwargs["extra_wxs"] = dict(
+            thing.split("=") for thing in extra_wxs.split(",")
+        )
+    if extra_features:
+        kwargs["extra_features"] = extra_features.split(",")
+
+    if sign_sn or sign_cert:
+        fn = wix.build_signed_installer
+        kwargs["name"] = name
+        kwargs["subject_name"] = sign_sn
+        kwargs["cert_path"] = sign_cert
+        kwargs["cert_password"] = sign_password
+        kwargs["timestamp_url"] = sign_timestamp_url
+
+    fn(**kwargs)
+
+
+def get_parser():
+    parser = argparse.ArgumentParser()
+
+    subparsers = parser.add_subparsers()
+
+    sp = subparsers.add_parser("inno", help="Build Inno Setup installer")
+    sp.add_argument("--python", required=True, help="path to python.exe to use")
+    sp.add_argument("--iscc", help="path to iscc.exe to use")
+    sp.add_argument(
+        "--version",
+        help="Mercurial version string to use "
+        "(detected from __version__.py if not defined",
+    )
+    sp.set_defaults(func=build_inno)
+
+    sp = subparsers.add_parser(
+        "wix", help="Build Windows installer with WiX Toolset"
+    )
+    sp.add_argument("--name", help="Application name", default="Mercurial")
+    sp.add_argument(
+        "--python", help="Path to Python executable to use", required=True
+    )
+    sp.add_argument(
+        "--sign-sn",
+        help="Subject name (or fragment thereof) of certificate "
+        "to use for signing",
+    )
+    sp.add_argument(
+        "--sign-cert", help="Path to certificate to use for signing"
+    )
+    sp.add_argument("--sign-password", help="Password for signing certificate")
+    sp.add_argument(
+        "--sign-timestamp-url",
+        help="URL of timestamp server to use for signing",
+    )
+    sp.add_argument("--version", help="Version string to use")
+    sp.add_argument(
+        "--extra-packages-script",
+        help=(
+            "Script to execute to include extra packages in " "py2exe binary."
+        ),
+    )
+    sp.add_argument(
+        "--extra-wxs", help="CSV of path_to_wxs_file=working_dir_for_wxs_file"
+    )
+    sp.add_argument(
+        "--extra-features",
+        help=(
+            "CSV of extra feature names to include "
+            "in the installer from the extra wxs files"
+        ),
+    )
+    sp.set_defaults(func=build_wix)
+
+    return parser
+
+
+def main():
+    parser = get_parser()
+    args = parser.parse_args()
+
+    if not hasattr(args, "func"):
+        parser.print_help()
+        return
+
+    kwargs = dict(vars(args))
+    del kwargs["func"]
+
+    args.func(**kwargs)

--- a/contrib/packaging/inno/build.py	Wed Oct 23 18:30:22 2019 -0700
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,60 +0,0 @@
-#!/usr/bin/env python3
-# build.py - Inno installer build script.
-#
-# Copyright 2019 Gregory Szorc <[e-mail]>
-#
-# This software may be used and distributed according to the terms of the
-# GNU General Public License version 2 or any later version.
-
-# This script automates the building of the Inno MSI installer for Mercurial.
-
-# no-check-code because Python 3 native.
-
-import argparse
-import os
-import pathlib
-import sys
-
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser()
-
-    parser.add_argument(
-        '--python', required=True, help='path to python.exe to use'
-    )
-    parser.add_argument('--iscc', help='path to iscc.exe to use')
-    parser.add_argument(
-        '--version',
-        help='Mercurial version string to use '
-        '(detected from __version__.py if not defined',
-    )
-
-    args = parser.parse_args()
-
-    if not os.path.isabs(args.python):
-        raise Exception('--python arg must be an absolute path')
-
-    if args.iscc:
-        iscc = pathlib.Path(args.iscc)
-    else:
-        iscc = (
-            pathlib.Path(os.environ['ProgramFiles(x86)'])
-            / 'Inno Setup 5'
-            / 'ISCC.exe'
-        )
-
-    here = pathlib.Path(os.path.abspath(os.path.dirname(__file__)))
-    source_dir = here.parent.parent.parent
-    build_dir = source_dir / 'build'
-
-    sys.path.insert(0, str(source_dir / 'contrib' / 'packaging'))
-
-    from hgpackaging.inno import build
-
-    build(
-        source_dir,
-        build_dir,
-        pathlib.Path(args.python),
-        iscc,
-        version=args.version,
-    )

--- a/contrib/packaging/inno/readme.rst	Wed Oct 23 18:30:22 2019 -0700
+++ b/contrib/packaging/inno/readme.rst	Wed Oct 23 18:37:36 2019 -0700
@@ -11,12 +11,12 @@
 * Inno Setup (http://jrsoftware.org/isdl.php) version 5.4 or newer.
   Be sure to install the optional Inno Setup Preprocessor feature,
   which is required.
-* Python 3.5+ (to run the ``build.py`` script)
+* Python 3.5+ (to run the ``packaging.py`` script)
 
 Building
 ========
 
-The ``build.py`` script automates the process of producing an
+The ``packaging.py`` script automates the process of producing an
 Inno installer. It manages fetching and configuring the
 non-system dependencies (such as py2exe, gettext, and various
 Python packages).
@@ -31,11 +31,11 @@
 From the prompt, change to the Mercurial source directory. e.g.
 ``cd c:\src\hg``.
 
-Next, invoke ``build.py`` to produce an Inno installer. You will
+Next, invoke ``packaging.py`` to produce an Inno installer. You will
 need to supply the path to the Python interpreter to use.::
 
-   $ python3.exe contrib\packaging\inno\build.py \
-       --python c:\python27\python.exe
+   $ python3.exe contrib\packaging\packaging.py \
+       inno --python c:\python27\python.exe
 
 .. note::
 
@@ -49,13 +49,13 @@
 and an installer placed in the ``dist`` sub-directory. The final
 line of output should print the name of the generated installer.
 
-Additional options may be configured. Run ``build.py --help`` to
-see a list of program flags.
+Additional options may be configured. Run
+``packaging.py inno --help`` to see a list of program flags.
 
 MinGW
 =====
 
 It is theoretically possible to generate an installer that uses
-MinGW. This isn't well tested and ``build.py`` and may properly
+MinGW. This isn't well tested and ``packaging.py`` and may properly
 support it. See old versions of this file in version control for
 potentially useful hints as to how to achieve this.

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/contrib/packaging/packaging.py	Wed Oct 23 18:37:36 2019 -0700
@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+#
+# packaging.py - Mercurial packaging functionality
+#
+# Copyright 2019 Gregory Szorc <[e-mail]>
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+import os
+import pathlib
+import subprocess
+import sys
+import venv
+
+
+HERE = pathlib.Path(os.path.abspath(__file__)).parent
+REQUIREMENTS_TXT = HERE / "requirements.txt"
+SOURCE_DIR = HERE.parent.parent
+VENV = SOURCE_DIR / "build" / "venv-packaging"
+
+
+def bootstrap():
+    venv_created = not VENV.exists()
+
+    VENV.parent.mkdir(exist_ok=True)
+
+    venv.create(VENV, with_pip=True)
+
+    if os.name == "nt":
+        venv_bin = VENV / "Scripts"
+        pip = venv_bin / "pip.exe"
+        python = venv_bin / "python.exe"
+    else:
+        venv_bin = VENV / "bin"
+        pip = venv_bin / "pip"
+        python = venv_bin / "python"
+
+    args = [
+        str(pip),
+        "install",
+        "-r",
+        str(REQUIREMENTS_TXT),
+        "--disable-pip-version-check",
+    ]
+
+    if not venv_created:
+        args.append("-q")
+
+    subprocess.run(args, check=True)
+
+    os.environ["HGPACKAGING_BOOTSTRAPPED"] = "1"
+    os.environ["PATH"] = "%s%s%s" % (venv_bin, os.pathsep, os.environ["PATH"])
+
+    subprocess.run([str(python), __file__] + sys.argv[1:], check=True)
+
+
+def run():
+    import hgpackaging.cli as cli
+
+    # Need to strip off main Python executable.
+    cli.main()
+
+
+if __name__ == "__main__":
+    try:
+        if "HGPACKAGING_BOOTSTRAPPED" not in os.environ:
+            bootstrap()
+        else:
+            run()
+    except subprocess.CalledProcessError as e:
+        sys.exit(e.returncode)
+    except KeyboardInterrupt:
+        sys.exit(1)

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/contrib/packaging/requirements.txt	Wed Oct 23 18:37:36 2019 -0700
@@ -0,0 +1,39 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile --generate-hashes --output-file=contrib/packaging/requirements.txt contrib/packaging/requirements.txt.in
+#
+jinja2==2.10.3 \
+    --hash=sha256:74320bb91f31270f9551d46522e33af46a80c3d619f4a4bf42b3164d30b5911f \
+    --hash=sha256:9fe95f19286cfefaa917656583d020be14e7859c6b0252588391e47db34527de
+markupsafe==1.1.1 \
+    --hash=sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473 \
+    --hash=sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161 \
+    --hash=sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235 \
+    --hash=sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5 \
+    --hash=sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff \
+    --hash=sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b \
+    --hash=sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1 \
+    --hash=sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e \
+    --hash=sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183 \
+    --hash=sha256:535f6fc4d397c1563d08b88e485c3496cf5784e927af890fb3c3aac7f933ec66 \
+    --hash=sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1 \
+    --hash=sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1 \
+    --hash=sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e \
+    --hash=sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b \
+    --hash=sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905 \
+    --hash=sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735 \
+    --hash=sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d \
+    --hash=sha256:98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e \
+    --hash=sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d \
+    --hash=sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c \
+    --hash=sha256:ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21 \
+    --hash=sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2 \
+    --hash=sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5 \
+    --hash=sha256:b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b \
+    --hash=sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6 \
+    --hash=sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f \
+    --hash=sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f \
+    --hash=sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7 \
+    # via jinja2

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/contrib/packaging/requirements.txt.in	Wed Oct 23 18:37:36 2019 -0700
@@ -0,0 +1,1 @@
+jinja2

--- a/contrib/packaging/wix/build.py	Wed Oct 23 18:30:22 2019 -0700
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,96 +0,0 @@
-#!/usr/bin/env python3
-# Copyright 2019 Gregory Szorc <[e-mail]>
-#
-# This software may be used and distributed according to the terms of the
-# GNU General Public License version 2 or any later version.
-
-# no-check-code because Python 3 native.
-
-"""Code to build Mercurial WiX installer."""
-
-import argparse
-import os
-import pathlib
-import sys
-
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser()
-
-    parser.add_argument('--name', help='Application name', default='Mercurial')
-    parser.add_argument(
-        '--python', help='Path to Python executable to use', required=True
-    )
-    parser.add_argument(
-        '--sign-sn',
-        help='Subject name (or fragment thereof) of certificate '
-        'to use for signing',
-    )
-    parser.add_argument(
-        '--sign-cert', help='Path to certificate to use for signing'
-    )
-    parser.add_argument(
-        '--sign-password', help='Password for signing certificate'
-    )
-    parser.add_argument(
-        '--sign-timestamp-url',
-        help='URL of timestamp server to use for signing',
-    )
-    parser.add_argument('--version', help='Version string to use')
-    parser.add_argument(
-        '--extra-packages-script',
-        help=(
-            'Script to execute to include extra packages in ' 'py2exe binary.'
-        ),
-    )
-    parser.add_argument(
-        '--extra-wxs', help='CSV of path_to_wxs_file=working_dir_for_wxs_file'
-    )
-    parser.add_argument(
-        '--extra-features',
-        help=(
-            'CSV of extra feature names to include '
-            'in the installer from the extra wxs files'
-        ),
-    )
-
-    args = parser.parse_args()
-
-    here = pathlib.Path(os.path.abspath(os.path.dirname(__file__)))
-    source_dir = here.parent.parent.parent
-
-    sys.path.insert(0, str(source_dir / 'contrib' / 'packaging'))
-
-    from hgpackaging.wix import (
-        build_installer,
-        build_signed_installer,
-    )
-
-    fn = build_installer
-    kwargs = {
-        'source_dir': source_dir,
-        'python_exe': pathlib.Path(args.python),
-        'version': args.version,
-    }
-
-    if not os.path.isabs(args.python):
-        raise Exception('--python arg must be an absolute path')
-
-    if args.extra_packages_script:
-        kwargs['extra_packages_script'] = args.extra_packages_script
-    if args.extra_wxs:
-        kwargs['extra_wxs'] = dict(
-            thing.split("=") for thing in args.extra_wxs.split(',')
-        )
-    if args.extra_features:
-        kwargs['extra_features'] = args.extra_features.split(',')
-
-    if args.sign_sn or args.sign_cert:
-        fn = build_signed_installer
-        kwargs['name'] = args.name
-        kwargs['subject_name'] = args.sign_sn
-        kwargs['cert_path'] = args.sign_cert
-        kwargs['cert_password'] = args.sign_password
-        kwargs['timestamp_url'] = args.sign_timestamp_url
-
-    fn(**kwargs)

--- a/contrib/packaging/wix/readme.rst	Wed Oct 23 18:30:22 2019 -0700
+++ b/contrib/packaging/wix/readme.rst	Wed Oct 23 18:37:36 2019 -0700
@@ -18,12 +18,12 @@
 * Python 2.7 (download from https://www.python.org/downloads/)
 * Microsoft Visual C++ Compiler for Python 2.7
   (https://www.microsoft.com/en-us/download/details.aspx?id=44266)
-* Python 3.5+ (to run the ``build.py`` script)
+* Python 3.5+ (to run the ``packaging.py`` script)
 
 Building
 ========
 
-The ``build.py`` script automates the process of producing an MSI
+The ``packaging.py`` script automates the process of producing an MSI
 installer. It manages fetching and configuring non-system dependencies
 (such as py2exe, gettext, and various Python packages).
 
@@ -37,11 +37,11 @@
 From the prompt, change to the Mercurial source directory. e.g.
 ``cd c:\src\hg``.
 
-Next, invoke ``build.py`` to produce an MSI installer. You will need
+Next, invoke ``packaging.py`` to produce an MSI installer. You will need
 to supply the path to the Python interpreter to use.::
 
-   $ python3 contrib\packaging\wix\build.py \
-      --python c:\python27\python.exe
+   $ python3 contrib\packaging\packaging.py \
+      wix --python c:\python27\python.exe
 
 .. note::
 
@@ -54,8 +54,8 @@
 and an installer placed in the ``dist`` sub-directory. The final line
 of output should print the name of the generated installer.
 
-Additional options may be configured. Run ``build.py --help`` to see
-a list of program flags.
+Additional options may be configured. Run ``packaging.py wix --help`` to
+see a list of program flags.
 
 Relationship to TortoiseHG
 ==========================

--- a/tests/test-check-code.t	Wed Oct 23 18:30:22 2019 -0700
+++ b/tests/test-check-code.t	Wed Oct 23 18:37:36 2019 -0700
@@ -21,13 +21,12 @@
   Skipping contrib/automation/hgautomation/try_server.py it has no-che?k-code (glob)
   Skipping contrib/automation/hgautomation/windows.py it has no-che?k-code (glob)
   Skipping contrib/automation/hgautomation/winrm.py it has no-che?k-code (glob)
+  Skipping contrib/packaging/hgpackaging/cli.py it has no-che?k-code (glob)
   Skipping contrib/packaging/hgpackaging/downloads.py it has no-che?k-code (glob)
   Skipping contrib/packaging/hgpackaging/inno.py it has no-che?k-code (glob)
   Skipping contrib/packaging/hgpackaging/py2exe.py it has no-che?k-code (glob)
   Skipping contrib/packaging/hgpackaging/util.py it has no-che?k-code (glob)
   Skipping contrib/packaging/hgpackaging/wix.py it has no-che?k-code (glob)
-  Skipping contrib/packaging/inno/build.py it has no-che?k-code (glob)
-  Skipping contrib/packaging/wix/build.py it has no-che?k-code (glob)
   Skipping i18n/polib.py it has no-che?k-code (glob)
   Skipping mercurial/statprof.py it has no-che?k-code (glob)
   Skipping tests/badserverext.py it has no-che?k-code (glob)

--- a/tests/test-check-py3-compat.t	Wed Oct 23 18:30:22 2019 -0700
+++ b/tests/test-check-py3-compat.t	Wed Oct 23 18:37:36 2019 -0700
@@ -8,6 +8,7 @@
   > -X contrib/automation/ \
   > -X contrib/packaging/hgpackaging/ \
   > -X contrib/packaging/inno/ \
+  > -X contrib/packaging/packaging.py \
   > -X contrib/packaging/wix/ \
   > -X hgdemandimport/demandimportpy2.py \
   > -X mercurial/thirdparty/cbor \