# HG changeset patch # User Gregory Szorc # Date 1571881056 25200 # Node ID 081a77df7bc65e18cce5c58dd3a879991d4d724e # Parent 5a0a4fa055ff9f3953939ca8a5cdb049eaf94f84 packaging: consolidate CLI functionality into packaging.py Consolidating functionality for invoking code in the hgpackaging package through a single CLI entry point will make things simpler when we add more complexity to that package. For example, it will allow us to run things out of a virtualenv with third party packages. This commit consolidates functionality from the Inno and WiX build.py scripts into a new packaging.py script. That script simply creates a virtualenv and runs the CLI functionality in it. The new virtualenv is populated with jinja2 because I felt it easier to incorporate requirements file processing in this commit and we will soon use jinja2 in an upcoming commit. The unified CLI functionality will also make it easier to script other packaging workflows going forward. e.g. RPM, Debian, and macOS packaging. Differential Revision: https://phab.mercurial-scm.org/D7156 diff -r 5a0a4fa055ff -r 081a77df7bc6 contrib/automation/hgautomation/windows.py --- a/contrib/automation/hgautomation/windows.py Wed Oct 23 18:30:22 2019 -0700 +++ b/contrib/automation/hgautomation/windows.py Wed Oct 23 18:37:36 2019 -0700 @@ -71,7 +71,7 @@ BUILD_INNO = r''' Set-Location C:\hgdev\src $python = "C:\hgdev\python27-{arch}\python.exe" -C:\hgdev\python37-x64\python.exe contrib\packaging\inno\build.py --python $python +C:\hgdev\python37-x64\python.exe contrib\packaging\packaging.py inno --python $python if ($LASTEXITCODE -ne 0) {{ throw "process exited non-0: $LASTEXITCODE" }} @@ -88,7 +88,7 @@ BUILD_WIX = r''' Set-Location C:\hgdev\src $python = "C:\hgdev\python27-{arch}\python.exe" -C:\hgdev\python37-x64\python.exe contrib\packaging\wix\build.py --python $python {extra_args} +C:\hgdev\python37-x64\python.exe contrib\packaging\packaging.py wix --python $python {extra_args} if ($LASTEXITCODE -ne 0) {{ throw "process exited non-0: $LASTEXITCODE" }} diff -r 5a0a4fa055ff -r 081a77df7bc6 contrib/packaging/hgpackaging/cli.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/packaging/hgpackaging/cli.py Wed Oct 23 18:37:36 2019 -0700 @@ -0,0 +1,153 @@ +# cli.py - Command line interface for automation +# +# Copyright 2019 Gregory Szorc +# +# This software may be used and distributed according to the terms of the +# GNU General Public License version 2 or any later version. + +# no-check-code because Python 3 native. + +import argparse +import os +import pathlib + +from . import ( + inno, + wix, +) + +HERE = pathlib.Path(os.path.abspath(os.path.dirname(__file__))) +SOURCE_DIR = HERE.parent.parent.parent + + +def build_inno(python=None, iscc=None, version=None): + if not os.path.isabs(python): + raise Exception("--python arg must be an absolute path") + + if iscc: + iscc = pathlib.Path(iscc) + else: + iscc = ( + pathlib.Path(os.environ["ProgramFiles(x86)"]) + / "Inno Setup 5" + / "ISCC.exe" + ) + + build_dir = SOURCE_DIR / "build" + + inno.build( + SOURCE_DIR, build_dir, pathlib.Path(python), iscc, version=version, + ) + + +def build_wix( + name=None, + python=None, + version=None, + sign_sn=None, + sign_cert=None, + sign_password=None, + sign_timestamp_url=None, + extra_packages_script=None, + extra_wxs=None, + extra_features=None, +): + fn = wix.build_installer + kwargs = { + "source_dir": SOURCE_DIR, + "python_exe": pathlib.Path(python), + "version": version, + } + + if not os.path.isabs(python): + raise Exception("--python arg must be an absolute path") + + if extra_packages_script: + kwargs["extra_packages_script"] = extra_packages_script + if extra_wxs: + kwargs["extra_wxs"] = dict( + thing.split("=") for thing in extra_wxs.split(",") + ) + if extra_features: + kwargs["extra_features"] = extra_features.split(",") + + if sign_sn or sign_cert: + fn = wix.build_signed_installer + kwargs["name"] = name + kwargs["subject_name"] = sign_sn + kwargs["cert_path"] = sign_cert + kwargs["cert_password"] = sign_password + kwargs["timestamp_url"] = sign_timestamp_url + + fn(**kwargs) + + +def get_parser(): + parser = argparse.ArgumentParser() + + subparsers = parser.add_subparsers() + + sp = subparsers.add_parser("inno", help="Build Inno Setup installer") + sp.add_argument("--python", required=True, help="path to python.exe to use") + sp.add_argument("--iscc", help="path to iscc.exe to use") + sp.add_argument( + "--version", + help="Mercurial version string to use " + "(detected from __version__.py if not defined", + ) + sp.set_defaults(func=build_inno) + + sp = subparsers.add_parser( + "wix", help="Build Windows installer with WiX Toolset" + ) + sp.add_argument("--name", help="Application name", default="Mercurial") + sp.add_argument( + "--python", help="Path to Python executable to use", required=True + ) + sp.add_argument( + "--sign-sn", + help="Subject name (or fragment thereof) of certificate " + "to use for signing", + ) + sp.add_argument( + "--sign-cert", help="Path to certificate to use for signing" + ) + sp.add_argument("--sign-password", help="Password for signing certificate") + sp.add_argument( + "--sign-timestamp-url", + help="URL of timestamp server to use for signing", + ) + sp.add_argument("--version", help="Version string to use") + sp.add_argument( + "--extra-packages-script", + help=( + "Script to execute to include extra packages in " "py2exe binary." + ), + ) + sp.add_argument( + "--extra-wxs", help="CSV of path_to_wxs_file=working_dir_for_wxs_file" + ) + sp.add_argument( + "--extra-features", + help=( + "CSV of extra feature names to include " + "in the installer from the extra wxs files" + ), + ) + sp.set_defaults(func=build_wix) + + return parser + + +def main(): + parser = get_parser() + args = parser.parse_args() + + if not hasattr(args, "func"): + parser.print_help() + return + + kwargs = dict(vars(args)) + del kwargs["func"] + + args.func(**kwargs) diff -r 5a0a4fa055ff -r 081a77df7bc6 contrib/packaging/inno/build.py --- a/contrib/packaging/inno/build.py Wed Oct 23 18:30:22 2019 -0700 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,60 +0,0 @@ -#!/usr/bin/env python3 -# build.py - Inno installer build script. -# -# Copyright 2019 Gregory Szorc -# -# This software may be used and distributed according to the terms of the -# GNU General Public License version 2 or any later version. - -# This script automates the building of the Inno MSI installer for Mercurial. - -# no-check-code because Python 3 native. - -import argparse -import os -import pathlib -import sys - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - - parser.add_argument( - '--python', required=True, help='path to python.exe to use' - ) - parser.add_argument('--iscc', help='path to iscc.exe to use') - parser.add_argument( - '--version', - help='Mercurial version string to use ' - '(detected from __version__.py if not defined', - ) - - args = parser.parse_args() - - if not os.path.isabs(args.python): - raise Exception('--python arg must be an absolute path') - - if args.iscc: - iscc = pathlib.Path(args.iscc) - else: - iscc = ( - pathlib.Path(os.environ['ProgramFiles(x86)']) - / 'Inno Setup 5' - / 'ISCC.exe' - ) - - here = pathlib.Path(os.path.abspath(os.path.dirname(__file__))) - source_dir = here.parent.parent.parent - build_dir = source_dir / 'build' - - sys.path.insert(0, str(source_dir / 'contrib' / 'packaging')) - - from hgpackaging.inno import build - - build( - source_dir, - build_dir, - pathlib.Path(args.python), - iscc, - version=args.version, - ) diff -r 5a0a4fa055ff -r 081a77df7bc6 contrib/packaging/inno/readme.rst --- a/contrib/packaging/inno/readme.rst Wed Oct 23 18:30:22 2019 -0700 +++ b/contrib/packaging/inno/readme.rst Wed Oct 23 18:37:36 2019 -0700 @@ -11,12 +11,12 @@ * Inno Setup (http://jrsoftware.org/isdl.php) version 5.4 or newer. Be sure to install the optional Inno Setup Preprocessor feature, which is required. -* Python 3.5+ (to run the ``build.py`` script) +* Python 3.5+ (to run the ``packaging.py`` script) Building ======== -The ``build.py`` script automates the process of producing an +The ``packaging.py`` script automates the process of producing an Inno installer. It manages fetching and configuring the non-system dependencies (such as py2exe, gettext, and various Python packages). @@ -31,11 +31,11 @@ From the prompt, change to the Mercurial source directory. e.g. ``cd c:\src\hg``. -Next, invoke ``build.py`` to produce an Inno installer. You will +Next, invoke ``packaging.py`` to produce an Inno installer. You will need to supply the path to the Python interpreter to use.:: - $ python3.exe contrib\packaging\inno\build.py \ - --python c:\python27\python.exe + $ python3.exe contrib\packaging\packaging.py \ + inno --python c:\python27\python.exe .. note:: @@ -49,13 +49,13 @@ and an installer placed in the ``dist`` sub-directory. The final line of output should print the name of the generated installer. -Additional options may be configured. Run ``build.py --help`` to -see a list of program flags. +Additional options may be configured. Run +``packaging.py inno --help`` to see a list of program flags. MinGW ===== It is theoretically possible to generate an installer that uses -MinGW. This isn't well tested and ``build.py`` and may properly +MinGW. This isn't well tested and ``packaging.py`` and may properly support it. See old versions of this file in version control for potentially useful hints as to how to achieve this. diff -r 5a0a4fa055ff -r 081a77df7bc6 contrib/packaging/packaging.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/packaging/packaging.py Wed Oct 23 18:37:36 2019 -0700 @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +# +# packaging.py - Mercurial packaging functionality +# +# Copyright 2019 Gregory Szorc +# +# This software may be used and distributed according to the terms of the +# GNU General Public License version 2 or any later version. + +import os +import pathlib +import subprocess +import sys +import venv + + +HERE = pathlib.Path(os.path.abspath(__file__)).parent +REQUIREMENTS_TXT = HERE / "requirements.txt" +SOURCE_DIR = HERE.parent.parent +VENV = SOURCE_DIR / "build" / "venv-packaging" + + +def bootstrap(): + venv_created = not VENV.exists() + + VENV.parent.mkdir(exist_ok=True) + + venv.create(VENV, with_pip=True) + + if os.name == "nt": + venv_bin = VENV / "Scripts" + pip = venv_bin / "pip.exe" + python = venv_bin / "python.exe" + else: + venv_bin = VENV / "bin" + pip = venv_bin / "pip" + python = venv_bin / "python" + + args = [ + str(pip), + "install", + "-r", + str(REQUIREMENTS_TXT), + "--disable-pip-version-check", + ] + + if not venv_created: + args.append("-q") + + subprocess.run(args, check=True) + + os.environ["HGPACKAGING_BOOTSTRAPPED"] = "1" + os.environ["PATH"] = "%s%s%s" % (venv_bin, os.pathsep, os.environ["PATH"]) + + subprocess.run([str(python), __file__] + sys.argv[1:], check=True) + + +def run(): + import hgpackaging.cli as cli + + # Need to strip off main Python executable. + cli.main() + + +if __name__ == "__main__": + try: + if "HGPACKAGING_BOOTSTRAPPED" not in os.environ: + bootstrap() + else: + run() + except subprocess.CalledProcessError as e: + sys.exit(e.returncode) + except KeyboardInterrupt: + sys.exit(1) diff -r 5a0a4fa055ff -r 081a77df7bc6 contrib/packaging/requirements.txt --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/packaging/requirements.txt Wed Oct 23 18:37:36 2019 -0700 @@ -0,0 +1,39 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --generate-hashes --output-file=contrib/packaging/requirements.txt contrib/packaging/requirements.txt.in +# +jinja2==2.10.3 \ + --hash=sha256:74320bb91f31270f9551d46522e33af46a80c3d619f4a4bf42b3164d30b5911f \ + --hash=sha256:9fe95f19286cfefaa917656583d020be14e7859c6b0252588391e47db34527de +markupsafe==1.1.1 \ + --hash=sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473 \ + --hash=sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161 \ + --hash=sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235 \ + --hash=sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5 \ + --hash=sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff \ + --hash=sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b \ + --hash=sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1 \ + --hash=sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e \ + --hash=sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183 \ + --hash=sha256:535f6fc4d397c1563d08b88e485c3496cf5784e927af890fb3c3aac7f933ec66 \ + --hash=sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1 \ + --hash=sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1 \ + --hash=sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e \ + --hash=sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b \ + --hash=sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905 \ + --hash=sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735 \ + --hash=sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d \ + --hash=sha256:98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e \ + --hash=sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d \ + --hash=sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c \ + --hash=sha256:ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21 \ + --hash=sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2 \ + --hash=sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5 \ + --hash=sha256:b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b \ + --hash=sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6 \ + --hash=sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f \ + --hash=sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f \ + --hash=sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7 \ + # via jinja2 diff -r 5a0a4fa055ff -r 081a77df7bc6 contrib/packaging/requirements.txt.in --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/contrib/packaging/requirements.txt.in Wed Oct 23 18:37:36 2019 -0700 @@ -0,0 +1,1 @@ +jinja2 diff -r 5a0a4fa055ff -r 081a77df7bc6 contrib/packaging/wix/build.py --- a/contrib/packaging/wix/build.py Wed Oct 23 18:30:22 2019 -0700 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,96 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2019 Gregory Szorc -# -# This software may be used and distributed according to the terms of the -# GNU General Public License version 2 or any later version. - -# no-check-code because Python 3 native. - -"""Code to build Mercurial WiX installer.""" - -import argparse -import os -import pathlib -import sys - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - - parser.add_argument('--name', help='Application name', default='Mercurial') - parser.add_argument( - '--python', help='Path to Python executable to use', required=True - ) - parser.add_argument( - '--sign-sn', - help='Subject name (or fragment thereof) of certificate ' - 'to use for signing', - ) - parser.add_argument( - '--sign-cert', help='Path to certificate to use for signing' - ) - parser.add_argument( - '--sign-password', help='Password for signing certificate' - ) - parser.add_argument( - '--sign-timestamp-url', - help='URL of timestamp server to use for signing', - ) - parser.add_argument('--version', help='Version string to use') - parser.add_argument( - '--extra-packages-script', - help=( - 'Script to execute to include extra packages in ' 'py2exe binary.' - ), - ) - parser.add_argument( - '--extra-wxs', help='CSV of path_to_wxs_file=working_dir_for_wxs_file' - ) - parser.add_argument( - '--extra-features', - help=( - 'CSV of extra feature names to include ' - 'in the installer from the extra wxs files' - ), - ) - - args = parser.parse_args() - - here = pathlib.Path(os.path.abspath(os.path.dirname(__file__))) - source_dir = here.parent.parent.parent - - sys.path.insert(0, str(source_dir / 'contrib' / 'packaging')) - - from hgpackaging.wix import ( - build_installer, - build_signed_installer, - ) - - fn = build_installer - kwargs = { - 'source_dir': source_dir, - 'python_exe': pathlib.Path(args.python), - 'version': args.version, - } - - if not os.path.isabs(args.python): - raise Exception('--python arg must be an absolute path') - - if args.extra_packages_script: - kwargs['extra_packages_script'] = args.extra_packages_script - if args.extra_wxs: - kwargs['extra_wxs'] = dict( - thing.split("=") for thing in args.extra_wxs.split(',') - ) - if args.extra_features: - kwargs['extra_features'] = args.extra_features.split(',') - - if args.sign_sn or args.sign_cert: - fn = build_signed_installer - kwargs['name'] = args.name - kwargs['subject_name'] = args.sign_sn - kwargs['cert_path'] = args.sign_cert - kwargs['cert_password'] = args.sign_password - kwargs['timestamp_url'] = args.sign_timestamp_url - - fn(**kwargs) diff -r 5a0a4fa055ff -r 081a77df7bc6 contrib/packaging/wix/readme.rst --- a/contrib/packaging/wix/readme.rst Wed Oct 23 18:30:22 2019 -0700 +++ b/contrib/packaging/wix/readme.rst Wed Oct 23 18:37:36 2019 -0700 @@ -18,12 +18,12 @@ * Python 2.7 (download from https://www.python.org/downloads/) * Microsoft Visual C++ Compiler for Python 2.7 (https://www.microsoft.com/en-us/download/details.aspx?id=44266) -* Python 3.5+ (to run the ``build.py`` script) +* Python 3.5+ (to run the ``packaging.py`` script) Building ======== -The ``build.py`` script automates the process of producing an MSI +The ``packaging.py`` script automates the process of producing an MSI installer. It manages fetching and configuring non-system dependencies (such as py2exe, gettext, and various Python packages). @@ -37,11 +37,11 @@ From the prompt, change to the Mercurial source directory. e.g. ``cd c:\src\hg``. -Next, invoke ``build.py`` to produce an MSI installer. You will need +Next, invoke ``packaging.py`` to produce an MSI installer. You will need to supply the path to the Python interpreter to use.:: - $ python3 contrib\packaging\wix\build.py \ - --python c:\python27\python.exe + $ python3 contrib\packaging\packaging.py \ + wix --python c:\python27\python.exe .. note:: @@ -54,8 +54,8 @@ and an installer placed in the ``dist`` sub-directory. The final line of output should print the name of the generated installer. -Additional options may be configured. Run ``build.py --help`` to see -a list of program flags. +Additional options may be configured. Run ``packaging.py wix --help`` to +see a list of program flags. Relationship to TortoiseHG ========================== diff -r 5a0a4fa055ff -r 081a77df7bc6 tests/test-check-code.t --- a/tests/test-check-code.t Wed Oct 23 18:30:22 2019 -0700 +++ b/tests/test-check-code.t Wed Oct 23 18:37:36 2019 -0700 @@ -21,13 +21,12 @@ Skipping contrib/automation/hgautomation/try_server.py it has no-che?k-code (glob) Skipping contrib/automation/hgautomation/windows.py it has no-che?k-code (glob) Skipping contrib/automation/hgautomation/winrm.py it has no-che?k-code (glob) + Skipping contrib/packaging/hgpackaging/cli.py it has no-che?k-code (glob) Skipping contrib/packaging/hgpackaging/downloads.py it has no-che?k-code (glob) Skipping contrib/packaging/hgpackaging/inno.py it has no-che?k-code (glob) Skipping contrib/packaging/hgpackaging/py2exe.py it has no-che?k-code (glob) Skipping contrib/packaging/hgpackaging/util.py it has no-che?k-code (glob) Skipping contrib/packaging/hgpackaging/wix.py it has no-che?k-code (glob) - Skipping contrib/packaging/inno/build.py it has no-che?k-code (glob) - Skipping contrib/packaging/wix/build.py it has no-che?k-code (glob) Skipping i18n/polib.py it has no-che?k-code (glob) Skipping mercurial/statprof.py it has no-che?k-code (glob) Skipping tests/badserverext.py it has no-che?k-code (glob) diff -r 5a0a4fa055ff -r 081a77df7bc6 tests/test-check-py3-compat.t --- a/tests/test-check-py3-compat.t Wed Oct 23 18:30:22 2019 -0700 +++ b/tests/test-check-py3-compat.t Wed Oct 23 18:37:36 2019 -0700 @@ -8,6 +8,7 @@ > -X contrib/automation/ \ > -X contrib/packaging/hgpackaging/ \ > -X contrib/packaging/inno/ \ + > -X contrib/packaging/packaging.py \ > -X contrib/packaging/wix/ \ > -X hgdemandimport/demandimportpy2.py \ > -X mercurial/thirdparty/cbor \