view rust/hgcli/pyoxidizer.bzl @ 49491:c6a1beba27e9

bisect: avoid copying ancestor list for non-merge commits

During a bisection, hg needs to compute a list of all ancestors for every
candidate commit. This is accomplished via a bottom-up traversal of the set of
candidates, during which each revision's ancestor list is populated using the
ancestor list of its parent(s). Previously, this involved copying the entire
list, which could be very long in if the bisection range was large.

To help improve this, we can observe that each candidate commit is visited
exactly once, at which point its ancestor list is copied into its children's
lists and then dropped. In the case of non-merge commits, a commit's ancestor
list consists exactly of its parent's list plus itself. This means that we can
trivially reuse the parent's existing list for one of its non-merge children,
which avoids copying entirely if that commit is the parent's only child. This
makes bisections over linear ranges of commits much faster.

During some informal testing in the large publicly-available `mozilla-central`
repository, this noticeably sped up bisections over large ranges of history:

Setup:
  $ cd mozilla-central
  $ hg bisect --reset
  $ hg bisect --good 0
  $ hg log -r tip -T '{rev}\n'
  628417

Test:
  $ time hg bisect --bad tip --noupdate

Before:
  real    3m35.927s
  user    3m35.553s
  sys     0m0.319s

After:
  real    1m41.142s
  user    1m40.810s
  sys     0m0.285s

# The following variables can be passed in as parameters:
#
# VERSION
#   Version string of program being produced.
#
# MSI_NAME
#   Root name of MSI installer.
#
# EXTRA_MSI_FEATURES
#   ; delimited string of extra features to advertise in the built MSA.
#
# SIGNING_PFX_PATH
#   Path to code signing certificate to use.
#
# SIGNING_PFX_PASSWORD
#   Password to code signing PFX file defined by SIGNING_PFX_PATH.
#
# SIGNING_SUBJECT_NAME
#   String fragment in code signing certificate subject name used to find
#   code signing certificate in Windows certificate store.
#
# TIME_STAMP_SERVER_URL
#   URL of time-stamp token authority (RFC 3161) servers to stamp code signatures.

ROOT = CWD + "/../.."

VERSION = VARS.get("VERSION", "0.0")
MSI_NAME = VARS.get("MSI_NAME", "mercurial")
EXTRA_MSI_FEATURES = VARS.get("EXTRA_MSI_FEATURES")
SIGNING_PFX_PATH = VARS.get("SIGNING_PFX_PATH")
SIGNING_PFX_PASSWORD = VARS.get("SIGNING_PFX_PASSWORD", "")
SIGNING_SUBJECT_NAME = VARS.get("SIGNING_SUBJECT_NAME")
TIME_STAMP_SERVER_URL = VARS.get("TIME_STAMP_SERVER_URL", "http://timestamp.digicert.com")

IS_WINDOWS = "windows" in BUILD_TARGET_TRIPLE

# Use in-memory resources for all resources. If false, most of the Python
# stdlib will be in memory, but other things such as Mercurial itself will not
# be. See the comment in resource_callback, below.
USE_IN_MEMORY_RESOURCES = not IS_WINDOWS

# Code to run in Python interpreter.
RUN_CODE = """
import os
import sys
extra_path = os.environ.get('PYTHONPATH')
if extra_path is not None:
    # extensions and hooks expect a working python environment
    # We do not prepend the values because the Mercurial library wants to be in
    # the front of the sys.path to avoid picking up other installations.
    sys.path.extend(extra_path.split(os.pathsep))
# Add user site to sys.path to load extensions without the full path
if os.name == 'nt':
    vi = sys.version_info
    appdata = os.environ.get('APPDATA')
    if appdata:
        sys.path.append(
            os.path.join(
                appdata,
                'Python',
                'Python%d%d' % (vi[0], vi[1]),
                'site-packages',
            )
        )
elif sys.platform == "darwin":
    vi = sys.version_info

    def joinuser(*args):
        return os.path.expanduser(os.path.join(*args))

    # Note: site.py uses `sys._framework` instead of hardcoding "Python" as the
    #   3rd arg, but that is set to an empty string in an oxidized binary.  It
    #   has a fallback to ~/.local when `sys._framework` isn't set, but we want
    #   to match what the system python uses, so it sees pip installed stuff.
    usersite = joinuser("~", "Library", "Python",
                        "%d.%d" % vi[:2], "lib/python/site-packages")

    sys.path.append(usersite)
import hgdemandimport;
hgdemandimport.enable();
from mercurial import dispatch;
dispatch.run();
"""

set_build_path(ROOT + "/build/pyoxidizer")

def make_distribution():
    return default_python_distribution(python_version = "3.9")

def resource_callback(policy, resource):
    if USE_IN_MEMORY_RESOURCES:
        resource.add_location = "in-memory"
        return

    # We use a custom resource routing policy to influence where things are loaded
    # from.
    #
    # For Python modules and resources, we load from memory if they are in
    # the standard library and from the filesystem if not. This is because
    # parts of Mercurial and some 3rd party packages aren't yet compatible
    # with memory loading.
    #
    # For Python extension modules, we load from the filesystem because
    # this yields greatest compatibility.
    if type(resource) in ("PythonModuleSource", "PythonPackageResource", "PythonPackageDistributionResource"):
        if resource.is_stdlib:
            resource.add_location = "in-memory"
        else:
            resource.add_location = "filesystem-relative:lib"

    elif type(resource) == "PythonExtensionModule":
        resource.add_location = "filesystem-relative:lib"

def make_exe(dist):
    """Builds a Rust-wrapped Mercurial binary."""
    packaging_policy = dist.make_python_packaging_policy()

    # Extension may depend on any Python functionality. Include all
    # extensions.
    packaging_policy.extension_module_filter = "all"
    packaging_policy.resources_location = "in-memory"
    if not USE_IN_MEMORY_RESOURCES:
        packaging_policy.resources_location_fallback = "filesystem-relative:lib"
    packaging_policy.register_resource_callback(resource_callback)

    config = dist.make_python_interpreter_config()
    config.allocator_backend = "default"
    config.run_command = RUN_CODE

    # We want to let the user load extensions from the file system
    config.filesystem_importer = True

    # We need this to make resourceutil happy, since it looks for sys.frozen.
    config.sys_frozen = True
    config.legacy_windows_stdio = True

    exe = dist.to_python_executable(
        name = "hg",
        packaging_policy = packaging_policy,
        config = config,
    )

    # Add Mercurial to resources.
    exe.add_python_resources(exe.pip_install(["--verbose", "--no-use-pep517", ROOT]))

    # On Windows, we install extra packages for convenience.
    if IS_WINDOWS:
        exe.add_python_resources(
            exe.pip_install(["-r", ROOT + "/contrib/packaging/requirements-windows-py3.txt"]),
        )
    extra_packages = VARS.get("extra_py_packages", "")
    if extra_packages:
        for extra in extra_packages.split(","):
            extra_src, pkgs = extra.split("=")
            pkgs = pkgs.split(":")
            exe.add_python_resources(exe.read_package_root(extra_src, pkgs))

    return exe

def make_manifest(dist, exe):
    m = FileManifest()
    m.add_python_resource(".", exe)

    return m


# This adjusts the InstallManifest produced from exe generation to provide
# additional files found in a Windows install layout.
def make_windows_install_layout(manifest):
    # Copy various files to new install locations. This can go away once
    # we're using the importlib resource reader.
    RECURSIVE_COPIES = {
        "lib/mercurial/locale/": "locale/",
        "lib/mercurial/templates/": "templates/",
    }
    for (search, replace) in RECURSIVE_COPIES.items():
        for path in manifest.paths():
            if path.startswith(search):
                new_path = path.replace(search, replace)
                print("copy %s to %s" % (path, new_path))
                file = manifest.get_file(path)
                manifest.add_file(file, path = new_path)

    # Similar to above, but with filename pattern matching.
    # lib/mercurial/helptext/**/*.txt -> helptext/
    # lib/mercurial/defaultrc/*.rc -> defaultrc/
    for path in manifest.paths():
        if path.startswith("lib/mercurial/helptext/") and path.endswith(".txt"):
            new_path = path[len("lib/mercurial/"):]
        elif path.startswith("lib/mercurial/defaultrc/") and path.endswith(".rc"):
            new_path = path[len("lib/mercurial/"):]
        else:
            continue

        print("copying %s to %s" % (path, new_path))
        manifest.add_file(manifest.get_file(path), path = new_path)

    extra_install_files = VARS.get("extra_install_files", "")
    if extra_install_files:
        for extra in extra_install_files.split(","):
            print("adding extra files from %s" % extra)
            # TODO: I expected a ** glob to work, but it didn't.
            #
            # TODO: I know this has forward-slash paths. As far as I can tell,
            # backslashes don't ever match glob() expansions in 
            # tugger-starlark, even on Windows.
            manifest.add_manifest(glob(include=[extra + "/*/*"], strip_prefix=extra+"/"))

    # We also install a handful of additional files.
    EXTRA_CONTRIB_FILES = [
        "bash_completion",
        "hgweb.fcgi",
        "hgweb.wsgi",
        "logo-droplets.svg",
        "mercurial.el",
        "mq.el",
        "tcsh_completion",
        "tcsh_completion_build.sh",
        "xml.rnc",
        "zsh_completion",
    ]

    for f in EXTRA_CONTRIB_FILES:
        manifest.add_file(FileContent(path = ROOT + "/contrib/" + f), directory = "contrib")

    # Individual files with full source to destination path mapping.
    EXTRA_FILES = {
        "contrib/hgk": "contrib/hgk.tcl",
        "contrib/win32/postinstall.txt": "ReleaseNotes.txt",
        "contrib/win32/ReadMe.html": "ReadMe.html",
        "doc/style.css": "doc/style.css",
        "COPYING": "Copying.txt",
    }

    for source, dest in EXTRA_FILES.items():
        print("adding extra file %s" % dest)
        manifest.add_file(FileContent(path = ROOT + "/" + source), path = dest)

    # And finally some wildcard matches.
    manifest.add_manifest(glob(
        include = [ROOT + "/contrib/vim/*"],
        strip_prefix = ROOT + "/"
    ))
    manifest.add_manifest(glob(
        include = [ROOT + "/doc/*.html"],
        strip_prefix = ROOT + "/"
    ))

    # But we don't ship hg-ssh on Windows, so exclude its documentation.
    manifest.remove("doc/hg-ssh.8.html")

    return manifest


def make_msi(manifest):
    manifest = make_windows_install_layout(manifest)

    if "x86_64" in BUILD_TARGET_TRIPLE:
        platform = "x64"
    else:
        platform = "x86"

    manifest.add_file(
        FileContent(path = ROOT + "/contrib/packaging/wix/COPYING.rtf"),
        path = "COPYING.rtf",
    )
    manifest.remove("Copying.txt")
    manifest.add_file(
        FileContent(path = ROOT + "/contrib/win32/mercurial.ini"),
        path = "defaultrc/mercurial.rc",
    )
    manifest.add_file(
        FileContent(filename = "editor.rc", content = "[ui]\neditor = notepad\n"),
        path = "defaultrc/editor.rc",
    )

    wix = WiXInstaller(
        "hg",
        "%s-%s-%s.msi" % (MSI_NAME, VERSION, platform),
        arch = platform,
    )

    # Materialize files in the manifest to the install layout.
    wix.add_install_files(manifest)

    # From mercurial.wxs.
    wix.install_files_root_directory_id = "INSTALLDIR"

    # Pull in our custom .wxs files.
    defines = {
        "PyOxidizer": "1",
        "Platform": platform,
        "Version": VERSION,
        "Comments": "Installs Mercurial version %s" % VERSION,
        "MercurialHasLib": "1",
    }

    if EXTRA_MSI_FEATURES:
        defines["MercurialExtraFeatures"] = EXTRA_MSI_FEATURES

    wix.add_wxs_file(
        ROOT + "/contrib/packaging/wix/mercurial.wxs",
        preprocessor_parameters=defines,
    )

    # Our .wxs references to other files. Pull those into the build environment.
    for f in ("defines.wxi", "guids.wxi", "COPYING.rtf"):
        wix.add_build_file(f, ROOT + "/contrib/packaging/wix/" + f)

    wix.add_build_file("mercurial.ico", ROOT + "/contrib/win32/mercurial.ico")

    return wix


def register_code_signers():
    if not IS_WINDOWS:
        return

    if SIGNING_PFX_PATH:
        signer = code_signer_from_pfx_file(SIGNING_PFX_PATH, SIGNING_PFX_PASSWORD)
    elif SIGNING_SUBJECT_NAME:
        signer = code_signer_from_windows_store_subject(SIGNING_SUBJECT_NAME)
    else:
        signer = None

    if signer:
        signer.set_time_stamp_server(TIME_STAMP_SERVER_URL)
        signer.activate()


register_code_signers()

register_target("distribution", make_distribution)
register_target("exe", make_exe, depends = ["distribution"])
register_target("app", make_manifest, depends = ["distribution", "exe"], default = True)
register_target("msi", make_msi, depends = ["app"])

resolve_targets()