comparison setup.py @ 42453:94167e701e12

rust: new rust options in setup.py

The --rust global option turns on usage (and by default compilation)
of the rust-cpython based mercurial.rustext.

Similarly to what's previously done for zstd, there is a --no-rust
option for the build_ext subcommand in order not to build
mercurial.rustext, allowing for an OS distribution to prebuild it.

The HGWITHRUSTEXT environment variable is still honored, and has
the same effect as before, but now it works mostly by making
the --rust global option defaulting to True, with some special
cases for the direct-ffi case (see more about that below)

Coincidentally, the --rust flag can also be passed from the make
commands, like actually all global options, in the PURE variable

   make local PURE=--rust

This feels inappropriate, though, and we should follow up with
a proper make variable for that case.

Although the direct-ffi bindings aren't directly useful any more, we
keep them at this stage because

- they provide a short prototyping path for experiments in which a C extension
  module has to call into a Rust extension. The proper way of doing that would
  be to use capsules, and it's best to wait for our pull request onto
  rust-cpython for that: https://github.com/dgrunwald/rust-cpython/pull/169
- Build support for capsules defined in Rust will probably need to reuse
  some of what's currently in use for direct-ffi.

42452:a3a8887e4426

            self.make_file([pofile], mobuildfile, spawn, (cmd,))


class hgdist(Distribution):
    pure = False
    cffi = ispypy

    global_options = Distribution.global_options + [
        ('pure', None, "use pure (slow) Python code instead of C extensions"),
    ]

    def has_ext_modules(self):
        # self.ext_modules is emptied in hgbuildpy.finalize_options which is
        # too late for some cases
        return not self.pure and Distribution.has_ext_modules(self)

# This is ugly as a one-liner. So use a variable.
buildextnegops = dict(getattr(build_ext, 'negative_options', {}))
buildextnegops['no-zstd'] = 'zstd'

class hgbuildext(build_ext):
    user_options = build_ext.user_options + [
        ('zstd', None, 'compile zstd bindings [default]'),
        ('no-zstd', None, 'do not compile zstd bindings'),
    ]

    boolean_options = build_ext.boolean_options + ['zstd']
    negative_opt = buildextnegops

    def initialize_options(self):
        self.zstd = True
        return build_ext.initialize_options(self)

    def build_extensions(self):
        ruststandalones = [e for e in self.extensions
                           if isinstance(e, RustStandaloneExtension)]

        # Filter out zstd if disabled via argument.
        if not self.zstd:
            self.extensions = [e for e in self.extensions
                               if e.name != 'mercurial.zstd']

        for rustext in ruststandalones:
            rustext.build('' if self.inplace else self.build_lib)

        return build_ext.build_extensions(self)

    def build_extension(self, ext):
        if isinstance(ext, RustExtension):
            ext.rustbuild()
        try:
            build_ext.build_extension(self, ext)
        except CCompilerError:
            if not getattr(ext, 'optional', False):
                raise

    def run(self):
        basepath = os.path.join(self.build_lib, 'mercurial')
        self.mkpath(basepath)

        if self.distribution.pure:
            modulepolicy = 'py'
        elif self.build_lib == '.':
            # in-place build should run without rebuilding C
            # and Rust extensions
            if hgrustext == 'cpython':
                modulepolicy = 'rust+c-allow'
            else:
                modulepolicy = 'allow'
        else:
            if hgrustext == 'cpython':
                modulepolicy = 'rust+c'
            else:
                modulepolicy = 'c'

        content = b''.join([
            b'# this file is autogenerated by setup.py\n',
            b'modulepolicy = b"%s"\n' % modulepolicy.encode('ascii'),
        ])

    rusttargetdir = os.path.join('rust', 'target', 'release')

    def __init__(self, mpath, sources, rustlibname, subcrate,
                 py3_features=None, **kw):
        Extension.__init__(self, mpath, sources, **kw)
        if hgrustext is None:
            return
        srcdir = self.rustsrcdir = os.path.join('rust', subcrate)
        self.py3_features = py3_features

        # adding Rust source and control files to depends so that the extension
        # gets rebuilt if they've changed

            self.depends.extend(os.path.join(dirpath, fname)
                                for fname in fnames
                                if os.path.splitext(fname)[1] == '.rs')

    def rustbuild(self):
        if hgrustext is None:
            return
        env = os.environ.copy()
        if 'HGTEST_RESTOREENV' in env:
            # Mercurial tests change HOME to a temporary directory,
            # but, if installed with rustup, the Rust toolchain needs
            # HOME to be correct (otherwise the 'no default toolchain'

        if hgrustext != 'direct-ffi':
            return
        self.extra_compile_args.append('-DWITH_RUST')
        self.libraries.append(rustlibname)
        self.library_dirs.append(self.rusttargetdir)

class RustStandaloneExtension(RustExtension):

    def __init__(self, pydottedname, rustcrate, dylibname, **kw):
        RustExtension.__init__(self, pydottedname, [], dylibname, rustcrate,

        'mercurial.thirdparty.zope.interface._zope_interface_coptimizations', [
        'mercurial/thirdparty/zope/interface/_zope_interface_coptimizations.c',
        ]),
    Extension('hgext.fsmonitor.pywatchman.bser',
              ['hgext/fsmonitor/pywatchman/bser.c']),
    ]

if hgrustext == 'cpython':
    extmodules.append(
        RustStandaloneExtension('mercurial.rustext', 'hg-cpython', 'librusthg',
                                py3_features='python3')
    )


sys.path.insert(0, 'contrib/python-zstandard')
import setup_zstd
extmodules.append(setup_zstd.get_c_extension(

42453:94167e701e12

            self.make_file([pofile], mobuildfile, spawn, (cmd,))


class hgdist(Distribution):
    pure = False
    rust = hgrustext is not None
    cffi = ispypy

    global_options = Distribution.global_options + [
        ('pure', None, "use pure (slow) Python code instead of C extensions"),
        ('rust', None, "use Rust extensions additionally to C extensions"),
    ]

    def has_ext_modules(self):
        # self.ext_modules is emptied in hgbuildpy.finalize_options which is
        # too late for some cases
        return not self.pure and Distribution.has_ext_modules(self)

# This is ugly as a one-liner. So use a variable.
buildextnegops = dict(getattr(build_ext, 'negative_options', {}))
buildextnegops['no-zstd'] = 'zstd'
buildextnegops['no-rust'] = 'rust'

class hgbuildext(build_ext):
    user_options = build_ext.user_options + [
        ('zstd', None, 'compile zstd bindings [default]'),
        ('no-zstd', None, 'do not compile zstd bindings'),
        ('rust', None,
         'compile Rust extensions if they are in use '
         '(requires Cargo) [default]'),
        ('no-rust', None, 'do not compile Rust extensions'),
    ]

    boolean_options = build_ext.boolean_options + ['zstd', 'rust']
    negative_opt = buildextnegops

    def initialize_options(self):
        self.zstd = True
        self.rust = True

        return build_ext.initialize_options(self)

    def build_extensions(self):
        ruststandalones = [e for e in self.extensions
                           if isinstance(e, RustStandaloneExtension)]

        # Filter out zstd if disabled via argument.
        if not self.zstd:
            self.extensions = [e for e in self.extensions
                               if e.name != 'mercurial.zstd']

        # Build Rust standalon extensions if it'll be used
        # and its build is not explictely disabled (for external build
        # as Linux distributions would do)
        if self.distribution.rust and self.rust and hgrustext != 'direct-ffi':
            for rustext in ruststandalones:
                rustext.build('' if self.inplace else self.build_lib)

        return build_ext.build_extensions(self)

    def build_extension(self, ext):
        if (self.distribution.rust and self.rust
            and isinstance(ext, RustExtension)):
                ext.rustbuild()
        try:
            build_ext.build_extension(self, ext)
        except CCompilerError:
            if not getattr(ext, 'optional', False):
                raise

    def run(self):
        basepath = os.path.join(self.build_lib, 'mercurial')
        self.mkpath(basepath)

        rust = self.distribution.rust
        if self.distribution.pure:
            modulepolicy = 'py'
        elif self.build_lib == '.':
            # in-place build should run without rebuilding and Rust extensions
            modulepolicy = 'rust+c-allow' if rust else 'allow'
        else:
            modulepolicy = 'rust+c' if rust else 'c'

        content = b''.join([
            b'# this file is autogenerated by setup.py\n',
            b'modulepolicy = b"%s"\n' % modulepolicy.encode('ascii'),
        ])

    rusttargetdir = os.path.join('rust', 'target', 'release')

    def __init__(self, mpath, sources, rustlibname, subcrate,
                 py3_features=None, **kw):
        Extension.__init__(self, mpath, sources, **kw)
        srcdir = self.rustsrcdir = os.path.join('rust', subcrate)
        self.py3_features = py3_features

        # adding Rust source and control files to depends so that the extension
        # gets rebuilt if they've changed

            self.depends.extend(os.path.join(dirpath, fname)
                                for fname in fnames
                                if os.path.splitext(fname)[1] == '.rs')

    def rustbuild(self):
        env = os.environ.copy()
        if 'HGTEST_RESTOREENV' in env:
            # Mercurial tests change HOME to a temporary directory,
            # but, if installed with rustup, the Rust toolchain needs
            # HOME to be correct (otherwise the 'no default toolchain'

        if hgrustext != 'direct-ffi':
            return
        self.extra_compile_args.append('-DWITH_RUST')
        self.libraries.append(rustlibname)
        self.library_dirs.append(self.rusttargetdir)

    def rustbuild(self):
        if hgrustext == 'direct-ffi':
            RustExtension.rustbuild(self)

class RustStandaloneExtension(RustExtension):

    def __init__(self, pydottedname, rustcrate, dylibname, **kw):
        RustExtension.__init__(self, pydottedname, [], dylibname, rustcrate,

        'mercurial.thirdparty.zope.interface._zope_interface_coptimizations', [
        'mercurial/thirdparty/zope/interface/_zope_interface_coptimizations.c',
        ]),
    Extension('hgext.fsmonitor.pywatchman.bser',
              ['hgext/fsmonitor/pywatchman/bser.c']),
    RustStandaloneExtension('mercurial.rustext', 'hg-cpython', 'librusthg',
                            py3_features='python3'),
    ]


sys.path.insert(0, 'contrib/python-zstandard')
import setup_zstd
extmodules.append(setup_zstd.get_c_extension(