rust: new rust options in setup.py

rust: new rust options in setup.py

The --rust global option turns on usage (and by default compilation)
of the rust-cpython based mercurial.rustext.

Similarly to what's previously done for zstd, there is a --no-rust
option for the build_ext subcommand in order not to build
mercurial.rustext, allowing for an OS distribution to prebuild it.

The HGWITHRUSTEXT environment variable is still honored, and has
the same effect as before, but now it works mostly by making
the --rust global option defaulting to True, with some special
cases for the direct-ffi case (see more about that below)

Coincidentally, the --rust flag can also be passed from the make
commands, like actually all global options, in the PURE variable

   make local PURE=--rust

This feels inappropriate, though, and we should follow up with
a proper make variable for that case.

Although the direct-ffi bindings aren't directly useful any more, we
keep them at this stage because

- they provide a short prototyping path for experiments in which a C extension
  module has to call into a Rust extension. The proper way of doing that would
  be to use capsules, and it's best to wait for our pull request onto
  rust-cpython for that: https://github.com/dgrunwald/rust-cpython/pull/169
- Build support for capsules defined in Rust will probably need to reuse
  some of what's currently in use for direct-ffi.

--- a/setup.py	Thu May 30 09:14:41 2019 +0200
+++ b/setup.py	Thu May 23 02:05:32 2019 +0200
@@ -446,10 +446,12 @@
 
 class hgdist(Distribution):
     pure = False
+    rust = hgrustext is not None
     cffi = ispypy
 
     global_options = Distribution.global_options + [
         ('pure', None, "use pure (slow) Python code instead of C extensions"),
+        ('rust', None, "use Rust extensions additionally to C extensions"),
     ]
 
     def has_ext_modules(self):
@@ -460,18 +462,25 @@
 # This is ugly as a one-liner. So use a variable.
 buildextnegops = dict(getattr(build_ext, 'negative_options', {}))
 buildextnegops['no-zstd'] = 'zstd'
+buildextnegops['no-rust'] = 'rust'
 
 class hgbuildext(build_ext):
     user_options = build_ext.user_options + [
         ('zstd', None, 'compile zstd bindings [default]'),
         ('no-zstd', None, 'do not compile zstd bindings'),
+        ('rust', None,
+         'compile Rust extensions if they are in use '
+         '(requires Cargo) [default]'),
+        ('no-rust', None, 'do not compile Rust extensions'),
     ]
 
-    boolean_options = build_ext.boolean_options + ['zstd']
+    boolean_options = build_ext.boolean_options + ['zstd', 'rust']
     negative_opt = buildextnegops
 
     def initialize_options(self):
         self.zstd = True
+        self.rust = True
+
         return build_ext.initialize_options(self)
 
     def build_extensions(self):
@@ -484,14 +493,19 @@
             self.extensions = [e for e in self.extensions
                                if e.name != 'mercurial.zstd']
 
-        for rustext in ruststandalones:
-            rustext.build('' if self.inplace else self.build_lib)
+        # Build Rust standalon extensions if it'll be used
+        # and its build is not explictely disabled (for external build
+        # as Linux distributions would do)
+        if self.distribution.rust and self.rust and hgrustext != 'direct-ffi':
+            for rustext in ruststandalones:
+                rustext.build('' if self.inplace else self.build_lib)
 
         return build_ext.build_extensions(self)
 
     def build_extension(self, ext):
-        if isinstance(ext, RustExtension):
-            ext.rustbuild()
+        if (self.distribution.rust and self.rust
+            and isinstance(ext, RustExtension)):
+                ext.rustbuild()
         try:
             build_ext.build_extension(self, ext)
         except CCompilerError:
@@ -553,20 +567,14 @@
         basepath = os.path.join(self.build_lib, 'mercurial')
         self.mkpath(basepath)
 
+        rust = self.distribution.rust
         if self.distribution.pure:
             modulepolicy = 'py'
         elif self.build_lib == '.':
-            # in-place build should run without rebuilding C
-            # and Rust extensions
-            if hgrustext == 'cpython':
-                modulepolicy = 'rust+c-allow'
-            else:
-                modulepolicy = 'allow'
+            # in-place build should run without rebuilding and Rust extensions
+            modulepolicy = 'rust+c-allow' if rust else 'allow'
         else:
-            if hgrustext == 'cpython':
-                modulepolicy = 'rust+c'
-            else:
-                modulepolicy = 'c'
+            modulepolicy = 'rust+c' if rust else 'c'
 
         content = b''.join([
             b'# this file is autogenerated by setup.py\n',
@@ -1138,8 +1146,6 @@
     def __init__(self, mpath, sources, rustlibname, subcrate,
                  py3_features=None, **kw):
         Extension.__init__(self, mpath, sources, **kw)
-        if hgrustext is None:
-            return
         srcdir = self.rustsrcdir = os.path.join('rust', subcrate)
         self.py3_features = py3_features
 
@@ -1155,8 +1161,6 @@
                                 if os.path.splitext(fname)[1] == '.rs')
 
     def rustbuild(self):
-        if hgrustext is None:
-            return
         env = os.environ.copy()
         if 'HGTEST_RESTOREENV' in env:
             # Mercurial tests change HOME to a temporary directory,
@@ -1208,6 +1212,10 @@
         self.libraries.append(rustlibname)
         self.library_dirs.append(self.rusttargetdir)
 
+    def rustbuild(self):
+        if hgrustext == 'direct-ffi':
+            RustExtension.rustbuild(self)
+
 class RustStandaloneExtension(RustExtension):
 
     def __init__(self, pydottedname, rustcrate, dylibname, **kw):
@@ -1262,14 +1270,10 @@
         ]),
     Extension('hgext.fsmonitor.pywatchman.bser',
               ['hgext/fsmonitor/pywatchman/bser.c']),
+    RustStandaloneExtension('mercurial.rustext', 'hg-cpython', 'librusthg',
+                            py3_features='python3'),
     ]
 
-if hgrustext == 'cpython':
-    extmodules.append(
-        RustStandaloneExtension('mercurial.rustext', 'hg-cpython', 'librusthg',
-                                py3_features='python3')
-    )
-
 
 sys.path.insert(0, 'contrib/python-zstandard')
 import setup_zstd