Mercurial > hg
changeset 40273:3b275f549777
rust: exposing in parsers module
To build with the Rust code, set the HGWITHRUSTEXT
environment variable.
At this point, it's possible to instantiate and use
a rustlazyancestors object from a Python interpreter.
The changes in setup.py are obviously a quick hack,
just good enough to test/bench without much
refactoring. We'd be happy to improve on that with
help from the community.
Rust bindings crate gets compiled as a static library,
which in turn gets linked within 'parsers.so'
With respect to the plans at
https://www.mercurial-scm.org/wiki/OxidationPlan
this would probably qualify as "roll our own FFI".
Also, it doesn't quite meet the target of getting
rid of C code, since it brings actually more, yet:
- the new C code does nothing else than parsing
arguments and calling Rust functions.
In particular, there's no complex allocation involved.
- subsequent changes could rewrite more of revlog.c, this
time resulting in an overall decrease of C code and
unsafety.
| author | Georges Racinet <gracinet@anybox.fr> |
|---|---|
| date | Thu, 27 Sep 2018 16:56:15 +0200 |
| parents | a36c5e23c055 |
| children | a8b9174517c7 |
| files | mercurial/cext/revlog.c setup.py |
| diffstat | 2 files changed, 221 insertions(+), 8 deletions(-) [+] |
line wrap: on
line diff
--- a/mercurial/cext/revlog.c Thu Sep 27 16:51:36 2018 +0200 +++ b/mercurial/cext/revlog.c Thu Sep 27 16:56:15 2018 +0200 @@ -2290,6 +2290,152 @@ return NULL; } +#ifdef WITH_RUST + +/* rustlazyancestors: iteration over ancestors implemented in Rust + * + * This class holds a reference to an index and to the Rust iterator. + */ +typedef struct rustlazyancestorsObjectStruct rustlazyancestorsObject; + +struct rustlazyancestorsObjectStruct { + PyObject_HEAD + /* Type-specific fields go here. */ + indexObject *index; /* Ref kept to avoid GC'ing the index */ + void *iter; /* Rust iterator */ +}; + +/* FFI exposed from Rust code */ +rustlazyancestorsObject *rustlazyancestors_init( + indexObject *index, + /* to pass index_get_parents() */ + int (*)(indexObject *, Py_ssize_t, int*, int), + /* intrevs vector */ + int initrevslen, long *initrevs, + long stoprev, + int inclusive); +void rustlazyancestors_drop(rustlazyancestorsObject *self); +int rustlazyancestors_next(rustlazyancestorsObject *self); + +/* CPython instance methods */ +static int rustla_init(rustlazyancestorsObject *self, + PyObject *args) { + PyObject *initrevsarg = NULL; + PyObject *inclusivearg = NULL; + long stoprev = 0; + long *initrevs = NULL; + int inclusive = 0; + Py_ssize_t i; + + indexObject *index; + if (!PyArg_ParseTuple(args, "O!O!lO!", + &indexType, &index, + &PyList_Type, &initrevsarg, + &stoprev, + &PyBool_Type, &inclusivearg)) + return -1; + + Py_INCREF(index); + self->index = index; + + if (inclusivearg == Py_True) + inclusive = 1; + + Py_ssize_t linit = PyList_GET_SIZE(initrevsarg); + + initrevs = (long*)calloc(linit, sizeof(long)); + + if (initrevs == NULL) { + PyErr_NoMemory(); + goto bail; + } + + for (i=0; i<linit; i++) { + initrevs[i] = PyInt_AsLong(PyList_GET_ITEM(initrevsarg, i)); + } + if (PyErr_Occurred()) + goto bail; + + self->iter = rustlazyancestors_init(index, + index_get_parents, + linit, initrevs, + stoprev, inclusive); + if (self->iter == NULL) { + /* if this is because of GraphError::ParentOutOfRange + * index_get_parents() has already set the proper ValueError */ + goto bail; + } + + free(initrevs); + return 0; + +bail: + free(initrevs); + return -1; +}; + +static void rustla_dealloc(rustlazyancestorsObject *self) +{ + Py_XDECREF(self->index); + if (self->iter != NULL) { /* can happen if rustla_init failed */ + rustlazyancestors_drop(self->iter); + } + PyObject_Del(self); +} + +static PyObject *rustla_next(rustlazyancestorsObject *self) { + int res = rustlazyancestors_next(self->iter); + if (res == -1) { + /* Setting an explicit exception seems unnecessary + * as examples from Python source code (Objects/rangeobjets.c and + * Modules/_io/stringio.c) seem to demonstrate. + */ + return NULL; + } + return PyInt_FromLong(res); +} + +static PyTypeObject rustlazyancestorsType = { + PyVarObject_HEAD_INIT(NULL, 0) /* header */ + "parsers.rustlazyancestors", /* tp_name */ + sizeof(rustlazyancestorsObject), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)rustla_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + "Iterator over ancestors, implemented in Rust", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + (iternextfunc)rustla_next, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)rustla_init, /* tp_init */ + 0, /* tp_alloc */ +}; +#endif /* WITH_RUST */ + void revlog_module_init(PyObject *mod) { indexType.tp_new = PyType_GenericNew; @@ -2310,4 +2456,14 @@ } if (nullentry) PyObject_GC_UnTrack(nullentry); + +#ifdef WITH_RUST + rustlazyancestorsType.tp_new = PyType_GenericNew; + if (PyType_Ready(&rustlazyancestorsType) < 0) + return; + Py_INCREF(&rustlazyancestorsType); + PyModule_AddObject(mod, "rustlazyancestors", + (PyObject *)&rustlazyancestorsType); +#endif + }
--- a/setup.py Thu Sep 27 16:51:36 2018 +0200 +++ b/setup.py Thu Sep 27 16:56:15 2018 +0200 @@ -132,6 +132,8 @@ ispypy = "PyPy" in sys.version +iswithrustextensions = 'HGWITHRUSTEXT' in os.environ + import ctypes import stat, subprocess, time import re @@ -460,6 +462,8 @@ return build_ext.build_extensions(self) def build_extension(self, ext): + if isinstance(ext, RustExtension): + ext.rustbuild() try: build_ext.build_extension(self, ext) except CCompilerError: @@ -884,6 +888,54 @@ 'mercurial/thirdparty/xdiff/xutils.h', ] +class RustExtension(Extension): + """A C Extension, conditionnally enhanced with Rust code. + + if iswithrustextensions is False, does nothing else than plain Extension + """ + + rusttargetdir = os.path.join('rust', 'target', 'release') + + def __init__(self, mpath, sources, rustlibname, subcrate, **kw): + Extension.__init__(self, mpath, sources, **kw) + if not iswithrustextensions: + return + srcdir = self.rustsrcdir = os.path.join('rust', subcrate) + self.libraries.append(rustlibname) + self.extra_compile_args.append('-DWITH_RUST') + + # adding Rust source and control files to depends so that the extension + # gets rebuilt if they've changed + self.depends.append(os.path.join(srcdir, 'Cargo.toml')) + cargo_lock = os.path.join(srcdir, 'Cargo.lock') + if os.path.exists(cargo_lock): + self.depends.append(cargo_lock) + for dirpath, subdir, fnames in os.walk(os.path.join(srcdir, 'src')): + self.depends.extend(os.path.join(dirpath, fname) + for fname in fnames + if os.path.splitext(fname)[1] == '.rs') + + def rustbuild(self): + if not iswithrustextensions: + return + env = os.environ.copy() + if 'HGTEST_RESTOREENV' in env: + # Mercurial tests change HOME to a temporary directory, + # but, if installed with rustup, the Rust toolchain needs + # HOME to be correct (otherwise the 'no default toolchain' + # error message is issued and the build fails). + # This happens currently with test-hghave.t, which does + # invoke this build. + + # Unix only fix (os.path.expanduser not really reliable if + # HOME is shadowed like this) + import pwd + env['HOME'] = pwd.getpwuid(os.getuid()).pw_dir + + subprocess.check_call(['cargo', 'build', '-vv', '--release'], + env=env, cwd=self.rustsrcdir) + self.library_dirs.append(self.rusttargetdir) + extmodules = [ Extension('mercurial.cext.base85', ['mercurial/cext/base85.c'], include_dirs=common_include_dirs, @@ -896,14 +948,19 @@ 'mercurial/cext/mpatch.c'], include_dirs=common_include_dirs, depends=common_depends), - Extension('mercurial.cext.parsers', ['mercurial/cext/charencode.c', - 'mercurial/cext/dirs.c', - 'mercurial/cext/manifest.c', - 'mercurial/cext/parsers.c', - 'mercurial/cext/pathencode.c', - 'mercurial/cext/revlog.c'], - include_dirs=common_include_dirs, - depends=common_depends + ['mercurial/cext/charencode.h']), + RustExtension('mercurial.cext.parsers', ['mercurial/cext/charencode.c', + 'mercurial/cext/dirs.c', + 'mercurial/cext/manifest.c', + 'mercurial/cext/parsers.c', + 'mercurial/cext/pathencode.c', + 'mercurial/cext/revlog.c'], + 'hgdirectffi', + 'hg-direct-ffi', + include_dirs=common_include_dirs, + depends=common_depends + ['mercurial/cext/charencode.h', + 'mercurial/rust/src/lib.rs', + 'mercurial/rust/src/ancestors.rs', + 'mercurial/rust/src/cpython.rs']), Extension('mercurial.cext.osutil', ['mercurial/cext/osutil.c'], include_dirs=common_include_dirs, extra_compile_args=osutil_cflags,