Mercurial > hg

changeset 44305:d8d4fa9a7f18

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
rust-re2: add wrapper for calling Re2 from Rust This assumes that Re2 is installed following Google's guide. I am not sure how we want to integrate it in the project, but I think a follow-up patch would be more appropriate for such work. As it stands, *not* having Re2 installed results in a compilation error, which is a problem as it breaks install compatibility. Hence, this is gated behind a non-default `with-re2` compilation feature. Differential Revision: https://phab.mercurial-scm.org/D7910
author Raphaël Gomès <rgomes@octobus.net>
date Thu, 16 Jan 2020 13:34:04 +0100
parents 2fe89bec8011
children a0ec05d93c8e
files rust/Cargo.lock rust/hg-core/Cargo.toml rust/hg-core/build.rs rust/hg-core/src/lib.rs rust/hg-core/src/re2/mod.rs rust/hg-core/src/re2/re2.rs rust/hg-core/src/re2/rust_re2.cpp rust/hg-cpython/Cargo.toml setup.py
diffstat 9 files changed, 195 insertions(+), 5 deletions(-) [+]
line wrap: on
line diff
--- a/rust/Cargo.lock	Thu Jan 16 11:27:12 2020 +0100
+++ b/rust/Cargo.lock	Thu Jan 16 13:34:04 2020 +0100
@@ -45,6 +45,11 @@
 ]
 
 [[package]]
+name = "cc"
+version = "1.0.50"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
 name = "cfg-if"
 version = "0.1.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -161,8 +166,10 @@
 version = "0.1.0"
 dependencies = [
  "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "cc 1.0.50 (registry+https://github.com/rust-lang/crates.io-index)",
  "hex 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
  "memchr 2.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "pretty_assertions 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -599,6 +606,7 @@
 "checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
 "checksum byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a7c3dd8985a7111efc5c80b44e23ecdd8c007de8ade3b96595387e812b957cf5"
 "checksum c2-chacha 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "214238caa1bf3a496ec3392968969cab8549f96ff30652c9e56885329315f6bb"
+"checksum cc 1.0.50 (registry+https://github.com/rust-lang/crates.io-index)" = "95e28fa049fda1c330bcf9d723be7663a899c4679724b34c81e9f5a326aab8cd"
 "checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
 "checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f"
 "checksum cpython 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "bfaf3847ab963e40c4f6dd8d6be279bdf74007ae2413786a0dcbb28c52139a95"
--- a/rust/hg-core/Cargo.toml	Thu Jan 16 11:27:12 2020 +0100
+++ b/rust/hg-core/Cargo.toml	Thu Jan 16 13:34:04 2020 +0100
@@ -4,6 +4,7 @@
 authors = ["Georges Racinet <gracinet@anybox.fr>"]
 description = "Mercurial pure Rust core library, with no assumption on Python bindings (FFI)"
 edition = "2018"
+build = "build.rs"
 
 [lib]
 name = "hg"
@@ -12,6 +13,7 @@
 byteorder = "1.3.1"
 hex = "0.4.0"
 lazy_static = "1.3.0"
+libc = { version = "0.2.66", optional = true }
 memchr = "2.2.0"
 rand = "0.6.5"
 rand_pcg = "0.1.1"
@@ -22,4 +24,11 @@
 
 [dev-dependencies]
 tempfile = "3.1.0"
-pretty_assertions = "0.6.1"
\ No newline at end of file
+pretty_assertions = "0.6.1"
+
+[build-dependencies]
+cc = { version = "1.0.48", optional = true }
+
+[features]
+default = []
+with-re2 = ["cc", "libc"]
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/rust/hg-core/build.rs	Thu Jan 16 13:34:04 2020 +0100
@@ -0,0 +1,25 @@
+// build.rs
+//
+// Copyright 2020 Raphaël Gomès <rgomes@octobus.net>
+//
+// This software may be used and distributed according to the terms of the
+// GNU General Public License version 2 or any later version.
+
+#[cfg(feature = "with-re2")]
+use cc;
+
+#[cfg(feature = "with-re2")]
+fn compile_re2() {
+    cc::Build::new()
+        .cpp(true)
+        .flag("-std=c++11")
+        .file("src/re2/rust_re2.cpp")
+        .compile("librustre.a");
+
+    println!("cargo:rustc-link-lib=re2");
+}
+
+fn main() {
+    #[cfg(feature = "with-re2")]
+    compile_re2();
+}
--- a/rust/hg-core/src/lib.rs	Thu Jan 16 11:27:12 2020 +0100
+++ b/rust/hg-core/src/lib.rs	Thu Jan 16 13:34:04 2020 +0100
@@ -21,6 +21,8 @@
 pub mod matchers;
 pub mod revlog;
 pub use revlog::*;
+#[cfg(feature = "with-re2")]
+pub mod re2;
 pub mod utils;
 
 use crate::utils::hg_path::{HgPathBuf, HgPathError};
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/rust/hg-core/src/re2/mod.rs	Thu Jan 16 13:34:04 2020 +0100
@@ -0,0 +1,21 @@
+/// re2 module
+///
+/// The Python implementation of Mercurial uses the Re2 regex engine when
+/// possible and if the bindings are installed, falling back to Python's `re`
+/// in case of unsupported syntax (Re2 is a non-backtracking engine).
+///
+/// Using it from Rust is not ideal. We need C++ bindings, a C++ compiler,
+/// Re2 needs to be installed... why not just use the `regex` crate?
+///
+/// Using Re2 from the Rust implementation guarantees backwards compatibility.
+/// We know it will work out of the box without needing to figure out the
+/// subtle differences in syntax. For example, `regex` currently does not
+/// support empty alternations (regex like `a||b`) which happens more often
+/// than we might think. Old benchmarks also showed worse performance from
+/// regex than with Re2, but the methodology and results were lost, so take
+/// this with a grain of salt.
+///
+/// The idea is to use Re2 for now as a temporary phase and then investigate
+/// how much work would be needed to use `regex`.
+mod re2;
+pub use re2::Re2;
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/rust/hg-core/src/re2/re2.rs	Thu Jan 16 13:34:04 2020 +0100
@@ -0,0 +1,66 @@
+/*
+re2.rs
+
+Rust FFI bindings to Re2.
+
+Copyright 2020 Valentin Gatien-Baron
+
+This software may be used and distributed according to the terms of the
+GNU General Public License version 2 or any later version.
+*/
+use libc::{c_int, c_void};
+
+type Re2Ptr = *const c_void;
+
+pub struct Re2(Re2Ptr);
+
+/// `re2.h` says:
+/// "An "RE2" object is safe for concurrent use by multiple threads."
+unsafe impl Sync for Re2 {}
+
+/// These bind to the C ABI in `rust_re2.cpp`.
+extern "C" {
+    fn rust_re2_create(data: *const u8, len: usize) -> Re2Ptr;
+    fn rust_re2_destroy(re2: Re2Ptr);
+    fn rust_re2_ok(re2: Re2Ptr) -> bool;
+    fn rust_re2_error(
+        re2: Re2Ptr,
+        outdata: *mut *const u8,
+        outlen: *mut usize,
+    ) -> bool;
+    fn rust_re2_match(
+        re2: Re2Ptr,
+        data: *const u8,
+        len: usize,
+        anchor: c_int,
+    ) -> bool;
+}
+
+impl Re2 {
+    pub fn new(pattern: &[u8]) -> Result<Re2, String> {
+        unsafe {
+            let re2 = rust_re2_create(pattern.as_ptr(), pattern.len());
+            if rust_re2_ok(re2) {
+                Ok(Re2(re2))
+            } else {
+                let mut data: *const u8 = std::ptr::null();
+                let mut len: usize = 0;
+                rust_re2_error(re2, &mut data, &mut len);
+                Err(String::from_utf8_lossy(std::slice::from_raw_parts(
+                    data, len,
+                ))
+                .to_string())
+            }
+        }
+    }
+
+    pub fn is_match(&self, data: &[u8]) -> bool {
+        unsafe { rust_re2_match(self.0, data.as_ptr(), data.len(), 1) }
+    }
+}
+
+impl Drop for Re2 {
+    fn drop(&mut self) {
+        unsafe { rust_re2_destroy(self.0) }
+    }
+}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/rust/hg-core/src/re2/rust_re2.cpp	Thu Jan 16 13:34:04 2020 +0100
@@ -0,0 +1,49 @@
+/*
+rust_re2.cpp
+
+C ABI export of Re2's C++ interface for Rust FFI.
+
+Copyright 2020 Valentin Gatien-Baron
+
+This software may be used and distributed according to the terms of the
+GNU General Public License version 2 or any later version.
+*/
+
+#include <re2/re2.h>
+using namespace re2;
+
+extern "C" {
+	RE2* rust_re2_create(const char* data, size_t len) {
+		RE2::Options o;
+		o.set_encoding(RE2::Options::Encoding::EncodingLatin1);
+		o.set_log_errors(false);
+		o.set_max_mem(50000000);
+
+		return new RE2(StringPiece(data, len), o);
+	}
+
+	void rust_re2_destroy(RE2* re) {
+		delete re;
+	}
+
+	bool rust_re2_ok(RE2* re) {
+		return re->ok();
+	}
+
+	void rust_re2_error(RE2* re, const char** outdata, size_t* outlen) {
+		const std::string& e = re->error();
+		*outdata = e.data();
+		*outlen = e.length();
+	}
+
+	bool rust_re2_match(RE2* re, char* data, size_t len, int ianchor) {
+		const StringPiece sp = StringPiece(data, len);
+
+		RE2::Anchor anchor =
+			ianchor == 0 ? RE2::Anchor::UNANCHORED :
+			(ianchor == 1 ? RE2::Anchor::ANCHOR_START :
+			 RE2::Anchor::ANCHOR_BOTH);
+
+		return re->Match(sp, 0, len, anchor, NULL, 0);
+	}
+}
--- a/rust/hg-cpython/Cargo.toml	Thu Jan 16 11:27:12 2020 +0100
+++ b/rust/hg-cpython/Cargo.toml	Thu Jan 16 13:34:04 2020 +0100
@@ -10,6 +10,7 @@
 
 [features]
 default = ["python27"]
+with-re2 = ["hg-core/with-re2"]
 
 # Features to build an extension module:
 python27 = ["cpython/python27-sys", "cpython/extension-module-2-7"]
@@ -21,7 +22,7 @@
 python3-bin = ["cpython/python3-sys"]
 
 [dependencies]
-hg-core = { path = "../hg-core" }
+hg-core = { path = "../hg-core"}
 libc = '*'
 
 [dependencies.cpython]
--- a/setup.py	Thu Jan 16 11:27:12 2020 +0100
+++ b/setup.py	Thu Jan 16 13:34:04 2020 +0100
@@ -1351,10 +1351,19 @@
             env['HOME'] = pwd.getpwuid(os.getuid()).pw_dir
 
         cargocmd = ['cargo', 'rustc', '-vv', '--release']
+
+        feature_flags = []
+
         if sys.version_info[0] == 3 and self.py3_features is not None:
-            cargocmd.extend(
-                ('--features', self.py3_features, '--no-default-features')
-            )
+            feature_flags.append(self.py3_features)
+            cargocmd.append('--no-default-features')
+
+        rust_features = env.get("HG_RUST_FEATURES")
+        if rust_features:
+            feature_flags.append(rust_features)
+
+        cargocmd.extend(('--features', " ".join(feature_flags)))
+
         cargocmd.append('--')
         if sys.platform == 'darwin':
             cargocmd.extend(