changeset 51391:a96ed440450e

hg-core: implement timestamp line parsing
author Arun Kulshreshtha <akulshreshtha@janestreet.com>
date Thu, 15 Feb 2024 11:39:18 -0500
parents 92c7765931e0
children 6603a1448f18
files rust/Cargo.lock rust/hg-core/Cargo.toml rust/hg-core/src/revlog/changelog.rs
diffstat 3 files changed, 442 insertions(+), 37 deletions(-) [+]
line wrap: on
line diff
--- a/rust/Cargo.lock	Wed Feb 14 15:21:44 2024 -0500
+++ b/rust/Cargo.lock	Thu Feb 15 11:39:18 2024 -0500
@@ -29,6 +29,12 @@
 ]
 
 [[package]]
+name = "android-tzdata"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0"
+
+[[package]]
 name = "android_system_properties"
 version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -148,17 +154,16 @@
 
 [[package]]
 name = "chrono"
-version = "0.4.23"
+version = "0.4.34"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f"
+checksum = "5bc015644b92d5890fab7489e49d21f879d5c990186827d42ec511919404f38b"
 dependencies = [
+ "android-tzdata",
  "iana-time-zone",
  "js-sys",
- "num-integer",
  "num-traits",
- "time",
  "wasm-bindgen",
- "winapi",
+ "windows-targets",
 ]
 
 [[package]]
@@ -537,6 +542,7 @@
  "bitvec",
  "byteorder",
  "bytes-cast",
+ "chrono",
  "clap",
  "crossbeam-channel",
  "derive_more",
@@ -795,16 +801,6 @@
 ]
 
 [[package]]
-name = "num-integer"
-version = "0.1.45"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9"
-dependencies = [
- "autocfg",
- "num-traits",
-]
-
-[[package]]
 name = "num-traits"
 version = "0.2.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1289,17 +1285,6 @@
 ]
 
 [[package]]
-name = "time"
-version = "0.1.44"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255"
-dependencies = [
- "libc",
- "wasi 0.10.0+wasi-snapshot-preview1",
- "winapi",
-]
-
-[[package]]
 name = "toml"
 version = "0.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1393,12 +1378,6 @@
 
 [[package]]
 name = "wasi"
-version = "0.10.0+wasi-snapshot-preview1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f"
-
-[[package]]
-name = "wasi"
 version = "0.11.0+wasi-snapshot-preview1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
@@ -1520,6 +1499,63 @@
 checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
 
 [[package]]
+name = "windows-targets"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd"
+dependencies = [
+ "windows_aarch64_gnullvm",
+ "windows_aarch64_msvc",
+ "windows_i686_gnu",
+ "windows_i686_msvc",
+ "windows_x86_64_gnu",
+ "windows_x86_64_gnullvm",
+ "windows_x86_64_msvc",
+]
+
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea"
+
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef"
+
+[[package]]
+name = "windows_i686_gnu"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313"
+
+[[package]]
+name = "windows_i686_msvc"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a"
+
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd"
+
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e"
+
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04"
+
+[[package]]
 name = "wyz"
 version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
--- a/rust/hg-core/Cargo.toml	Wed Feb 14 15:21:44 2024 -0500
+++ b/rust/hg-core/Cargo.toml	Thu Feb 15 11:39:18 2024 -0500
@@ -40,6 +40,7 @@
 format-bytes = "0.3.0"
 once_cell = "1.16.0"
 bitvec = "1.0.1"
+chrono = "0.4.34"
 
 # We don't use the `miniz-oxide` backend to not change rhg benchmarks and until
 # we have a clearer view of which backend is the fastest.
--- a/rust/hg-core/src/revlog/changelog.rs	Wed Feb 14 15:21:44 2024 -0500
+++ b/rust/hg-core/src/revlog/changelog.rs	Thu Feb 15 11:39:18 2024 -0500
@@ -1,3 +1,12 @@
+use std::ascii::escape_default;
+use std::borrow::Cow;
+use std::collections::BTreeMap;
+use std::fmt::{Debug, Formatter};
+use std::{iter, str};
+
+use chrono::{DateTime, FixedOffset, NaiveDateTime};
+use itertools::{Either, Itertools};
+
 use crate::errors::HgError;
 use crate::revlog::Revision;
 use crate::revlog::{Node, NodePrefix};
@@ -5,11 +14,6 @@
 use crate::utils::hg_path::HgPath;
 use crate::vfs::Vfs;
 use crate::{Graph, GraphError, RevlogOpenOptions, UncheckedRevision};
-use itertools::{Either, Itertools};
-use std::ascii::escape_default;
-use std::borrow::Cow;
-use std::fmt::{Debug, Formatter};
-use std::iter;
 
 /// A specialized `Revlog` to work with changelog data format.
 pub struct Changelog {
@@ -229,6 +233,11 @@
         &self.bytes[self.user_end + 1..self.timestamp_end]
     }
 
+    /// Parsed timestamp line, including optional extras.
+    pub fn parsed_timestamp(&self) -> Result<TimestampAndExtra, HgError> {
+        TimestampAndExtra::from_bytes(self.timestamp_line())
+    }
+
     /// The files changed in this revision.
     pub fn files(&self) -> impl Iterator<Item = &HgPath> {
         if self.timestamp_end == self.files_end {
@@ -286,6 +295,203 @@
     .to_string()
 }
 
+/// Parsed timestamp line, including the timestamp and optional extras.
+#[derive(Clone, Debug)]
+pub struct TimestampAndExtra {
+    pub timestamp: DateTime<FixedOffset>,
+    pub extra: BTreeMap<String, Vec<u8>>,
+}
+
+impl TimestampAndExtra {
+    /// Parse the raw bytes of the timestamp line from a changelog entry.
+    ///
+    /// According to the documentation in `hg help dates` and the
+    /// implementation in `changelog.py`, the format of the timestamp line
+    /// is `time tz extra\n` where:
+    ///
+    /// - `time` is an ASCII-encoded signed int or float denoting a UTC
+    ///   timestamp as seconds since the UNIX epoch.
+    ///
+    /// - `tz` is the timezone offset as an ASCII-encoded signed integer
+    ///   denoting seconds WEST of UTC (so negative for timezones east of UTC,
+    ///   which is the opposite of the sign in ISO 8601 timestamps).
+    ///
+    /// - `extra` is an optional set of NUL-delimited key-value pairs, with the
+    ///   key and value in each pair separated by an ASCII colon. Keys are
+    ///   limited to ASCII letters, digits, hyphens, and underscores, whereas
+    ///   values can be arbitrary bytes.
+    fn from_bytes(line: &[u8]) -> Result<Self, HgError> {
+        let mut parts = line.splitn(3, |c| *c == b' ');
+
+        let timestamp_bytes = parts
+            .next()
+            .ok_or_else(|| HgError::corrupted("missing timestamp"))?;
+        let timestamp_str = str::from_utf8(timestamp_bytes).map_err(|e| {
+            HgError::corrupted(format!("timestamp is not valid UTF-8: {e}"))
+        })?;
+        let timestamp_utc = timestamp_str
+            .parse()
+            .map_err(|e| {
+                HgError::corrupted(format!("failed to parse timestamp: {e}"))
+            })
+            .and_then(|secs| {
+                NaiveDateTime::from_timestamp_opt(secs, 0).ok_or_else(|| {
+                    HgError::corrupted(format!(
+                        "integer timestamp out of valid range: {secs}"
+                    ))
+                })
+            })
+            // Attempt to parse the timestamp as a float if we can't parse
+            // it as an int. It doesn't seem like float timestamps are actually
+            // used in practice, but the Python code supports them.
+            .or_else(|_| parse_float_timestamp(timestamp_str))?;
+
+        let timezone_bytes = parts
+            .next()
+            .ok_or_else(|| HgError::corrupted("missing timezone"))?;
+        let timezone_secs: i32 = str::from_utf8(timezone_bytes)
+            .map_err(|e| {
+                HgError::corrupted(format!("timezone is not valid UTF-8: {e}"))
+            })?
+            .parse()
+            .map_err(|e| {
+                HgError::corrupted(format!("timezone is not an integer: {e}"))
+            })?;
+        let timezone =
+            FixedOffset::west_opt(timezone_secs).ok_or_else(|| {
+                HgError::corrupted("timezone offset out of bounds")
+            })?;
+
+        let timestamp =
+            DateTime::from_naive_utc_and_offset(timestamp_utc, timezone);
+        let extra = parts
+            .next()
+            .map(parse_extra)
+            .transpose()?
+            .unwrap_or_default();
+
+        Ok(Self { timestamp, extra })
+    }
+}
+
+/// Attempt to parse the given string as floating-point timestamp, and
+/// convert the result into a `chrono::NaiveDateTime`.
+fn parse_float_timestamp(
+    timestamp_str: &str,
+) -> Result<NaiveDateTime, HgError> {
+    let timestamp = timestamp_str.parse::<f64>().map_err(|e| {
+        HgError::corrupted(format!("failed to parse timestamp: {e}"))
+    })?;
+
+    // To construct a `NaiveDateTime` we'll need to convert the float
+    // into signed integer seconds and unsigned integer nanoseconds.
+    let mut secs = timestamp.trunc() as i64;
+    let mut subsecs = timestamp.fract();
+
+    // If the timestamp is negative, we need to express the fractional
+    // component as positive nanoseconds since the previous second.
+    if timestamp < 0.0 {
+        secs -= 1;
+        subsecs += 1.0;
+    }
+
+    // This cast should be safe because the fractional component is
+    // by definition less than 1.0, so this value should not exceed
+    // 1 billion, which is representable as an f64 without loss of
+    // precision and should fit into a u32 without overflowing.
+    //
+    // (Any loss of precision in the fractional component will have
+    // already happened at the time of initial parsing; in general,
+    // f64s are insufficiently precise to provide nanosecond-level
+    // precision with present-day timestamps.)
+    let nsecs = (subsecs * 1_000_000_000.0) as u32;
+
+    NaiveDateTime::from_timestamp_opt(secs, nsecs).ok_or_else(|| {
+        HgError::corrupted(format!(
+            "float timestamp out of valid range: {timestamp}"
+        ))
+    })
+}
+
+/// Parse the "extra" fields from a changeset's timestamp line.
+///
+/// Extras are null-delimited key-value pairs where the key consists of ASCII
+/// alphanumeric characters plus hyphens and underscores, and the value can
+/// contain arbitrary bytes.
+fn parse_extra(extra: &[u8]) -> Result<BTreeMap<String, Vec<u8>>, HgError> {
+    extra
+        .split(|c| *c == b'\0')
+        .map(|pair| {
+            let pair = unescape_extra(pair);
+            let mut iter = pair.splitn(2, |c| *c == b':');
+
+            let key_bytes =
+                iter.next().filter(|k| !k.is_empty()).ok_or_else(|| {
+                    HgError::corrupted("empty key in changeset extras")
+                })?;
+
+            let key = str::from_utf8(key_bytes)
+                .ok()
+                .filter(|k| {
+                    k.chars().all(|c| {
+                        c.is_ascii_alphanumeric() || c == '_' || c == '-'
+                    })
+                })
+                .ok_or_else(|| {
+                    let key = String::from_utf8_lossy(key_bytes);
+                    HgError::corrupted(format!(
+                        "invalid key in changeset extras: {key}",
+                    ))
+                })?
+                .to_string();
+
+            let value = iter.next().map(Into::into).ok_or_else(|| {
+                HgError::corrupted(format!(
+                    "missing value for changeset extra: {key}"
+                ))
+            })?;
+
+            Ok((key, value))
+        })
+        .collect()
+}
+
+/// Decode Mercurial's escaping for changelog extras.
+///
+/// The `_string_escape` function in `changelog.py` only escapes 4 characters
+/// (null, backslash, newline, and carriage return) so we only decode those.
+///
+/// The Python code also includes a workaround for decoding escaped nuls
+/// that are followed by an ASCII octal digit, since Python's built-in
+/// `string_escape` codec will interpret that as an escaped octal byte value.
+/// That workaround is omitted here since we don't support decoding octal.
+fn unescape_extra(bytes: &[u8]) -> Vec<u8> {
+    let mut output = Vec::with_capacity(bytes.len());
+    let mut input = bytes.iter().copied();
+
+    while let Some(c) = input.next() {
+        if c != b'\\' {
+            output.push(c);
+            continue;
+        }
+
+        match input.next() {
+            Some(b'0') => output.push(b'\0'),
+            Some(b'\\') => output.push(b'\\'),
+            Some(b'n') => output.push(b'\n'),
+            Some(b'r') => output.push(b'\r'),
+            // The following cases should never occur in theory because any
+            // backslashes in the original input should have been escaped
+            // with another backslash, so it should not be possible to
+            // observe an escape sequence other than the 4 above.
+            Some(c) => output.extend_from_slice(&[b'\\', c]),
+            None => output.push(b'\\'),
+        }
+    }
+
+    output
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -375,4 +581,166 @@
             .collect_vec()
             .is_empty());
     }
+
+    #[test]
+    fn test_unescape_basic() {
+        // '\0', '\\', '\n', and '\r' are correctly unescaped.
+        let expected = b"AAA\0BBB\\CCC\nDDD\rEEE";
+        let escaped = br"AAA\0BBB\\CCC\nDDD\rEEE";
+        let unescaped = unescape_extra(escaped);
+        assert_eq!(&expected[..], &unescaped[..]);
+    }
+
+    #[test]
+    fn test_unescape_unsupported_sequence() {
+        // Other escape sequences are left unaltered.
+        for c in 0u8..255 {
+            match c {
+                b'0' | b'\\' | b'n' | b'r' => continue,
+                c => {
+                    let expected = &[b'\\', c][..];
+                    let unescaped = unescape_extra(expected);
+                    assert_eq!(expected, &unescaped[..]);
+                }
+            }
+        }
+    }
+
+    #[test]
+    fn test_unescape_trailing_backslash() {
+        // Trailing backslashes are OK.
+        let expected = br"hi\";
+        let unescaped = unescape_extra(expected);
+        assert_eq!(&expected[..], &unescaped[..]);
+    }
+
+    #[test]
+    fn test_unescape_nul_followed_by_octal() {
+        // Escaped NUL chars followed by octal digits are decoded correctly.
+        let expected = b"\012";
+        let escaped = br"\012";
+        let unescaped = unescape_extra(escaped);
+        assert_eq!(&expected[..], &unescaped[..]);
+    }
+
+    #[test]
+    fn test_parse_float_timestamp() {
+        let test_cases = [
+            // Zero should map to the UNIX epoch.
+            ("0.0", "1970-01-01 00:00:00"),
+            // Negative zero should be the same as positive zero.
+            ("-0.0", "1970-01-01 00:00:00"),
+            // Values without fractional components should work like integers.
+            // (Assuming the timestamp is within the limits of f64 precision.)
+            ("1115154970.0", "2005-05-03 21:16:10"),
+            // We expect some loss of precision in the fractional component
+            // when parsing arbitrary floating-point values.
+            ("1115154970.123456789", "2005-05-03 21:16:10.123456716"),
+            // But representable f64 values should parse losslessly.
+            ("1115154970.123456716", "2005-05-03 21:16:10.123456716"),
+            // Negative fractional components are subtracted from the epoch.
+            ("-1.333", "1969-12-31 23:59:58.667"),
+        ];
+
+        for (input, expected) in test_cases {
+            let res = parse_float_timestamp(input).unwrap().to_string();
+            assert_eq!(res, expected);
+        }
+    }
+
+    fn escape_extra(bytes: &[u8]) -> Vec<u8> {
+        let mut output = Vec::with_capacity(bytes.len());
+
+        for c in bytes.iter().copied() {
+            output.extend_from_slice(match c {
+                b'\0' => &b"\\0"[..],
+                b'\\' => &b"\\\\"[..],
+                b'\n' => &b"\\n"[..],
+                b'\r' => &b"\\r"[..],
+                _ => {
+                    output.push(c);
+                    continue;
+                }
+            });
+        }
+
+        output
+    }
+
+    fn encode_extra<K, V>(pairs: impl IntoIterator<Item = (K, V)>) -> Vec<u8>
+    where
+        K: AsRef<[u8]>,
+        V: AsRef<[u8]>,
+    {
+        let extras = pairs.into_iter().map(|(k, v)| {
+            escape_extra(&[k.as_ref(), b":", v.as_ref()].concat())
+        });
+        // Use fully-qualified syntax to avoid a future naming conflict with
+        // the standard library: https://github.com/rust-lang/rust/issues/79524
+        Itertools::intersperse(extras, b"\0".to_vec()).concat()
+    }
+
+    #[test]
+    fn test_parse_extra() {
+        let extra = [
+            ("branch".into(), b"default".to_vec()),
+            ("key-with-hyphens".into(), b"value1".to_vec()),
+            ("key_with_underscores".into(), b"value2".to_vec()),
+            ("empty-value".into(), b"".to_vec()),
+            ("binary-value".into(), (0u8..=255).collect::<Vec<_>>()),
+        ]
+        .into_iter()
+        .collect::<BTreeMap<String, Vec<u8>>>();
+
+        let encoded = encode_extra(&extra);
+        let parsed = parse_extra(&encoded).unwrap();
+
+        assert_eq!(extra, parsed);
+    }
+
+    #[test]
+    fn test_corrupt_extra() {
+        let test_cases = [
+            (&b""[..], "empty input"),
+            (&b"\0"[..], "unexpected null byte"),
+            (&b":empty-key"[..], "empty key"),
+            (&b"\0leading-null:"[..], "leading null"),
+            (&b"trailing-null:\0"[..], "trailing null"),
+            (&b"missing-value"[..], "missing value"),
+            (&b"$!@# non-alphanum-key:"[..], "non-alphanumeric key"),
+            (&b"\xF0\x9F\xA6\x80 non-ascii-key:"[..], "non-ASCII key"),
+        ];
+
+        for (extra, msg) in test_cases {
+            assert!(
+                parse_extra(&extra).is_err(),
+                "corrupt extra should have failed to parse: {}",
+                msg
+            );
+        }
+    }
+
+    #[test]
+    fn test_parse_timestamp_line() {
+        let extra = [
+            ("branch".into(), b"default".to_vec()),
+            ("key-with-hyphens".into(), b"value1".to_vec()),
+            ("key_with_underscores".into(), b"value2".to_vec()),
+            ("empty-value".into(), b"".to_vec()),
+            ("binary-value".into(), (0u8..=255).collect::<Vec<_>>()),
+        ]
+        .into_iter()
+        .collect::<BTreeMap<String, Vec<u8>>>();
+
+        let mut line: Vec<u8> = b"1115154970 28800 ".to_vec();
+        line.extend_from_slice(&encode_extra(&extra));
+
+        let parsed = TimestampAndExtra::from_bytes(&line).unwrap();
+
+        assert_eq!(
+            &parsed.timestamp.to_rfc3339(),
+            "2005-05-03T13:16:10-08:00"
+        );
+        assert_eq!(extra, parsed.extra);
+    }
 }