nodemap: write nodemap data on disk

nodemap: write nodemap data on disk

Let us start writing data on disk (so that we can read it from there later).
This series of changeset is going to focus first on having data on disk and
updating it.

Right now the data is written right next to the revlog data, in the store. We
might move it to cache (with proper cache validation mechanism) later, but for
now revlog have a storevfs instance and it is simpler to us it. The right
location for this data is not the focus of this series.

Differential Revision: https://phab.mercurial-scm.org/D7835

--- a/mercurial/changelog.py	Wed Jan 15 15:47:12 2020 +0100
+++ b/mercurial/changelog.py	Wed Jan 15 15:47:21 2020 +0100
@@ -385,6 +385,9 @@
             datafile=datafile,
             checkambig=True,
             mmaplargeindex=True,
+            persistentnodemap=opener.options.get(
+                b'exp-persistent-nodemap', False
+            ),
         )
 
         if self._initempty and (self.version & 0xFFFF == revlog.REVLOGV1):

--- a/mercurial/configitems.py	Wed Jan 15 15:47:12 2020 +0100
+++ b/mercurial/configitems.py	Wed Jan 15 15:47:21 2020 +0100
@@ -660,6 +660,9 @@
     b'experimental', b'rust.index', default=False,
 )
 coreconfigitem(
+    b'experimental', b'exp-persistent-nodemap', default=False,
+)
+coreconfigitem(
     b'experimental', b'server.filesdata.recommended-batch-size', default=50000,
 )
 coreconfigitem(

--- a/mercurial/localrepo.py	Wed Jan 15 15:47:12 2020 +0100
+++ b/mercurial/localrepo.py	Wed Jan 15 15:47:21 2020 +0100
@@ -932,6 +932,8 @@
 
     if ui.configbool(b'experimental', b'rust.index'):
         options[b'rust.index'] = True
+    if ui.configbool(b'experimental', b'exp-persistent-nodemap'):
+        options[b'exp-persistent-nodemap'] = True
 
     return options
 

--- a/mercurial/revlog.py	Wed Jan 15 15:47:12 2020 +0100
+++ b/mercurial/revlog.py	Wed Jan 15 15:47:21 2020 +0100
@@ -407,6 +407,7 @@
         mmaplargeindex=False,
         censorable=False,
         upperboundcomp=None,
+        persistentnodemap=False,
     ):
         """
         create a revlog object
@@ -418,6 +419,10 @@
         self.upperboundcomp = upperboundcomp
         self.indexfile = indexfile
         self.datafile = datafile or (indexfile[:-2] + b".d")
+        self.nodemap_file = None
+        if persistentnodemap:
+            self.nodemap_file = indexfile[:-2] + b".n"
+
         self.opener = opener
         #  When True, indexfile is opened with checkambig=True at writing, to
         #  avoid file stat ambiguity.
@@ -2286,6 +2291,7 @@
             ifh.write(data[0])
             ifh.write(data[1])
             self._enforceinlinesize(transaction, ifh)
+        nodemaputil.setup_persistent_nodemap(transaction, self)
 
     def addgroup(self, deltas, linkmapper, transaction, addrevisioncb=None):
         """

--- a/mercurial/revlogutils/nodemap.py	Wed Jan 15 15:47:12 2020 +0100
+++ b/mercurial/revlogutils/nodemap.py	Wed Jan 15 15:47:21 2020 +0100
@@ -22,6 +22,39 @@
         raise error.RevlogError(b'unknown node: %s' % x)
 
 
+def setup_persistent_nodemap(tr, revlog):
+    """Install whatever is needed transaction side to persist a nodemap on disk
+
+    (only actually persist the nodemap if this is relevant for this revlog)
+    """
+    if revlog.nodemap_file is None:
+        return  # we do not use persistent_nodemap on this revlog
+    callback_id = b"revlog-persistent-nodemap-%s" % revlog.nodemap_file
+    if tr.hasfinalize(callback_id):
+        return  # no need to register again
+    tr.addfinalize(callback_id, lambda tr: _persist_nodemap(tr, revlog))
+
+
+def _persist_nodemap(tr, revlog):
+    """Write nodemap data on disk for a given revlog
+    """
+    if getattr(revlog, 'filteredrevs', ()):
+        raise error.ProgrammingError(
+            "cannot persist nodemap of a filtered changelog"
+        )
+    if revlog.nodemap_file is None:
+        msg = "calling persist nodemap on a revlog without the feature enableb"
+        raise error.ProgrammingError(msg)
+    data = persistent_data(revlog.index)
+    # EXP-TODO: if this is a cache, this should use a cache vfs, not a
+    # store vfs
+    with revlog.opener(revlog.nodemap_file, b'w') as f:
+        f.write(data)
+    # EXP-TODO: if the transaction abort, we should remove the new data and
+    # reinstall the old one. (This will be simpler when the file format get a
+    # bit more advanced)
+
+
 ### Nodemap Trie
 #
 # This is a simple reference implementation to compute and persist a nodemap

--- a/tests/test-persistent-nodemap.t	Wed Jan 15 15:47:12 2020 +0100
+++ b/tests/test-persistent-nodemap.t	Wed Jan 15 15:47:21 2020 +0100
@@ -5,8 +5,14 @@
 
   $ hg init test-repo
   $ cd test-repo
+  $ cat << EOF >> .hg/hgrc
+  > [experimental]
+  > exp-persistent-nodemap=yes
+  > EOF
   $ hg debugbuilddag .+5000
-  $ hg debugnodemap --dump | f --sha256 --bytes=256 --hexdump --size
+  $ hg debugnodemap --dump | f --sha256 --size
+  size=122880, sha256=b961925120e1c9bc345c199b2cc442abc477029fdece37ef9d99cbe59c0558b7
+  $ f --sha256 --bytes=256 --hexdump --size < .hg/store/00changelog.n
   size=122880, sha256=b961925120e1c9bc345c199b2cc442abc477029fdece37ef9d99cbe59c0558b7
   0000: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
   0010: ff ff ff ff ff ff ff ff ff ff fa c2 ff ff ff ff |................|