upgrade: allow upgrade to repository using sidedata

upgrade: allow upgrade to repository using sidedata

Repository can now be migrated to support sidedata. More requirements and
migration will be needed to actual side-data usage. This is a step in that
direction.

To test the feature, we leverage the test extension. It make sure the `update`
part of the side-data migration actually works.

Differential Revision: https://phab.mercurial-scm.org/D6942

--- a/mercurial/upgrade.py	Sun Oct 06 23:36:51 2019 -0400
+++ b/mercurial/upgrade.py	Sun Oct 06 23:36:51 2019 -0400
@@ -31,6 +31,7 @@
 RECLONES_REQUIREMENTS = {
     b'generaldelta',
     localrepo.SPARSEREVLOG_REQUIREMENT,
+    localrepo.SIDEDATA_REQUIREMENT,
 }
 
 
@@ -100,6 +101,7 @@
         b'revlogv1',
         b'store',
         localrepo.SPARSEREVLOG_REQUIREMENT,
+        localrepo.SIDEDATA_REQUIREMENT,
     }
     for name in compression.compengines:
         engine = compression.compengines[name]
@@ -125,6 +127,7 @@
         b'fncache',
         b'generaldelta',
         localrepo.SPARSEREVLOG_REQUIREMENT,
+        localrepo.SIDEDATA_REQUIREMENT,
     }
     for name in compression.compengines:
         engine = compression.compengines[name]
@@ -676,6 +679,10 @@
 )
 
 
+def getsidedatacompanion(srcrepo, destrepo):
+    return None
+
+
 def matchrevlog(revlogfilter, entry):
     """check is a revlog is selected for cloning
 
@@ -779,6 +786,8 @@
     def oncopiedrevision(rl, rev, node):
         progress.increment()
 
+    sidedatacompanion = getsidedatacompanion(srcrepo, dstrepo)
+
     # Do the actual copying.
     # FUTURE this operation can be farmed off to worker processes.
     seen = set()
@@ -870,6 +879,7 @@
                 addrevisioncb=oncopiedrevision,
                 deltareuse=deltareuse,
                 forcedeltabothparents=forcedeltabothparents,
+                sidedatacompanion=sidedatacompanion,
             )
         else:
             msg = _(b'blindly copying %s containing %i revisions\n')

--- a/tests/test-sidedata.t	Sun Oct 06 23:36:51 2019 -0400
+++ b/tests/test-sidedata.t	Sun Oct 06 23:36:51 2019 -0400
@@ -45,8 +45,8 @@
 
 Right now, sidedata has not upgrade support
 
-Check that we cannot upgrade to sidedata
-----------------------------------------
+Check that we can upgrade to sidedata
+-------------------------------------
 
   $ hg init up-no-side-data --config format.use-side-data=no
   $ hg debugformat -v -R up-no-side-data
@@ -69,9 +69,7 @@
   plain-cl-delta:    yes    yes     yes
   compression:       zlib   zlib    zlib
   compression-level: default default default
-  $ hg debugupgraderepo -R up-no-side-data --config format.use-side-data=yes
-  abort: cannot upgrade repository; do not support adding requirement: exp-sidedata-flag
-  [255]
+  $ hg debugupgraderepo -R up-no-side-data --config format.use-side-data=yes > /dev/null
 
 Check that we cannot upgrade to sidedata
 ----------------------------------------

--- a/tests/test-upgrade-repo.t	Sun Oct 06 23:36:51 2019 -0400
+++ b/tests/test-upgrade-repo.t	Sun Oct 06 23:36:51 2019 -0400
@@ -1329,6 +1329,35 @@
   sparserevlog
   store
 
-  $ cd ..
+#endif
+
+Check upgrading to a side-data revlog
+-------------------------------------
+
+upgrade
 
-#endif
+  $ hg --config format.use-side-data=yes debugupgraderepo --run  --no-backup --config "extensions.sidedata=$TESTDIR/testlib/ext-sidedata.py" >/dev/null
+  $ hg debugformat -v
+  format-variant    repo config default
+  fncache:           yes    yes     yes
+  dotencode:         yes    yes     yes
+  generaldelta:      yes    yes     yes
+  sparserevlog:      yes    yes     yes
+  sidedata:          yes     no      no
+  plain-cl-delta:    yes    yes     yes
+  compression:       zstd   zstd    zlib
+  compression-level: default default default
+  $ cat .hg/requires
+  dotencode
+  exp-sidedata-flag
+  fncache
+  generaldelta
+  revlog-compression-zstd
+  revlogv1
+  sparserevlog
+  store
+  $ hg debugsidedata -c 0
+  2 sidedata entries
+   entry-0001 size 4
+   entry-0002 size 32
+

--- a/tests/testlib/ext-sidedata.py	Sun Oct 06 23:36:51 2019 -0400
+++ b/tests/testlib/ext-sidedata.py	Sun Oct 06 23:36:51 2019 -0400
@@ -12,8 +12,10 @@
 
 from mercurial import (
     extensions,
+    localrepo,
     node,
     revlog,
+    upgrade,
 )
 
 from mercurial.revlogutils import sidedata
@@ -36,6 +38,8 @@
 
 def wraprevision(orig, self, nodeorrev, *args, **kwargs):
     text = orig(self, nodeorrev, *args, **kwargs)
+    if getattr(self, 'sidedatanocheck', False):
+        return text
     if nodeorrev != node.nullrev and nodeorrev != node.nullid:
         sd = self.sidedata(nodeorrev)
         if len(text) != struct.unpack('>I', sd[sidedata.SD_TEST1])[0]:
@@ -47,6 +51,33 @@
     return text
 
 
+def wrapgetsidedatacompanion(orig, srcrepo, dstrepo):
+    sidedatacompanion = orig(srcrepo, dstrepo)
+    addedreqs = dstrepo.requirements - srcrepo.requirements
+    if localrepo.SIDEDATA_REQUIREMENT in addedreqs:
+        assert sidedatacompanion is None  # deal with composition later
+
+        def sidedatacompanion(revlog, rev):
+            update = {}
+            revlog.sidedatanocheck = True
+            try:
+                text = revlog.revision(rev)
+            finally:
+                del revlog.sidedatanocheck
+            ## let's store some arbitrary data just for testing
+            # text length
+            update[sidedata.SD_TEST1] = struct.pack('>I', len(text))
+            # and sha2 hashes
+            sha256 = hashlib.sha256(text).digest()
+            update[sidedata.SD_TEST2] = struct.pack('>32s', sha256)
+            return False, (), update
+
+    return sidedatacompanion
+
+
 def extsetup(ui):
     extensions.wrapfunction(revlog.revlog, 'addrevision', wrapaddrevision)
     extensions.wrapfunction(revlog.revlog, 'revision', wraprevision)
+    extensions.wrapfunction(
+        upgrade, 'getsidedatacompanion', wrapgetsidedatacompanion
+    )