diff mercurial/revlog.py @ 39794:a6b3c4c1019f

revlog: move censor logic out of censor extension

The censor extension is doing very low-level things with revlogs.
It is fundamentally impossible for this logic to remain in the censor
extension while support multiple storage backends: we need each
storage backend to implement censor in its own storage-specific
way.

This commit effectively moves the revlog-specific censoring code to
be a method of revlogs themselves.

We've defined a new API on the file storage interface for censoring
an individual node. Even though the current censoring code doesn't
use it, the API requires a transaction instance because it logically
makes sense for storage backends to require an active transaction
(which implies a held write lock) in order to rewrite storage.

After this commit, the censor extension has been reduced to
boilerplate precondition checking before invoking the generic
storage API.

I tried to keep the code as similar as possible. But some minor
changes were made:

* We use self._io instead of instantiating a new revlogio instance.
* We compare self.version against REVLOGV0 instead of != REVLOGV1
  because presumably all future revlog versions will support censoring.
* We use self.opener instead of going through repo.svfs (we don't have
  a handle on the repo instance from a revlog).
* "revlog" dropped
* Replace "flog" with "self".

Differential Revision: https://phab.mercurial-scm.org/D4656

--- a/mercurial/revlog.py	Tue Sep 18 16:47:09 2018 -0700
+++ b/mercurial/revlog.py	Tue Sep 18 17:51:43 2018 -0700
@@ -2492,3 +2492,92 @@
         finally:
             destrevlog._lazydeltabase = oldlazydeltabase
             destrevlog._deltabothparents = oldamd
+
+    def censorrevision(self, node, tombstone=b''):
+        if (self.version & 0xFFFF) == REVLOGV0:
+            raise error.RevlogError(_('cannot censor with version %d revlogs') %
+                                    self.version)
+
+        rev = self.rev(node)
+        tombstone = packmeta({b'censored': tombstone}, b'')
+
+        if len(tombstone) > self.rawsize(rev):
+            raise error.Abort(_('censor tombstone must be no longer than '
+                                'censored data'))
+
+        # Using two files instead of one makes it easy to rewrite entry-by-entry
+        idxread = self.opener(self.indexfile, 'r')
+        idxwrite = self.opener(self.indexfile, 'wb', atomictemp=True)
+        if self.version & FLAG_INLINE_DATA:
+            dataread, datawrite = idxread, idxwrite
+        else:
+            dataread = self.opener(self.datafile, 'r')
+            datawrite = self.opener(self.datafile, 'wb', atomictemp=True)
+
+        # Copy all revlog data up to the entry to be censored.
+        offset = self.start(rev)
+
+        for chunk in util.filechunkiter(idxread, limit=rev * self._io.size):
+            idxwrite.write(chunk)
+        for chunk in util.filechunkiter(dataread, limit=offset):
+            datawrite.write(chunk)
+
+        def rewriteindex(r, newoffs, newdata=None):
+            """Rewrite the index entry with a new data offset and new data.
+
+            The newdata argument, if given, is a tuple of three positive
+            integers: (new compressed, new uncompressed, added flag bits).
+            """
+            offlags, comp, uncomp, base, link, p1, p2, nodeid = self.index[r]
+            flags = gettype(offlags)
+            if newdata:
+                comp, uncomp, nflags = newdata
+                flags |= nflags
+            offlags = offset_type(newoffs, flags)
+            e = (offlags, comp, uncomp, r, link, p1, p2, nodeid)
+            idxwrite.write(self._io.packentry(e, None, self.version, r))
+            idxread.seek(self._io.size, 1)
+
+        def rewrite(r, offs, data, nflags=REVIDX_DEFAULT_FLAGS):
+            """Write the given fulltext with the given data offset.
+
+            Returns:
+                The integer number of data bytes written, for tracking data
+                offsets.
+            """
+            flag, compdata = self.compress(data)
+            newcomp = len(flag) + len(compdata)
+            rewriteindex(r, offs, (newcomp, len(data), nflags))
+            datawrite.write(flag)
+            datawrite.write(compdata)
+            dataread.seek(self.length(r), 1)
+            return newcomp
+
+        # Rewrite censored entry with (padded) tombstone data.
+        pad = ' ' * (self.rawsize(rev) - len(tombstone))
+        offset += rewrite(rev, offset, tombstone + pad, REVIDX_ISCENSORED)
+
+        # Rewrite all following filelog revisions fixing up offsets and deltas.
+        for srev in pycompat.xrange(rev + 1, len(self)):
+            if rev in self.parentrevs(srev):
+                # Immediate children of censored node must be re-added as
+                # fulltext.
+                try:
+                    revdata = self.revision(srev)
+                except error.CensoredNodeError as e:
+                    revdata = e.tombstone
+                dlen = rewrite(srev, offset, revdata)
+            else:
+                # Copy any other revision data verbatim after fixing up the
+                # offset.
+                rewriteindex(srev, offset)
+                dlen = self.length(srev)
+                for chunk in util.filechunkiter(dataread, limit=dlen):
+                    datawrite.write(chunk)
+            offset += dlen
+
+        idxread.close()
+        idxwrite.close()
+        if dataread is not idxread:
+            dataread.close()
+            datawrite.close()