comparison mercurial/revlog.py @ 50316:87f0155d68aa stable

revlog: improve the robustness of the splitting process

The previous "in-place" splitting, preserving the splitting on transaction
failure had a couple of issue in case of transaction rollback:
- a race windows that could still lead to a crash and data loss
- it corrupted the `fncache`.

So instead, we use a new approach that we summarized as "we do a backup of the
inline revlog pre-split, and we restore this in case of failure".
To make readers live easier, we don't overwrite the inline index file until
transaction finalization. (once the transaction get into its finalization phase,
it is not expected to rollback, unless some crash happens).

To do so, we write the index of the split index in a temporary file that we use
until transaction finalization. We also keep a backup of the initial inline file
to be able to rollback the split if needed.

As a result, transaction rollback cancel the split and no longer corrupt
fncache. We also no longer have a small inconsistency windows where the
transaction could be unrecoverable.

50315:cf6e1d535602

        censorable=False,
        upperboundcomp=None,
        persistentnodemap=False,
        concurrencychecker=None,
        trypending=False,
        canonical_parent_order=True,
    ):
        """
        create a revlog object

        self._datafile = None
        self._sidedatafile = None
        self._nodemap_file = None
        self.postfix = postfix
        self._trypending = trypending
        self.opener = opener
        if persistentnodemap:
            self._nodemap_file = nodemaputil.get_nodemap_file(self)

        assert target[0] in ALL_KINDS

        if self.postfix is not None:
            entry_point = b'%s.i.%s' % (self.radix, self.postfix)
        elif self._trypending and self.opener.exists(b'%s.i.a' % self.radix):
            entry_point = b'%s.i.a' % self.radix
        else:
            entry_point = b'%s.i' % self.radix

        if docket is not None:
            self._docket = docket

        except error.RevlogError:
            if self._censorable and storageutil.iscensoredtext(text):
                raise error.CensoredNodeError(self.display_id, node, text)
            raise

    def _enforceinlinesize(self, tr):
        """Check if the revlog is too big for inline and convert if so.

        This should be called after revisions are added to the revlog. If the
        revlog has grown too large to be an inline revlog, it will convert it
        to use multiple index and data files.

        troffset = tr.findoffset(self._indexfile)
        if troffset is None:
            raise error.RevlogError(
                _(b"%s not found in the transaction") % self._indexfile
            )
        trindex = None
        tr.add(self._datafile, 0)

        existing_handles = False
        if self._writinghandles is not None:
            existing_handles = True

            self._writinghandles = None
            self._segmentfile.writing_handle = None
            # No need to deal with sidedata writing handle as it is only
            # relevant with revlog-v2 which is never inline, not reaching
            # this code

        new_dfh = self._datafp(b'w+')
        new_dfh.truncate(0)  # drop any potentially existing data
        try:
            with self._indexfp() as read_ifh:
                for r in self:
                    new_dfh.write(self._getsegmentforrevs(r, r, df=read_ifh)[1])
                    if (
                        trindex is None
                        and troffset
                        <= self.start(r) + r * self.index.entry_size
                    ):
                        trindex = r
                new_dfh.flush()

            if trindex is None:
                trindex = 0

            with self.__index_new_fp() as fp:
                self._format_flags &= ~FLAG_INLINE_DATA
                self._inline = False
                for i in self:
                    e = self.index.entry_binary(i)

                        e = header + e
                    fp.write(e)
                if self._docket is not None:
                    self._docket.index_end = fp.tell()

                # There is a small transactional race here. If the rename of
                # the index fails, we should remove the datafile. It is more
                # important to ensure that the data file is not truncated
                # when the index is replaced as otherwise data is lost.
                tr.replace(self._datafile, self.start(trindex))

                # the temp file replace the real index when we exit the context
                # manager

            tr.replace(self._indexfile, trindex * self.index.entry_size)
            nodemaputil.setup_persistent_nodemap(tr, self)
            self._segmentfile = randomaccessfile.randomaccessfile(
                self.opener,
                self._datafile,
                self._chunkcachesize,

50316:87f0155d68aa

        censorable=False,
        upperboundcomp=None,
        persistentnodemap=False,
        concurrencychecker=None,
        trypending=False,
        try_split=False,
        canonical_parent_order=True,
    ):
        """
        create a revlog object

        self._datafile = None
        self._sidedatafile = None
        self._nodemap_file = None
        self.postfix = postfix
        self._trypending = trypending
        self._try_split = try_split
        self.opener = opener
        if persistentnodemap:
            self._nodemap_file = nodemaputil.get_nodemap_file(self)

        assert target[0] in ALL_KINDS

        if self.postfix is not None:
            entry_point = b'%s.i.%s' % (self.radix, self.postfix)
        elif self._trypending and self.opener.exists(b'%s.i.a' % self.radix):
            entry_point = b'%s.i.a' % self.radix
        elif self._try_split and self.opener.exists(b'%s.i.s' % self.radix):
            entry_point = b'%s.i.s' % self.radix
        else:
            entry_point = b'%s.i' % self.radix

        if docket is not None:
            self._docket = docket

        except error.RevlogError:
            if self._censorable and storageutil.iscensoredtext(text):
                raise error.CensoredNodeError(self.display_id, node, text)
            raise

    def _enforceinlinesize(self, tr, side_write=True):
        """Check if the revlog is too big for inline and convert if so.

        This should be called after revisions are added to the revlog. If the
        revlog has grown too large to be an inline revlog, it will convert it
        to use multiple index and data files.

        troffset = tr.findoffset(self._indexfile)
        if troffset is None:
            raise error.RevlogError(
                _(b"%s not found in the transaction") % self._indexfile
            )
        if troffset:
            tr.addbackup(self._indexfile, for_offset=True)
        tr.add(self._datafile, 0)

        existing_handles = False
        if self._writinghandles is not None:
            existing_handles = True

            self._writinghandles = None
            self._segmentfile.writing_handle = None
            # No need to deal with sidedata writing handle as it is only
            # relevant with revlog-v2 which is never inline, not reaching
            # this code
        if side_write:
            old_index_file_path = self._indexfile
            new_index_file_path = self._indexfile + b'.s'
            opener = self.opener

            fncache = getattr(opener, 'fncache', None)
            if fncache is not None:
                fncache.addignore(new_index_file_path)

            # the "split" index replace the real index when the transaction is finalized
            def finalize_callback(tr):
                opener.rename(
                    new_index_file_path,
                    old_index_file_path,
                    checkambig=True,
                )

            tr.registertmp(new_index_file_path)
            if self.target[1] is not None:
                finalize_id = b'000-revlog-split-%d-%s' % self.target
            else:
                finalize_id = b'000-revlog-split-%d' % self.target[0]
            tr.addfinalize(finalize_id, finalize_callback)

        new_dfh = self._datafp(b'w+')
        new_dfh.truncate(0)  # drop any potentially existing data
        try:
            with self._indexfp() as read_ifh:
                for r in self:
                    new_dfh.write(self._getsegmentforrevs(r, r, df=read_ifh)[1])
                new_dfh.flush()

            if side_write:
                self._indexfile = new_index_file_path
            with self.__index_new_fp() as fp:
                self._format_flags &= ~FLAG_INLINE_DATA
                self._inline = False
                for i in self:
                    e = self.index.entry_binary(i)

                        e = header + e
                    fp.write(e)
                if self._docket is not None:
                    self._docket.index_end = fp.tell()

                # If we don't use side-write, the temp file replace the real
                # index when we exit the context manager

            nodemaputil.setup_persistent_nodemap(tr, self)
            self._segmentfile = randomaccessfile.randomaccessfile(
                self.opener,
                self._datafile,
                self._chunkcachesize,