comparison mercurial/revlog.py @ 47426:cac0e0621ceb

revlog: use file read caching for sidedata

The previous changeset extracted this caching logic from the revlog class into
a new class. Adding a new instance of that class allows using the same logic
for side data.

Differential Revision: https://phab.mercurial-scm.org/D10879

47425:e0a314bcbc9d

            self.opener,
            (self._indexfile if self._inline else self._datafile),
            self._chunkcachesize,
            chunkcache,
        )
        # revnum -> (chain-length, sum-delta-length)
        self._chaininfocache = util.lrucachedict(500)
        # revlog header -> revlog compressor
        self._decompressors = {}

    def clearcaches(self):
        self._revisioncache = None
        self._chainbasecache.clear()
        self._segmentfile.clear_cache()
        self._pcache = {}
        self._nodemap_docket = None
        self.index.clearcaches()
        # The python code is the one responsible for validating the docket, we
        # end up having to refresh it here.

        if self._inline:
            sidedata_offset += self.index.entry_size * (1 + rev)
        if sidedata_size == 0:
            return {}

        # XXX this need caching, as we do for data
        with self._sidedatareadfp() as sdf:
            if self._docket.sidedata_end < sidedata_offset + sidedata_size:
                filename = self._sidedatafile
                end = self._docket.sidedata_end
                offset = sidedata_offset
                length = sidedata_size
                m = FILE_TOO_SHORT_MSG % (filename, length, offset, end)
                raise error.RevlogError(m)

            sdf.seek(sidedata_offset, os.SEEK_SET)
            comp_segment = sdf.read(sidedata_size)

            if len(comp_segment) < sidedata_size:
                filename = self._sidedatafile
                length = sidedata_size
                offset = sidedata_offset
                got = len(comp_segment)
                m = randomaccessfile.PARTIAL_READ_MSG % (
                    filename,
                    length,
                    offset,
                    got,
                )
                raise error.RevlogError(m)

        comp = self.index[rev][11]
        if comp == COMP_MODE_PLAIN:
            segment = comp_segment
        elif comp == COMP_MODE_DEFAULT:

            fp.close()
            # We can't use the cached file handle after close(). So prevent
            # its usage.
            self._writinghandles = None
            self._segmentfile.writing_handle = None

        new_dfh = self._datafp(b'w+')
        new_dfh.truncate(0)  # drop any potentially existing data
        try:
            with self._indexfp() as read_ifh:

                # switched from inline to conventional reopen the index
                ifh = self.__index_write_fp()
                self._writinghandles = (ifh, new_dfh, None)
                self._segmentfile.writing_handle = new_dfh
                new_dfh = None
        finally:
            if new_dfh is not None:
                new_dfh.close()

    def _nodeduplicatecallback(self, transaction, node):

                else:
                    transaction.add(self._indexfile, isize)
                # exposing all file handle for writing.
                self._writinghandles = (ifh, dfh, sdfh)
                self._segmentfile.writing_handle = ifh if self._inline else dfh
                yield
                if self._docket is not None:
                    self._write_docket(transaction)
            finally:
                self._writinghandles = None
                self._segmentfile.writing_handle = None
                if dfh is not None:
                    dfh.close()
                if sdfh is not None:
                    sdfh.close()
                # closing the index file last to avoid exposing referent to

        # then reset internal state in memory to forget those revisions
        self._revisioncache = None
        self._chaininfocache = util.lrucachedict(500)
        self._segmentfile.clear_cache()

        del self.index[rev:-1]

    def checksize(self):
        """Check size of index and data files

47426:cac0e0621ceb

            self.opener,
            (self._indexfile if self._inline else self._datafile),
            self._chunkcachesize,
            chunkcache,
        )
        self._segmentfile_sidedata = randomaccessfile.randomaccessfile(
            self.opener,
            self._sidedatafile,
            self._chunkcachesize,
        )
        # revnum -> (chain-length, sum-delta-length)
        self._chaininfocache = util.lrucachedict(500)
        # revlog header -> revlog compressor
        self._decompressors = {}

    def clearcaches(self):
        self._revisioncache = None
        self._chainbasecache.clear()
        self._segmentfile.clear_cache()
        self._segmentfile_sidedata.clear_cache()
        self._pcache = {}
        self._nodemap_docket = None
        self.index.clearcaches()
        # The python code is the one responsible for validating the docket, we
        # end up having to refresh it here.

        if self._inline:
            sidedata_offset += self.index.entry_size * (1 + rev)
        if sidedata_size == 0:
            return {}

        if self._docket.sidedata_end < sidedata_offset + sidedata_size:
            filename = self._sidedatafile
            end = self._docket.sidedata_end
            offset = sidedata_offset
            length = sidedata_size
            m = FILE_TOO_SHORT_MSG % (filename, length, offset, end)
            raise error.RevlogError(m)

        comp_segment = self._segmentfile_sidedata.read_chunk(
            sidedata_offset, sidedata_size
        )

        comp = self.index[rev][11]
        if comp == COMP_MODE_PLAIN:
            segment = comp_segment
        elif comp == COMP_MODE_DEFAULT:

            fp.close()
            # We can't use the cached file handle after close(). So prevent
            # its usage.
            self._writinghandles = None
            self._segmentfile.writing_handle = None
            # No need to deal with sidedata writing handle as it is only
            # relevant with revlog-v2 which is never inline, not reaching
            # this code

        new_dfh = self._datafp(b'w+')
        new_dfh.truncate(0)  # drop any potentially existing data
        try:
            with self._indexfp() as read_ifh:

                # switched from inline to conventional reopen the index
                ifh = self.__index_write_fp()
                self._writinghandles = (ifh, new_dfh, None)
                self._segmentfile.writing_handle = new_dfh
                new_dfh = None
                # No need to deal with sidedata writing handle as it is only
                # relevant with revlog-v2 which is never inline, not reaching
                # this code
        finally:
            if new_dfh is not None:
                new_dfh.close()

    def _nodeduplicatecallback(self, transaction, node):

                else:
                    transaction.add(self._indexfile, isize)
                # exposing all file handle for writing.
                self._writinghandles = (ifh, dfh, sdfh)
                self._segmentfile.writing_handle = ifh if self._inline else dfh
                self._segmentfile_sidedata.writing_handle = sdfh
                yield
                if self._docket is not None:
                    self._write_docket(transaction)
            finally:
                self._writinghandles = None
                self._segmentfile.writing_handle = None
                self._segmentfile_sidedata.writing_handle = None
                if dfh is not None:
                    dfh.close()
                if sdfh is not None:
                    sdfh.close()
                # closing the index file last to avoid exposing referent to

        # then reset internal state in memory to forget those revisions
        self._revisioncache = None
        self._chaininfocache = util.lrucachedict(500)
        self._segmentfile.clear_cache()
        self._segmentfile_sidedata.clear_cache()

        del self.index[rev:-1]

    def checksize(self):
        """Check size of index and data files