comparison mercurial/revlog.py @ 47238:6597255a4f94

revlogv2: track current index size in the docket

This help use to fix transaction safety on repos. See next changesets for details.

Differential Revision: https://phab.mercurial-scm.org/D10628

47237:864f4ebe6a8d

                % self._chunkcachesize
            )
        force_nodemap = opts.get(b'devel-force-nodemap', False)
        return new_header, mmapindexthreshold, force_nodemap

    def _get_data(self, filepath, mmap_threshold):
        """return a file content with or without mmap

        If the file is missing return the empty string"""
        try:
            with self.opener(filepath) as fp:
                if mmap_threshold is not None:
                    file_size = self.opener.fstat(fp).st_size
                    if file_size >= mmap_threshold:
                        # TODO: should .close() to release resources without
                        # relying on Python GC
                        return util.buffer(util.mmapread(fp))
                return fp.read()
        except IOError as inst:
            if inst.errno != errno.ENOENT:
                raise
            return b''

            if self._initempty:
                self._docket = docketutil.default_docket(self, header)
            else:
                self._docket = docketutil.parse_docket(self, entry_data)
            self._indexfile = self._docket.index_filepath()
            index_data = self._get_data(self._indexfile, mmapindexthreshold)
            self._inline = False
            # generaldelta implied by version 2 revlogs.
            self._generaldelta = True
            # the logic for persistent nodemap will be dealt with within the
            # main docket, so disable it for now.

        # You should not use this directly and use `_writing` instead
        try:
            f = self.opener(
                self._indexfile, mode=b"r+", checkambig=self._checkambig
            )
            f.seek(0, os.SEEK_END)
            return f
        except IOError as inst:
            if inst.errno != errno.ENOENT:
                raise
            return self.opener(

                    if i == 0 and self._docket is None:
                        header = self._format_flags | self._format_version
                        header = self.index.pack_header(header)
                        e = header + e
                    fp.write(e)
                # the temp file replace the real index when we exit the context
                # manager

            tr.replace(self._indexfile, trindex * self.index.entry_size)
            nodemaputil.setup_persistent_nodemap(tr, self)

        # to be careful before changing this.
        if self._writinghandles is None:
            msg = b'adding revision outside `revlog._writing` context'
            raise error.ProgrammingError(msg)
        ifh, dfh = self._writinghandles
        ifh.seek(0, os.SEEK_END)
        if dfh:
            dfh.seek(0, os.SEEK_END)

        curr = len(self) - 1
        if not self._inline:

            ifh.write(data[0])
            ifh.write(data[1])
            if sidedata:
                ifh.write(sidedata)
            self._enforceinlinesize(transaction)
        nodemaputil.setup_persistent_nodemap(transaction, self)

    def addgroup(
        self,
        deltas,

            end = rev * self.index.entry_size
        else:
            end += rev * self.index.entry_size

        transaction.add(self._indexfile, end)

        # then reset internal state in memory to forget those revisions
        self._revisioncache = None
        self._chaininfocache = util.lrucachedict(500)
        self._chunkclear()

47238:6597255a4f94

                % self._chunkcachesize
            )
        force_nodemap = opts.get(b'devel-force-nodemap', False)
        return new_header, mmapindexthreshold, force_nodemap

    def _get_data(self, filepath, mmap_threshold, size=None):
        """return a file content with or without mmap

        If the file is missing return the empty string"""
        try:
            with self.opener(filepath) as fp:
                if mmap_threshold is not None:
                    file_size = self.opener.fstat(fp).st_size
                    if file_size >= mmap_threshold:
                        if size is not None:
                            # avoid potentiel mmap crash
                            size = min(file_size, size)
                        # TODO: should .close() to release resources without
                        # relying on Python GC
                        if size is None:
                            return util.buffer(util.mmapread(fp))
                        else:
                            return util.buffer(util.mmapread(fp, size))
                if size is None:
                    return fp.read()
                else:
                    return fp.read(size)
        except IOError as inst:
            if inst.errno != errno.ENOENT:
                raise
            return b''

            if self._initempty:
                self._docket = docketutil.default_docket(self, header)
            else:
                self._docket = docketutil.parse_docket(self, entry_data)
            self._indexfile = self._docket.index_filepath()
            index_data = b''
            index_size = self._docket.index_end
            if index_size > 0:
                index_data = self._get_data(
                    self._indexfile, mmapindexthreshold, size=index_size
                )
                if len(index_data) < index_size:
                    msg = _(b'too few index data for %s: got %d, expected %d')
                    msg %= (self.display_id, len(index_data), index_size)
                    raise error.RevlogError(msg)

            self._inline = False
            # generaldelta implied by version 2 revlogs.
            self._generaldelta = True
            # the logic for persistent nodemap will be dealt with within the
            # main docket, so disable it for now.

        # You should not use this directly and use `_writing` instead
        try:
            f = self.opener(
                self._indexfile, mode=b"r+", checkambig=self._checkambig
            )
            if self._docket is None:
                f.seek(0, os.SEEK_END)
            else:
                f.seek(self._docket.index_end, os.SEEK_SET)
            return f
        except IOError as inst:
            if inst.errno != errno.ENOENT:
                raise
            return self.opener(

                    if i == 0 and self._docket is None:
                        header = self._format_flags | self._format_version
                        header = self.index.pack_header(header)
                        e = header + e
                    fp.write(e)
                if self._docket is not None:
                    self._docket.index_end = fp.tell()
                # the temp file replace the real index when we exit the context
                # manager

            tr.replace(self._indexfile, trindex * self.index.entry_size)
            nodemaputil.setup_persistent_nodemap(tr, self)

        # to be careful before changing this.
        if self._writinghandles is None:
            msg = b'adding revision outside `revlog._writing` context'
            raise error.ProgrammingError(msg)
        ifh, dfh = self._writinghandles
        if self._docket is None:
            ifh.seek(0, os.SEEK_END)
        else:
            ifh.seek(self._docket.index_end, os.SEEK_SET)
        if dfh:
            dfh.seek(0, os.SEEK_END)

        curr = len(self) - 1
        if not self._inline:

            ifh.write(data[0])
            ifh.write(data[1])
            if sidedata:
                ifh.write(sidedata)
            self._enforceinlinesize(transaction)
        if self._docket is not None:
            self._docket.index_end = self._writinghandles[0].tell()

        nodemaputil.setup_persistent_nodemap(transaction, self)

    def addgroup(
        self,
        deltas,

            end = rev * self.index.entry_size
        else:
            end += rev * self.index.entry_size

        transaction.add(self._indexfile, end)
        if self._docket is not None:
            # XXX we could, leverage the docket while stripping. However it is
            # not powerfull enough at the time of this comment
            self._docket.index_end = end
            self._docket.write(transaction, stripping=True)

        # then reset internal state in memory to forget those revisions
        self._revisioncache = None
        self._chaininfocache = util.lrucachedict(500)
        self._chunkclear()