diff -r f77404040776 -r e0a314bcbc9d mercurial/revlog.py --- a/mercurial/revlog.py Fri Jun 11 23:39:31 2021 -0400 +++ b/mercurial/revlog.py Tue Jun 08 19:55:00 2021 +0200 @@ -86,6 +86,7 @@ docket as docketutil, flagutil, nodemap as nodemaputil, + randomaccessfile, revlogv0, sidedata as sidedatautil, ) @@ -125,7 +126,6 @@ # max size of revlog with inline data _maxinline = 131072 -_chunksize = 1048576 # Flag processors for REVIDX_ELLIPSIS. def ellipsisreadprocessor(rl, text): @@ -232,10 +232,6 @@ # signed integer) _maxentrysize = 0x7FFFFFFF -PARTIAL_READ_MSG = _( - b'partial read of revlog %s; expected %d bytes from offset %d, got %d' -) - FILE_TOO_SHORT_MSG = _( b'cannot read from revlog %s;' b' expected %d bytes from offset %d, data size is %d' @@ -605,7 +601,7 @@ self._parse_index = parse_index_v1_mixed try: d = self._parse_index(index_data, self._inline) - index, _chunkcache = d + index, chunkcache = d use_nodemap = ( not self._inline and self._nodemap_file is not None @@ -626,9 +622,13 @@ raise error.RevlogError( _(b"index %s is corrupted") % self.display_id ) - self.index, self._chunkcache = d - if not self._chunkcache: - self._chunkclear() + self.index = index + self._segmentfile = randomaccessfile.randomaccessfile( + self.opener, + (self._indexfile if self._inline else self._datafile), + self._chunkcachesize, + chunkcache, + ) # revnum -> (chain-length, sum-delta-length) self._chaininfocache = util.lrucachedict(500) # revlog header -> revlog compressor @@ -709,32 +709,6 @@ return self.opener(self._datafile, mode=mode) @contextlib.contextmanager - def _datareadfp(self, existingfp=None): - """file object suitable to read data""" - # Use explicit file handle, if given. - if existingfp is not None: - yield existingfp - - # Use a file handle being actively used for writes, if available. - # There is some danger to doing this because reads will seek the - # file. However, _writeentry() performs a SEEK_END before all writes, - # so we should be safe. - elif self._writinghandles: - if self._inline: - yield self._writinghandles[0] - else: - yield self._writinghandles[1] - - # Otherwise open a new file handle. - else: - if self._inline: - func = self._indexfp - else: - func = self._datafp - with func() as fp: - yield fp - - @contextlib.contextmanager def _sidedatareadfp(self): """file object suitable to read sidedata""" if self._writinghandles: @@ -807,7 +781,7 @@ def clearcaches(self): self._revisioncache = None self._chainbasecache.clear() - self._chunkcache = (0, b'') + self._segmentfile.clear_cache() self._pcache = {} self._nodemap_docket = None self.index.clearcaches() @@ -1629,85 +1603,6 @@ p1, p2 = self.parents(node) return storageutil.hashrevisionsha1(text, p1, p2) != node - def _cachesegment(self, offset, data): - """Add a segment to the revlog cache. - - Accepts an absolute offset and the data that is at that location. - """ - o, d = self._chunkcache - # try to add to existing cache - if o + len(d) == offset and len(d) + len(data) < _chunksize: - self._chunkcache = o, d + data - else: - self._chunkcache = offset, data - - def _readsegment(self, offset, length, df=None): - """Load a segment of raw data from the revlog. - - Accepts an absolute offset, length to read, and an optional existing - file handle to read from. - - If an existing file handle is passed, it will be seeked and the - original seek position will NOT be restored. - - Returns a str or buffer of raw byte data. - - Raises if the requested number of bytes could not be read. - """ - # Cache data both forward and backward around the requested - # data, in a fixed size window. This helps speed up operations - # involving reading the revlog backwards. - cachesize = self._chunkcachesize - realoffset = offset & ~(cachesize - 1) - reallength = ( - (offset + length + cachesize) & ~(cachesize - 1) - ) - realoffset - with self._datareadfp(df) as df: - df.seek(realoffset) - d = df.read(reallength) - - self._cachesegment(realoffset, d) - if offset != realoffset or reallength != length: - startoffset = offset - realoffset - if len(d) - startoffset < length: - filename = self._indexfile if self._inline else self._datafile - got = len(d) - startoffset - m = PARTIAL_READ_MSG % (filename, length, offset, got) - raise error.RevlogError(m) - return util.buffer(d, startoffset, length) - - if len(d) < length: - filename = self._indexfile if self._inline else self._datafile - got = len(d) - startoffset - m = PARTIAL_READ_MSG % (filename, length, offset, got) - raise error.RevlogError(m) - - return d - - def _getsegment(self, offset, length, df=None): - """Obtain a segment of raw data from the revlog. - - Accepts an absolute offset, length of bytes to obtain, and an - optional file handle to the already-opened revlog. If the file - handle is used, it's original seek position will not be preserved. - - Requests for data may be returned from a cache. - - Returns a str or a buffer instance of raw byte data. - """ - o, d = self._chunkcache - l = len(d) - - # is it in the cache? - cachestart = offset - o - cacheend = cachestart + length - if cachestart >= 0 and cacheend <= l: - if cachestart == 0 and cacheend == l: - return d # avoid a copy - return util.buffer(d, cachestart, cacheend - cachestart) - - return self._readsegment(offset, length, df=df) - def _getsegmentforrevs(self, startrev, endrev, df=None): """Obtain a segment of raw data corresponding to a range of revisions. @@ -1740,7 +1635,7 @@ end += (endrev + 1) * self.index.entry_size length = end - start - return start, self._getsegment(start, length, df=df) + return start, self._segmentfile.read_chunk(start, length, df) def _chunk(self, rev, df=None): """Obtain a single decompressed chunk for a revision. @@ -1832,10 +1727,6 @@ return l - def _chunkclear(self): - """Clear the raw chunk cache.""" - self._chunkcache = (0, b'') - def deltaparent(self, rev): """return deltaparent of the given revision""" base = self.index[rev][3] @@ -2043,7 +1934,12 @@ length = sidedata_size offset = sidedata_offset got = len(comp_segment) - m = PARTIAL_READ_MSG % (filename, length, offset, got) + m = randomaccessfile.PARTIAL_READ_MSG % ( + filename, + length, + offset, + got, + ) raise error.RevlogError(m) comp = self.index[rev][11] @@ -2136,6 +2032,7 @@ # We can't use the cached file handle after close(). So prevent # its usage. self._writinghandles = None + self._segmentfile.writing_handle = None new_dfh = self._datafp(b'w+') new_dfh.truncate(0) # drop any potentially existing data @@ -2171,12 +2068,17 @@ tr.replace(self._indexfile, trindex * self.index.entry_size) nodemaputil.setup_persistent_nodemap(tr, self) - self._chunkclear() + self._segmentfile = randomaccessfile.randomaccessfile( + self.opener, + self._datafile, + self._chunkcachesize, + ) if existing_handles: # switched from inline to conventional reopen the index ifh = self.__index_write_fp() self._writinghandles = (ifh, new_dfh, None) + self._segmentfile.writing_handle = new_dfh new_dfh = None finally: if new_dfh is not None: @@ -2235,11 +2137,13 @@ transaction.add(self._indexfile, isize) # exposing all file handle for writing. self._writinghandles = (ifh, dfh, sdfh) + self._segmentfile.writing_handle = ifh if self._inline else dfh yield if self._docket is not None: self._write_docket(transaction) finally: self._writinghandles = None + self._segmentfile.writing_handle = None if dfh is not None: dfh.close() if sdfh is not None: @@ -2873,7 +2777,7 @@ # then reset internal state in memory to forget those revisions self._revisioncache = None self._chaininfocache = util.lrucachedict(500) - self._chunkclear() + self._segmentfile.clear_cache() del self.index[rev:-1]