hgext/remotefilelog/basepack.py

43076:2372284d9457

if pycompat.isposix and not pycompat.ispy3:
    # With glibc 2.7+ the 'e' flag uses O_CLOEXEC when opening.
    # The 'e' flag will be ignored on older versions of glibc.
    # Python 3 can't handle the 'e' flag.
    PACKOPENMODE = 'rbe'
else:
    PACKOPENMODE = 'rb'


class _cachebackedpacks(object):
    def __init__(self, packs, cachesize):
        self._packs = set(packs)

                #
                # If this is an ENOENT error then don't even bother logging.
                # Someone could have removed the file since we retrieved the
                # list of paths.
                if getattr(ex, 'errno', None) != errno.ENOENT:
                    ui.warn(_('unable to load pack %s: %s\n') % (filepath, ex))
                continue
            packs.append(pack)

        self.packs = _cachebackedpacks(packs, self.DEFAULTCACHESIZE)

    def getmetrics(self):
        """Returns metrics on the state of this store."""
        size, count = self.gettotalsizeandcount()
        return {
            'numpacks': count,
            'totalpacksize': size,
        }

    def getpack(self, path):
        raise NotImplementedError()

        if version in self.SUPPORTED_VERSIONS:
            if self.VERSION is None:
                # only affect this instance
                self.VERSION = version
            elif self.VERSION != version:
                raise RuntimeError('inconsistent version: %d' % version)
        else:
            raise RuntimeError('unsupported version: %d' % version)


class basepack(versionmixin):
    # The maximum amount we should read via mmap before remmaping so the old
    # pages can be released (100MB)

        self._index = None
        self._data = None
        self.freememory()  # initialize the mmap

        version = struct.unpack('!B', self._data[:PACKVERSIONSIZE])[0]
        self._checkversion(version)

        version, config = struct.unpack('!BB', self._index[:INDEXVERSIONSIZE])
        self._checkversion(version)

        if 0b10000000 & config:
            self.params = indexparams(LARGEFANOUTPREFIX, version)
        else:

        params = self.params
        rawfanout = self._index[FANOUTSTART : FANOUTSTART + params.fanoutsize]
        fanouttable = []
        for i in pycompat.xrange(0, params.fanoutcount):
            loc = i * 4
            fanoutentry = struct.unpack('!I', rawfanout[loc : loc + 4])[0]
            fanouttable.append(fanoutentry)
        return fanouttable

    @util.propertycache
    def _indexend(self):
        nodecount = struct.unpack_from(
            '!Q', self._index, self.params.indexstart - 8
        )[0]
        return self.params.indexstart + nodecount * self.INDEXENTRYLENGTH

    def freememory(self):
        """Unmap and remap the memory to free it up after known expensive

class mutablebasepack(versionmixin):
    def __init__(self, ui, packdir, version=2):
        self._checkversion(version)
        # TODO(augie): make this configurable
        self._compressor = 'GZ'
        opener = vfsmod.vfs(packdir)
        opener.createmode = 0o444
        self.opener = opener

        self.entries = {}

        shallowutil.mkstickygroupdir(ui, packdir)
        self.packfp, self.packpath = opener.mkstemp(
            suffix=self.PACKSUFFIX + '-tmp'
        )
        self.idxfp, self.idxpath = opener.mkstemp(
            suffix=self.INDEXSUFFIX + '-tmp'
        )
        self.packfp = os.fdopen(self.packfp, r'wb+')
        self.idxfp = os.fdopen(self.idxfp, r'wb+')
        self.sha = hashlib.sha1()
        self._closed = False

        opener._fixfilemode(opener.join(self.idxpath))

        # Write header
        # TODO: make it extensible (ex: allow specifying compression algorithm,
        # a flexible key/value header, delta algorithm, fanout size, etc)
        versionbuf = struct.pack('!B', self.VERSION)  # unsigned 1 byte int
        self.writeraw(versionbuf)

    def __enter__(self):
        return self

                params.fanoutstruct, node[: params.fanoutprefix]
            )[0]
            if fanouttable[fanoutkey] == EMPTYFANOUT:
                fanouttable[fanoutkey] = location

        rawfanouttable = ''
        last = 0
        for offset in fanouttable:
            offset = offset if offset != EMPTYFANOUT else last
            last = offset
            rawfanouttable += struct.pack('!I', offset)

        rawentrieslength = struct.pack('!Q', len(self.entries))

        # The index offset is the it's location in the file. So after the 2 byte
        # header and the fanouttable.
        rawindex = self.createindex(locations, 2 + len(rawfanouttable))

        #    <large fanout: 1 bit> # 1 means 2^16, 0 means 2^8
        #    <unused: 7 bit> # future use (compression, delta format, etc)
        config = 0
        if indexparams.fanoutprefix == LARGEFANOUTPREFIX:
            config = 0b10000000
        self.idxfp.write(struct.pack('!BB', self.VERSION, config))


class indexparams(object):
    __slots__ = (
        r'fanoutprefix',

        # The struct pack format for fanout table location (i.e. the format that
        # converts the node prefix into an integer location in the fanout
        # table).
        if prefixsize == SMALLFANOUTPREFIX:
            self.fanoutstruct = '!B'
        elif prefixsize == LARGEFANOUTPREFIX:
            self.fanoutstruct = '!H'
        else:
            raise ValueError("invalid fanout prefix size: %s" % prefixsize)

        # The number of fanout table entries
        self.fanoutcount = 2 ** (prefixsize * 8)

        # The total bytes used by the fanout table

43077:687b865b95ad

if pycompat.isposix and not pycompat.ispy3:
    # With glibc 2.7+ the 'e' flag uses O_CLOEXEC when opening.
    # The 'e' flag will be ignored on older versions of glibc.
    # Python 3 can't handle the 'e' flag.
    PACKOPENMODE = b'rbe'
else:
    PACKOPENMODE = b'rb'


class _cachebackedpacks(object):
    def __init__(self, packs, cachesize):
        self._packs = set(packs)

                #
                # If this is an ENOENT error then don't even bother logging.
                # Someone could have removed the file since we retrieved the
                # list of paths.
                if getattr(ex, 'errno', None) != errno.ENOENT:
                    ui.warn(_(b'unable to load pack %s: %s\n') % (filepath, ex))
                continue
            packs.append(pack)

        self.packs = _cachebackedpacks(packs, self.DEFAULTCACHESIZE)

    def getmetrics(self):
        """Returns metrics on the state of this store."""
        size, count = self.gettotalsizeandcount()
        return {
            b'numpacks': count,
            b'totalpacksize': size,
        }

    def getpack(self, path):
        raise NotImplementedError()

        if version in self.SUPPORTED_VERSIONS:
            if self.VERSION is None:
                # only affect this instance
                self.VERSION = version
            elif self.VERSION != version:
                raise RuntimeError(b'inconsistent version: %d' % version)
        else:
            raise RuntimeError(b'unsupported version: %d' % version)


class basepack(versionmixin):
    # The maximum amount we should read via mmap before remmaping so the old
    # pages can be released (100MB)

        self._index = None
        self._data = None
        self.freememory()  # initialize the mmap

        version = struct.unpack(b'!B', self._data[:PACKVERSIONSIZE])[0]
        self._checkversion(version)

        version, config = struct.unpack(b'!BB', self._index[:INDEXVERSIONSIZE])
        self._checkversion(version)

        if 0b10000000 & config:
            self.params = indexparams(LARGEFANOUTPREFIX, version)
        else:

        params = self.params
        rawfanout = self._index[FANOUTSTART : FANOUTSTART + params.fanoutsize]
        fanouttable = []
        for i in pycompat.xrange(0, params.fanoutcount):
            loc = i * 4
            fanoutentry = struct.unpack(b'!I', rawfanout[loc : loc + 4])[0]
            fanouttable.append(fanoutentry)
        return fanouttable

    @util.propertycache
    def _indexend(self):
        nodecount = struct.unpack_from(
            b'!Q', self._index, self.params.indexstart - 8
        )[0]
        return self.params.indexstart + nodecount * self.INDEXENTRYLENGTH

    def freememory(self):
        """Unmap and remap the memory to free it up after known expensive

class mutablebasepack(versionmixin):
    def __init__(self, ui, packdir, version=2):
        self._checkversion(version)
        # TODO(augie): make this configurable
        self._compressor = b'GZ'
        opener = vfsmod.vfs(packdir)
        opener.createmode = 0o444
        self.opener = opener

        self.entries = {}

        shallowutil.mkstickygroupdir(ui, packdir)
        self.packfp, self.packpath = opener.mkstemp(
            suffix=self.PACKSUFFIX + b'-tmp'
        )
        self.idxfp, self.idxpath = opener.mkstemp(
            suffix=self.INDEXSUFFIX + b'-tmp'
        )
        self.packfp = os.fdopen(self.packfp, r'wb+')
        self.idxfp = os.fdopen(self.idxfp, r'wb+')
        self.sha = hashlib.sha1()
        self._closed = False

        opener._fixfilemode(opener.join(self.idxpath))

        # Write header
        # TODO: make it extensible (ex: allow specifying compression algorithm,
        # a flexible key/value header, delta algorithm, fanout size, etc)
        versionbuf = struct.pack(b'!B', self.VERSION)  # unsigned 1 byte int
        self.writeraw(versionbuf)

    def __enter__(self):
        return self

                params.fanoutstruct, node[: params.fanoutprefix]
            )[0]
            if fanouttable[fanoutkey] == EMPTYFANOUT:
                fanouttable[fanoutkey] = location

        rawfanouttable = b''
        last = 0
        for offset in fanouttable:
            offset = offset if offset != EMPTYFANOUT else last
            last = offset
            rawfanouttable += struct.pack(b'!I', offset)

        rawentrieslength = struct.pack(b'!Q', len(self.entries))

        # The index offset is the it's location in the file. So after the 2 byte
        # header and the fanouttable.
        rawindex = self.createindex(locations, 2 + len(rawfanouttable))

        #    <large fanout: 1 bit> # 1 means 2^16, 0 means 2^8
        #    <unused: 7 bit> # future use (compression, delta format, etc)
        config = 0
        if indexparams.fanoutprefix == LARGEFANOUTPREFIX:
            config = 0b10000000
        self.idxfp.write(struct.pack(b'!BB', self.VERSION, config))


class indexparams(object):
    __slots__ = (
        r'fanoutprefix',

        # The struct pack format for fanout table location (i.e. the format that
        # converts the node prefix into an integer location in the fanout
        # table).
        if prefixsize == SMALLFANOUTPREFIX:
            self.fanoutstruct = b'!B'
        elif prefixsize == LARGEFANOUTPREFIX:
            self.fanoutstruct = b'!H'
        else:
            raise ValueError(b"invalid fanout prefix size: %s" % prefixsize)

        # The number of fanout table entries
        self.fanoutcount = 2 ** (prefixsize * 8)

        # The total bytes used by the fanout table