comparison hgext/remotefilelog/datapack.py @ 43076:2372284d9457

formatting: blacken the codebase

This is using my patch to black
(https://github.com/psf/black/pull/826) so we don't un-wrap collection
literals.

Done with:

  hg files 'set:**.py - mercurial/thirdparty/** - "contrib/python-zstandard/**"' | xargs black -S

# skip-blame mass-reformatting only

# no-check-commit reformats foo_bar functions

Differential Revision: https://phab.mercurial-scm.org/D6971

43075:57875cf423c9

NOBASEINDEXMARK = -2

INDEXSUFFIX = '.dataidx'
PACKSUFFIX = '.datapack'

class datapackstore(basepack.basepackstore):
    INDEXSUFFIX = INDEXSUFFIX
    PACKSUFFIX = PACKSUFFIX

    def __init__(self, ui, path):

        raise KeyError((name, hex(node)))

    def add(self, name, node, data):
        raise RuntimeError("cannot add to datapackstore")

class datapack(basepack.basepack):
    INDEXSUFFIX = INDEXSUFFIX
    PACKSUFFIX = PACKSUFFIX

                missing.append((name, node))

        return missing

    def get(self, name, node):
        raise RuntimeError("must use getdeltachain with datapack (%s:%s)"
                           % (name, hex(node)))

    def getmeta(self, name, node):
        value = self._find(node)
        if value is None:
            raise KeyError((name, hex(node)))

        node, deltabaseoffset, offset, size = value
        rawentry = self._data[offset:offset + size]

        # see docstring of mutabledatapack for the format
        offset = 0
        offset += struct.unpack_from('!H', rawentry, offset)[0] + 2 # filename
        offset += 40 # node, deltabase node
        offset += struct.unpack_from('!Q', rawentry, offset)[0] + 8 # delta

        metalen = struct.unpack_from('!I', rawentry, offset)[0]
        offset += 4

        meta = shallowutil.parsepackmeta(rawentry[offset:offset + metalen])

        return meta

    def getdelta(self, name, node):
        value = self._find(node)

        # Precompute chains
        chain = [value]
        deltabaseoffset = value[1]
        entrylen = self.INDEXENTRYLENGTH
        while (deltabaseoffset != FULLTEXTINDEXMARK
               and deltabaseoffset != NOBASEINDEXMARK):
            loc = params.indexstart + deltabaseoffset
            value = struct.unpack(self.INDEXFORMAT,
                                  self._index[loc:loc + entrylen])
            deltabaseoffset = value[1]
            chain.append(value)

        # Read chain data
        deltachain = []

        self.freememory()

        return deltachain

    def _readentry(self, offset, size, getmeta=False):
        rawentry = self._data[offset:offset + size]
        self._pagedin += len(rawentry)

        # <2 byte len> + <filename>
        lengthsize = 2
        filenamelen = struct.unpack('!H', rawentry[:2])[0]
        filename = rawentry[lengthsize:lengthsize + filenamelen]

        # <20 byte node> + <20 byte deltabase>
        nodestart = lengthsize + filenamelen
        deltabasestart = nodestart + NODELENGTH
        node = rawentry[nodestart:deltabasestart]
        deltabasenode = rawentry[deltabasestart:deltabasestart + NODELENGTH]

        # <8 byte len> + <delta>
        deltastart = deltabasestart + NODELENGTH
        rawdeltalen = rawentry[deltastart:deltastart + 8]
        deltalen = struct.unpack('!Q', rawdeltalen)[0]

        delta = rawentry[deltastart + 8:deltastart + 8 + deltalen]
        delta = self._decompress(delta)

        if getmeta:
            metastart = deltastart + 8 + deltalen
            metalen = struct.unpack_from('!I', rawentry, metastart)[0]

            rawmeta = rawentry[metastart + 4:metastart + 4 + metalen]
            meta = shallowutil.parsepackmeta(rawmeta)
            return filename, node, deltabasenode, delta, meta
        else:
            return filename, node, deltabasenode, delta

    def add(self, name, node, data):
        raise RuntimeError("cannot add to datapack (%s:%s)" % (name, node))

    def _find(self, node):
        params = self.params
        fanoutkey = struct.unpack(params.fanoutstruct,
                                  node[:params.fanoutprefix])[0]
        fanout = self._fanouttable

        start = fanout[fanoutkey] + params.indexstart
        indexend = self._indexend

        else:
            end = indexend

        # Bisect between start and end to find node
        index = self._index
        startnode = index[start:start + NODELENGTH]
        endnode = index[end:end + NODELENGTH]
        entrylen = self.INDEXENTRYLENGTH
        if startnode == node:
            entry = index[start:start + entrylen]
        elif endnode == node:
            entry = index[end:end + entrylen]
        else:
            while start < end - entrylen:
                mid = start + (end - start) // 2
                mid = mid - ((mid - params.indexstart) % entrylen)
                midnode = index[mid:mid + NODELENGTH]
                if midnode == node:
                    entry = index[mid:mid + entrylen]
                    break
                if node > midnode:
                    start = mid
                elif node < midnode:
                    end = mid

            ledger.markdataentry(self, filename, node)

    def cleanup(self, ledger):
        entries = ledger.sources.get(self, [])
        allkeys = set(self)
        repackedkeys = set((e.filename, e.node) for e in entries if
                           e.datarepacked or e.gced)

        if len(allkeys - repackedkeys) == 0:
            if self.path not in ledger.created:
                util.unlinkpath(self.indexpath, ignoremissing=True)
                util.unlinkpath(self.packpath, ignoremissing=True)

        data = self._data
        while offset < self.datasize:
            oldoffset = offset

            # <2 byte len> + <filename>
            filenamelen = struct.unpack('!H', data[offset:offset + 2])[0]
            offset += 2
            filename = data[offset:offset + filenamelen]
            offset += filenamelen

            # <20 byte node>
            node = data[offset:offset + constants.NODESIZE]
            offset += constants.NODESIZE
            # <20 byte deltabase>
            deltabase = data[offset:offset + constants.NODESIZE]
            offset += constants.NODESIZE

            # <8 byte len> + <delta>
            rawdeltalen = data[offset:offset + 8]
            deltalen = struct.unpack('!Q', rawdeltalen)[0]
            offset += 8

            # TODO(augie): we should store a header that is the
            # uncompressed size.
            uncompressedlen = len(self._decompress(
                data[offset:offset + deltalen]))
            offset += deltalen

            # <4 byte len> + <metadata-list>
            metalen = struct.unpack_from('!I', data, offset)[0]
            offset += 4 + metalen

            # If we've read a lot of data from the mmap, free some memory.
            self._pagedin += offset - oldoffset
            if self.freememory():
                data = self._data

class mutabledatapack(basepack.mutablebasepack):
    """A class for constructing and serializing a datapack file and index.

    A datapack is a pair of files that contain the revision contents for various

                     <pack entry offset: 8 byte unsigned int>
                     <pack entry size: 8 byte unsigned int>

    [1]: new in version 1.
    """
    INDEXSUFFIX = INDEXSUFFIX
    PACKSUFFIX = PACKSUFFIX

    # v[01] index format: <node><delta offset><pack data offset><pack data size>
    INDEXFORMAT = datapack.INDEXFORMAT

    def _compress(self, data):
        return zlib.compress(data)

    def add(self, name, node, deltabasenode, delta, metadata=None):
        # metadata is a dict, ex. {METAKEYFLAG: flag}
        if len(name) > 2**16:
            raise RuntimeError(_("name too long %s") % name)
        if len(node) != 20:
            raise RuntimeError(_("node should be 20 bytes %s") % node)

        if node in self.entries:

            return

        # TODO: allow configurable compression
        delta = self._compress(delta)

        rawdata = ''.join((
            struct.pack('!H', len(name)), # unsigned 2 byte int
            name,
            node,
            deltabasenode,
            struct.pack('!Q', len(delta)), # unsigned 8 byte int
            delta,
        ))

        # v1 support metadata
        rawmeta = shallowutil.buildpackmeta(metadata)
        rawdata += struct.pack('!I', len(rawmeta)) # unsigned 4 byte
        rawdata += rawmeta

        offset = self.packfp.tell()

        size = len(rawdata)

        self.entries[node] = (deltabasenode, offset, size)

        self.writeraw(rawdata)

    def createindex(self, nodelocations, indexoffset):
        entries = sorted((n, db, o, s) for n, (db, o, s)
                         in self.entries.iteritems())

        rawindex = ''
        fmt = self.INDEXFORMAT
        for node, deltabase, offset, size in entries:
            if deltabase == nullid:
                deltabaselocation = FULLTEXTINDEXMARK
            else:
                # Instead of storing the deltabase node in the index, let's
                # store a pointer directly to the index entry for the deltabase.
                deltabaselocation = nodelocations.get(deltabase,
                                                      NOBASEINDEXMARK)

            entry = struct.pack(fmt, node, deltabaselocation, offset, size)
            rawindex += entry

        return rawindex

43076:2372284d9457

NOBASEINDEXMARK = -2

INDEXSUFFIX = '.dataidx'
PACKSUFFIX = '.datapack'


class datapackstore(basepack.basepackstore):
    INDEXSUFFIX = INDEXSUFFIX
    PACKSUFFIX = PACKSUFFIX

    def __init__(self, ui, path):

        raise KeyError((name, hex(node)))

    def add(self, name, node, data):
        raise RuntimeError("cannot add to datapackstore")


class datapack(basepack.basepack):
    INDEXSUFFIX = INDEXSUFFIX
    PACKSUFFIX = PACKSUFFIX

                missing.append((name, node))

        return missing

    def get(self, name, node):
        raise RuntimeError(
            "must use getdeltachain with datapack (%s:%s)" % (name, hex(node))
        )

    def getmeta(self, name, node):
        value = self._find(node)
        if value is None:
            raise KeyError((name, hex(node)))

        node, deltabaseoffset, offset, size = value
        rawentry = self._data[offset : offset + size]

        # see docstring of mutabledatapack for the format
        offset = 0
        offset += struct.unpack_from('!H', rawentry, offset)[0] + 2  # filename
        offset += 40  # node, deltabase node
        offset += struct.unpack_from('!Q', rawentry, offset)[0] + 8  # delta

        metalen = struct.unpack_from('!I', rawentry, offset)[0]
        offset += 4

        meta = shallowutil.parsepackmeta(rawentry[offset : offset + metalen])

        return meta

    def getdelta(self, name, node):
        value = self._find(node)

        # Precompute chains
        chain = [value]
        deltabaseoffset = value[1]
        entrylen = self.INDEXENTRYLENGTH
        while (
            deltabaseoffset != FULLTEXTINDEXMARK
            and deltabaseoffset != NOBASEINDEXMARK
        ):
            loc = params.indexstart + deltabaseoffset
            value = struct.unpack(
                self.INDEXFORMAT, self._index[loc : loc + entrylen]
            )
            deltabaseoffset = value[1]
            chain.append(value)

        # Read chain data
        deltachain = []

        self.freememory()

        return deltachain

    def _readentry(self, offset, size, getmeta=False):
        rawentry = self._data[offset : offset + size]
        self._pagedin += len(rawentry)

        # <2 byte len> + <filename>
        lengthsize = 2
        filenamelen = struct.unpack('!H', rawentry[:2])[0]
        filename = rawentry[lengthsize : lengthsize + filenamelen]

        # <20 byte node> + <20 byte deltabase>
        nodestart = lengthsize + filenamelen
        deltabasestart = nodestart + NODELENGTH
        node = rawentry[nodestart:deltabasestart]
        deltabasenode = rawentry[deltabasestart : deltabasestart + NODELENGTH]

        # <8 byte len> + <delta>
        deltastart = deltabasestart + NODELENGTH
        rawdeltalen = rawentry[deltastart : deltastart + 8]
        deltalen = struct.unpack('!Q', rawdeltalen)[0]

        delta = rawentry[deltastart + 8 : deltastart + 8 + deltalen]
        delta = self._decompress(delta)

        if getmeta:
            metastart = deltastart + 8 + deltalen
            metalen = struct.unpack_from('!I', rawentry, metastart)[0]

            rawmeta = rawentry[metastart + 4 : metastart + 4 + metalen]
            meta = shallowutil.parsepackmeta(rawmeta)
            return filename, node, deltabasenode, delta, meta
        else:
            return filename, node, deltabasenode, delta

    def add(self, name, node, data):
        raise RuntimeError("cannot add to datapack (%s:%s)" % (name, node))

    def _find(self, node):
        params = self.params
        fanoutkey = struct.unpack(
            params.fanoutstruct, node[: params.fanoutprefix]
        )[0]
        fanout = self._fanouttable

        start = fanout[fanoutkey] + params.indexstart
        indexend = self._indexend

        else:
            end = indexend

        # Bisect between start and end to find node
        index = self._index
        startnode = index[start : start + NODELENGTH]
        endnode = index[end : end + NODELENGTH]
        entrylen = self.INDEXENTRYLENGTH
        if startnode == node:
            entry = index[start : start + entrylen]
        elif endnode == node:
            entry = index[end : end + entrylen]
        else:
            while start < end - entrylen:
                mid = start + (end - start) // 2
                mid = mid - ((mid - params.indexstart) % entrylen)
                midnode = index[mid : mid + NODELENGTH]
                if midnode == node:
                    entry = index[mid : mid + entrylen]
                    break
                if node > midnode:
                    start = mid
                elif node < midnode:
                    end = mid

            ledger.markdataentry(self, filename, node)

    def cleanup(self, ledger):
        entries = ledger.sources.get(self, [])
        allkeys = set(self)
        repackedkeys = set(
            (e.filename, e.node) for e in entries if e.datarepacked or e.gced
        )

        if len(allkeys - repackedkeys) == 0:
            if self.path not in ledger.created:
                util.unlinkpath(self.indexpath, ignoremissing=True)
                util.unlinkpath(self.packpath, ignoremissing=True)

        data = self._data
        while offset < self.datasize:
            oldoffset = offset

            # <2 byte len> + <filename>
            filenamelen = struct.unpack('!H', data[offset : offset + 2])[0]
            offset += 2
            filename = data[offset : offset + filenamelen]
            offset += filenamelen

            # <20 byte node>
            node = data[offset : offset + constants.NODESIZE]
            offset += constants.NODESIZE
            # <20 byte deltabase>
            deltabase = data[offset : offset + constants.NODESIZE]
            offset += constants.NODESIZE

            # <8 byte len> + <delta>
            rawdeltalen = data[offset : offset + 8]
            deltalen = struct.unpack('!Q', rawdeltalen)[0]
            offset += 8

            # TODO(augie): we should store a header that is the
            # uncompressed size.
            uncompressedlen = len(
                self._decompress(data[offset : offset + deltalen])
            )
            offset += deltalen

            # <4 byte len> + <metadata-list>
            metalen = struct.unpack_from('!I', data, offset)[0]
            offset += 4 + metalen

            # If we've read a lot of data from the mmap, free some memory.
            self._pagedin += offset - oldoffset
            if self.freememory():
                data = self._data


class mutabledatapack(basepack.mutablebasepack):
    """A class for constructing and serializing a datapack file and index.

    A datapack is a pair of files that contain the revision contents for various

                     <pack entry offset: 8 byte unsigned int>
                     <pack entry size: 8 byte unsigned int>

    [1]: new in version 1.
    """

    INDEXSUFFIX = INDEXSUFFIX
    PACKSUFFIX = PACKSUFFIX

    # v[01] index format: <node><delta offset><pack data offset><pack data size>
    INDEXFORMAT = datapack.INDEXFORMAT

    def _compress(self, data):
        return zlib.compress(data)

    def add(self, name, node, deltabasenode, delta, metadata=None):
        # metadata is a dict, ex. {METAKEYFLAG: flag}
        if len(name) > 2 ** 16:
            raise RuntimeError(_("name too long %s") % name)
        if len(node) != 20:
            raise RuntimeError(_("node should be 20 bytes %s") % node)

        if node in self.entries:

            return

        # TODO: allow configurable compression
        delta = self._compress(delta)

        rawdata = ''.join(
            (
                struct.pack('!H', len(name)),  # unsigned 2 byte int
                name,
                node,
                deltabasenode,
                struct.pack('!Q', len(delta)),  # unsigned 8 byte int
                delta,
            )
        )

        # v1 support metadata
        rawmeta = shallowutil.buildpackmeta(metadata)
        rawdata += struct.pack('!I', len(rawmeta))  # unsigned 4 byte
        rawdata += rawmeta

        offset = self.packfp.tell()

        size = len(rawdata)

        self.entries[node] = (deltabasenode, offset, size)

        self.writeraw(rawdata)

    def createindex(self, nodelocations, indexoffset):
        entries = sorted(
            (n, db, o, s) for n, (db, o, s) in self.entries.iteritems()
        )

        rawindex = ''
        fmt = self.INDEXFORMAT
        for node, deltabase, offset, size in entries:
            if deltabase == nullid:
                deltabaselocation = FULLTEXTINDEXMARK
            else:
                # Instead of storing the deltabase node in the index, let's
                # store a pointer directly to the index entry for the deltabase.
                deltabaselocation = nodelocations.get(
                    deltabase, NOBASEINDEXMARK
                )

            entry = struct.pack(fmt, node, deltabaselocation, offset, size)
            rawindex += entry

        return rawindex