tests/test-remotefilelog-datapack.py

43075:57875cf423c9

    basepack,
    constants,
    datapack,
)

class datapacktestsbase(object):
    def __init__(self, datapackreader, paramsavailable):
        self.datapackreader = datapackreader
        self.paramsavailable = paramsavailable

    def getHash(self, content):
        return hashlib.sha1(content).digest()

    def getFakeHash(self):
        return b''.join(pycompat.bytechr(random.randint(0, 255))
                        for _ in range(20))

    def createPack(self, revisions=None, packdir=None):
        if revisions is None:
            revisions = [(b"filename", self.getFakeHash(), nullid, b"content")]

        node = self.getHash(content)

        revisions = [(filename, node, nullid, content)]
        pack = self.createPack(revisions)
        if self.paramsavailable:
            self.assertEqual(pack.params.fanoutprefix,
                             basepack.SMALLFANOUTPREFIX)

        chain = pack.getdeltachain(filename, node)
        self.assertEqual(content, chain[0][4])

    def testAddSingle(self):

        revisions = []
        for i in range(100):
            filename = b'%d.txt' % i
            content = b'put-something-here \n' * i
            node = self.getHash(content)
            meta = {constants.METAKEYFLAG: i ** 4,
                    constants.METAKEYSIZE: len(content),
                    b'Z': b'random_string',
                    b'_': b'\0' * i}
            revisions.append((filename, node, nullid, content, meta))
        pack = self.createPack(revisions)
        for name, node, x, content, origmeta in revisions:
            parsedmeta = pack.getmeta(name, node)
            # flag == 0 should be optimized out

        pack = self.createPack(revisions)

        missing = pack.getmissing([(b"foo", revisions[0][1])])
        self.assertFalse(missing)

        missing = pack.getmissing([(b"foo", revisions[0][1]),
                                   (b"foo", revisions[1][1])])
        self.assertFalse(missing)

        fakenode = self.getFakeHash()
        missing = pack.getmissing([(b"foo", revisions[0][1]),
                                   (b"foo", fakenode)])
        self.assertEqual(missing, [(b"foo", fakenode)])

    def testAddThrows(self):
        pack = self.createPack()

            blobs[(filename, node)] = content
            revisions.append((filename, node, nullid, content))

        pack = self.createPack(revisions)
        if self.paramsavailable:
            self.assertEqual(pack.params.fanoutprefix,
                             basepack.LARGEFANOUTPREFIX)

        for (filename, node), content in blobs.items():
            actualcontent = pack.getdeltachain(filename, node)[0][4]
            self.assertEqual(actualcontent, content)

                chain.append(revision)
                revision = (
                    b'%d' % i,
                    self.getFakeHash(),
                    revision[1],
                    self.getFakeHash()
                )

            self.createPack(chain, packdir)
            deltachains.append(chain)

            revision = random.choice(randomchain)
            chain = store.getdeltachain(revision[0], revision[1])

            mostrecentpack = next(iter(store.packs), None)
            self.assertEqual(
                mostrecentpack.getdeltachain(revision[0], revision[1]),
                chain
            )

            self.assertEqual(randomchain.index(revision) + 1, len(chain))

    # perf test off by default since it's slow

            path = self.createPack(revisions).path

            # Perf of large multi-get
            import gc
            gc.disable()
            pack = self.datapackreader(path)
            for lookupsize in lookupsizes:
                if lookupsize > packsize:
                    continue

                findnodes = [(rev[0], rev[1]) for rev in revisions]

                start = time.time()
                pack.getmissing(findnodes[:lookupsize])
                elapsed = time.time() - start
                print ("%s pack %d lookups = %0.04f" %
                       (('%d' % packsize).rjust(7),
                        ('%d' % lookupsize).rjust(7),
                        elapsed))

            print("")
            gc.enable()

        # The perf test is meant to produce output, so we always fail the test
        # so the user sees the output.
        raise RuntimeError("perf test always fails")

class datapacktests(datapacktestsbase, unittest.TestCase):
    def __init__(self, *args, **kwargs):
        datapacktestsbase.__init__(self, datapack.datapack, True)
        unittest.TestCase.__init__(self, *args, **kwargs)

# TODO:
# datapack store:
# - getmissing
# - GC two packs into one

if __name__ == '__main__':
    if pycompat.iswindows:
        sys.exit(80)    # Skip on Windows
    silenttestrunner.main(__name__)

43076:2372284d9457

    basepack,
    constants,
    datapack,
)


class datapacktestsbase(object):
    def __init__(self, datapackreader, paramsavailable):
        self.datapackreader = datapackreader
        self.paramsavailable = paramsavailable

    def getHash(self, content):
        return hashlib.sha1(content).digest()

    def getFakeHash(self):
        return b''.join(
            pycompat.bytechr(random.randint(0, 255)) for _ in range(20)
        )

    def createPack(self, revisions=None, packdir=None):
        if revisions is None:
            revisions = [(b"filename", self.getFakeHash(), nullid, b"content")]

        node = self.getHash(content)

        revisions = [(filename, node, nullid, content)]
        pack = self.createPack(revisions)
        if self.paramsavailable:
            self.assertEqual(
                pack.params.fanoutprefix, basepack.SMALLFANOUTPREFIX
            )

        chain = pack.getdeltachain(filename, node)
        self.assertEqual(content, chain[0][4])

    def testAddSingle(self):

        revisions = []
        for i in range(100):
            filename = b'%d.txt' % i
            content = b'put-something-here \n' * i
            node = self.getHash(content)
            meta = {
                constants.METAKEYFLAG: i ** 4,
                constants.METAKEYSIZE: len(content),
                b'Z': b'random_string',
                b'_': b'\0' * i,
            }
            revisions.append((filename, node, nullid, content, meta))
        pack = self.createPack(revisions)
        for name, node, x, content, origmeta in revisions:
            parsedmeta = pack.getmeta(name, node)
            # flag == 0 should be optimized out

        pack = self.createPack(revisions)

        missing = pack.getmissing([(b"foo", revisions[0][1])])
        self.assertFalse(missing)

        missing = pack.getmissing(
            [(b"foo", revisions[0][1]), (b"foo", revisions[1][1])]
        )
        self.assertFalse(missing)

        fakenode = self.getFakeHash()
        missing = pack.getmissing(
            [(b"foo", revisions[0][1]), (b"foo", fakenode)]
        )
        self.assertEqual(missing, [(b"foo", fakenode)])

    def testAddThrows(self):
        pack = self.createPack()

            blobs[(filename, node)] = content
            revisions.append((filename, node, nullid, content))

        pack = self.createPack(revisions)
        if self.paramsavailable:
            self.assertEqual(
                pack.params.fanoutprefix, basepack.LARGEFANOUTPREFIX
            )

        for (filename, node), content in blobs.items():
            actualcontent = pack.getdeltachain(filename, node)[0][4]
            self.assertEqual(actualcontent, content)

                chain.append(revision)
                revision = (
                    b'%d' % i,
                    self.getFakeHash(),
                    revision[1],
                    self.getFakeHash(),
                )

            self.createPack(chain, packdir)
            deltachains.append(chain)

            revision = random.choice(randomchain)
            chain = store.getdeltachain(revision[0], revision[1])

            mostrecentpack = next(iter(store.packs), None)
            self.assertEqual(
                mostrecentpack.getdeltachain(revision[0], revision[1]), chain
            )

            self.assertEqual(randomchain.index(revision) + 1, len(chain))

    # perf test off by default since it's slow

            path = self.createPack(revisions).path

            # Perf of large multi-get
            import gc

            gc.disable()
            pack = self.datapackreader(path)
            for lookupsize in lookupsizes:
                if lookupsize > packsize:
                    continue

                findnodes = [(rev[0], rev[1]) for rev in revisions]

                start = time.time()
                pack.getmissing(findnodes[:lookupsize])
                elapsed = time.time() - start
                print(
                    "%s pack %d lookups = %0.04f"
                    % (
                        ('%d' % packsize).rjust(7),
                        ('%d' % lookupsize).rjust(7),
                        elapsed,
                    )
                )

            print("")
            gc.enable()

        # The perf test is meant to produce output, so we always fail the test
        # so the user sees the output.
        raise RuntimeError("perf test always fails")


class datapacktests(datapacktestsbase, unittest.TestCase):
    def __init__(self, *args, **kwargs):
        datapacktestsbase.__init__(self, datapack.datapack, True)
        unittest.TestCase.__init__(self, *args, **kwargs)


# TODO:
# datapack store:
# - getmissing
# - GC two packs into one

if __name__ == '__main__':
    if pycompat.iswindows:
        sys.exit(80)  # Skip on Windows
    silenttestrunner.main(__name__)