view tests/test-fastannotate-revmap.py @ 51977:42a116f1cdc1

branchmap-v3: introduce a "stop_rev" argument to `headsrevs` The `headsrevs` method of the revlog already have a `revs` argument to compute the headrevs of a limited set of heads. However, it disable the use of the native compiled code to compute the heads, which slows down the branchmap v3 code a lot. The branchmap v3 usage is actually quite constrained as we will always only ignores a part at the top of the graph. So we could be significantly faster. We start by making small change to the python side to improve the situation and introduce the new API. More collaboration with the native code are coming later. This massively speedup operation and close most of the remaining gaps between branchmap-v3 and branchmap-v2. especially on repository with many revs like mozilla-try. A small overhead remains mostly because the `headrevs` logic currently has some inefficiently. We will look into them from there. ### benchmark.name = hg.command.unbundle # bin-env-vars.hg.py-re2-module = default # benchmark.variants.issue6528 = disabled # benchmark.variants.resource-usage = default # benchmark.variants.reuse-external-delta-parent = yes # benchmark.variants.revs = any-1-extra-rev # benchmark.variants.source = unbundle # benchmark.variants.validate = default # benchmark.variants.verbosity = quiet ## data-env-vars.name = netbeans-2018-08-01-zstd-sparse-revlog # bin-env-vars.hg.flavor = default branch-v2: 0.233711 ~~~~~ branch-v3 before: 0.368769 (+57.79%, +0.14) branch-v3 after: 0.239857 (+2.63%, +0.01) # bin-env-vars.hg.flavor = rust branch-v2: 0.235230 ~~~~~ branch-v3 before: 0.372460 (+58.34%, +0.14) branch-v3 after: 0.240972 (+2.44%, +0.01) ## data-env-vars.name = netbeans-2018-08-01-ds2-pnm # bin-env-vars.hg.flavor = rust branch-v2: 0.255586 ~~~~~ branch-v3 before: 0.318907 (+24.78%, +0.06) branch-v3 after: 0.268560 (+5.08%, +0.01) ## data-env-vars.name = mozilla-central-2024-03-22-zstd-sparse-revlog # bin-env-vars.hg.flavor = default branch-v2: 0.339010 ~~~~~ branch-v3 before: 0.349752 (+3.17%, +0.01) branch-v3 after: 0.349389 (+3.06%, +0.01) # bin-env-vars.hg.flavor = rust branch-v2: 0.346525 ~~~~~ branch-v3 before: 0.354300 (+2.24%, +0.01) branch-v3 after: 0.355661 (+2.64%, +0.01) ## data-env-vars.name = mozilla-central-2024-03-22-ds2-pnm # bin-env-vars.hg.flavor = rust branch-v2: 0.380202 ~~~~~ branch-v3 before: 0.396293 (+4.23%, +0.02) branch-v3 after: 0.408851 (+7.54%, +0.03) ## data-env-vars.name = mozilla-unified-2024-03-22-zstd-sparse-revlog # bin-env-vars.hg.flavor = default branch-v2: 0.412165 ~~~~~ branch-v3 before: 0.424769 (+3.06%, +0.01) branch-v3 after: 0.427782 (+3.79%, +0.02) # bin-env-vars.hg.flavor = rust branch-v2: 0.412397 ~~~~~ branch-v3 before: 0.421796 (+2.28%, +0.01) branch-v3 after: 0.422354 (+2.41%, +0.01) ## data-env-vars.name = mozilla-unified-2024-03-22-ds2-pnm # bin-env-vars.hg.flavor = rust branch-v2: 0.429501 ~~~~~ branch-v3 before: 0.443849 (+3.34%, +0.01) branch-v3 after: 0.443197 (+3.19%, +0.01) ## data-env-vars.name = mozilla-try-2024-03-26-zstd-sparse-revlog # bin-env-vars.hg.flavor = default branch-v2: 3.403171 ~~~~~ branch-v3 before: 6.234055 (+83.18%, +2.83) branch-v3 after: 3.819477 (+12.23%, +0.42) # bin-env-vars.hg.flavor = rust branch-v2: 3.454876 ~~~~~ branch-v3 before: 6.307813 (+82.58%, +2.85) branch-v3 after: 3.590284 (+3.92%, +0.14) ## data-env-vars.name = mozilla-try-2024-03-26-ds2-pnm # bin-env-vars.hg.flavor = rust branch-v2: 3.465435 ~~~~~ branch-v3 before: 5.176076 (+49.36%, +1.71) branch-v3 after: 3.633278 (+4.84%, +0.17)
author Pierre-Yves David <pierre-yves.david@octobus.net>
date Tue, 03 Sep 2024 11:11:17 +0200
parents 56f98406831b
children
line wrap: on
line source

import os
import tempfile

from mercurial import (
    pycompat,
    util,
)

from hgext.fastannotate import error, revmap


def genhsh(i):
    return pycompat.bytechr(i) + b'\0' * 19


def gettemppath():
    fd, path = tempfile.mkstemp()
    os.close(fd)
    os.unlink(path)
    return path


def ensure(condition):
    if not condition:
        raise RuntimeError('Unexpected')


def testbasicreadwrite():
    path = gettemppath()

    rm = revmap.revmap(path)
    ensure(rm.maxrev == 0)
    for i in range(5):
        ensure(rm.rev2hsh(i) is None)
    ensure(rm.hsh2rev(b'\0' * 20) is None)

    paths = [
        b'',
        b'a',
        None,
        b'b',
        b'b',
        b'c',
        b'c',
        None,
        b'a',
        b'b',
        b'a',
        b'a',
    ]
    for i in range(1, 5):
        ensure(rm.append(genhsh(i), sidebranch=(i & 1), path=paths[i]) == i)

    ensure(rm.maxrev == 4)
    for i in range(1, 5):
        ensure(rm.hsh2rev(genhsh(i)) == i)
        ensure(rm.rev2hsh(i) == genhsh(i))

    # re-load and verify
    rm.flush()
    rm = revmap.revmap(path)
    ensure(rm.maxrev == 4)
    for i in range(1, 5):
        ensure(rm.hsh2rev(genhsh(i)) == i)
        ensure(rm.rev2hsh(i) == genhsh(i))
        ensure(bool(rm.rev2flag(i) & revmap.sidebranchflag) == bool(i & 1))

    # append without calling save() explicitly
    for i in range(5, 12):
        ensure(
            rm.append(genhsh(i), sidebranch=(i & 1), path=paths[i], flush=True)
            == i
        )

    # re-load and verify
    rm = revmap.revmap(path)
    ensure(rm.maxrev == 11)
    for i in range(1, 12):
        ensure(rm.hsh2rev(genhsh(i)) == i)
        ensure(rm.rev2hsh(i) == genhsh(i))
        ensure(rm.rev2path(i) == paths[i] or paths[i - 1])
        ensure(bool(rm.rev2flag(i) & revmap.sidebranchflag) == bool(i & 1))

    os.unlink(path)

    # missing keys
    ensure(rm.rev2hsh(12) is None)
    ensure(rm.rev2hsh(0) is None)
    ensure(rm.rev2hsh(-1) is None)
    ensure(rm.rev2flag(12) is None)
    ensure(rm.rev2path(12) is None)
    ensure(rm.hsh2rev(b'\1' * 20) is None)

    # illformed hash (not 20 bytes)
    try:
        rm.append(b'\0')
        ensure(False)
    except Exception:
        pass


def testcorruptformat():
    path = gettemppath()

    # incorrect header
    with open(path, 'wb') as f:
        f.write(b'NOT A VALID HEADER')
    try:
        revmap.revmap(path)
        ensure(False)
    except error.CorruptedFileError:
        pass

    # rewrite the file
    os.unlink(path)
    rm = revmap.revmap(path)
    rm.append(genhsh(0), flush=True)

    rm = revmap.revmap(path)
    ensure(rm.maxrev == 1)

    # corrupt the file by appending a byte
    size = os.stat(path).st_size
    with open(path, 'ab') as f:
        f.write(b'\xff')
    try:
        revmap.revmap(path)
        ensure(False)
    except error.CorruptedFileError:
        pass

    # corrupt the file by removing the last byte
    ensure(size > 0)
    with open(path, 'wb') as f:
        f.truncate(size - 1)
    try:
        revmap.revmap(path)
        ensure(False)
    except error.CorruptedFileError:
        pass

    os.unlink(path)


def testcopyfrom():
    path = gettemppath()
    rm = revmap.revmap(path)
    for i in range(1, 10):
        ensure(
            rm.append(genhsh(i), sidebranch=(i & 1), path=(b'%d' % (i // 3)))
            == i
        )
    rm.flush()

    # copy rm to rm2
    rm2 = revmap.revmap()
    rm2.copyfrom(rm)
    path2 = gettemppath()
    rm2.path = path2
    rm2.flush()

    # two files should be the same
    ensure(len({util.readfile(p) for p in [path, path2]}) == 1)

    os.unlink(path)
    os.unlink(path2)


class fakefctx:
    def __init__(self, node, path=None):
        self._node = node
        self._path = path

    def node(self):
        return self._node

    def path(self):
        return self._path


def testcontains():
    path = gettemppath()

    rm = revmap.revmap(path)
    for i in range(1, 5):
        ensure(rm.append(genhsh(i), sidebranch=(i & 1)) == i)

    for i in range(1, 5):
        ensure(((genhsh(i), None) in rm) == ((i & 1) == 0))
        ensure((fakefctx(genhsh(i)) in rm) == ((i & 1) == 0))
    for i in range(5, 10):
        ensure(fakefctx(genhsh(i)) not in rm)
        ensure((genhsh(i), None) not in rm)

    # "contains" checks paths
    rm = revmap.revmap()
    for i in range(1, 5):
        ensure(rm.append(genhsh(i), path=(b'%d' % (i // 2))) == i)
    for i in range(1, 5):
        ensure(fakefctx(genhsh(i), path=(b'%d' % (i // 2))) in rm)
        ensure(fakefctx(genhsh(i), path=b'a') not in rm)


def testlastnode():
    path = gettemppath()
    ensure(revmap.getlastnode(path) is None)
    rm = revmap.revmap(path)
    ensure(revmap.getlastnode(path) is None)
    for i in range(1, 10):
        hsh = genhsh(i)
        rm.append(hsh, path=(b'%d' % (i // 2)), flush=True)
        ensure(revmap.getlastnode(path) == hsh)
        rm2 = revmap.revmap(path)
        ensure(rm2.rev2hsh(rm2.maxrev) == hsh)


testbasicreadwrite()
testcorruptformat()
testcopyfrom()
testcontains()
testlastnode()