comparison mercurial/upgrade_utils/engine.py @ 46896:cf49e54ef965

upgrade: take advantage of the new information returned by `store.walk`

Before this change the upgrade code had to analyse filename to process them directly. Lets keep that logic private to the store and more to a more robust explicit approach.

Differential Revision: https://phab.mercurial-scm.org/D10316

46895:6085b7f1536d

    metadata,
    pycompat,
    requirements,
    revlog,
    scmutil,
    util,
    vfs as vfsmod,
)
from ..revlogutils import nodemap


def _revlogfrompath(repo, path):
    """Obtain a revlog from a repo path.

    An instance of the appropriate class is returned.
    """
    if path == b'00changelog.i':
        return changelog.changelog(repo.svfs)
    elif path.endswith(b'00manifest.i'):
        mandir = path[: -len(b'00manifest.i')]
        return manifest.manifestrevlog(
            repo.nodeconstants, repo.svfs, tree=mandir
        )
    else:
        # reverse of "/".join(("data", path + ".i"))
        return filelog.filelog(repo.svfs, path[5:-2])


def _copyrevlog(tr, destrepo, oldrl, unencodedname):
    """copy all relevant files for `oldrl` into `destrepo` store

    Files are copied "as is" without any transformation. The copy is performed
    without extra checks. Callers are responsible for making sure the copied
    content is compatible with format of the destination repository.
    """
    oldrl = getattr(oldrl, '_revlog', oldrl)
    newrl = _revlogfrompath(destrepo, unencodedname)
    newrl = getattr(newrl, '_revlog', newrl)

    oldvfs = oldrl.opener
    newvfs = newrl.opener
    oldindex = oldvfs.join(oldrl.indexfile)

    util.copyfile(oldindex, newindex)
    copydata = oldrl.opener.exists(oldrl.datafile)
    if copydata:
        util.copyfile(olddata, newdata)

    if not (
        unencodedname.endswith(b'00changelog.i')
        or unencodedname.endswith(b'00manifest.i')
    ):
        destrepo.svfs.fncache.add(unencodedname)
        if copydata:
            destrepo.svfs.fncache.add(unencodedname[:-2] + b'.d')

    elif requirements.COPIESSDC_REQUIREMENT in removedreqs:
        sidedatacompanion = metadata.getsidedataremover(srcrepo, dstrepo)
    return sidedatacompanion


def matchrevlog(revlogfilter, entry):
    """check if a revlog is selected for cloning.

    In other words, are there any updates which need to be done on revlog
    or it can be blindly copied.

    The store entry is checked against the passed filter"""
    if entry.endswith(b'00changelog.i'):
        return UPGRADE_CHANGELOG in revlogfilter
    elif entry.endswith(b'00manifest.i'):
        return UPGRADE_MANIFEST in revlogfilter
    return UPGRADE_FILELOGS in revlogfilter


def _perform_clone(
    ui,
    dstrepo,
    tr,
    old_revlog,
    unencoded,
    upgrade_op,
    sidedatacompanion,
    oncopiedrevision,
):
    """ returns the new revlog object created"""
    newrl = None
    if matchrevlog(upgrade_op.revlogs_to_process, unencoded):
        ui.note(
            _(b'cloning %d revisions from %s\n') % (len(old_revlog), unencoded)
        )
        newrl = _revlogfrompath(dstrepo, unencoded)
        old_revlog.clone(
            tr,
            newrl,
            addrevisioncb=oncopiedrevision,
            deltareuse=upgrade_op.delta_reuse_mode,

            sidedatacompanion=sidedatacompanion,
        )
    else:
        msg = _(b'blindly copying %s containing %i revisions\n')
        ui.note(msg % (unencoded, len(old_revlog)))
        _copyrevlog(tr, dstrepo, old_revlog, unencoded)

        newrl = _revlogfrompath(dstrepo, unencoded)
    return newrl


def _clonerevlogs(
    ui,

    changelogs = {}
    filelogs = {}

    # Perform a pass to collect metadata. This validates we can open all
    # source files and allows a unified progress bar to be displayed.
    for revlog_type, unencoded, encoded, size in alldatafiles:
        if not unencoded.endswith(b'.i'):
            continue

        rl = _revlogfrompath(srcrepo, unencoded)

        info = rl.storageinfo(
            exclusivefiles=True,
            revisionscount=True,
            trackedsize=True,

        srcsize += datasize
        srcrawsize += rawsize

        # This is for the separate progress bars.
        if isinstance(rl, changelog.changelog):
            changelogs[unencoded] = rl
            crevcount += len(rl)
            csrcsize += datasize
            crawsize += rawsize
        elif isinstance(rl, manifest.manifestrevlog):
            manifests[unencoded] = rl
            mcount += 1
            mrevcount += len(rl)
            msrcsize += datasize
            mrawsize += rawsize
        elif isinstance(rl, filelog.filelog):
            filelogs[unencoded] = rl
            fcount += 1
            frevcount += len(rl)
            fsrcsize += datasize
            frawsize += rawsize
        else:

            util.bytecount(fsrcsize),
            util.bytecount(frawsize),
        )
    )
    progress = srcrepo.ui.makeprogress(_(b'file revisions'), total=frevcount)
    for unencoded, oldrl in sorted(filelogs.items()):
        newrl = _perform_clone(
            ui,
            dstrepo,
            tr,
            oldrl,
            unencoded,
            upgrade_op,
            sidedatacompanion,
            oncopiedrevision,
        )

    if progress:
        progress.complete()
    progress = srcrepo.ui.makeprogress(
        _(b'manifest revisions'), total=mrevcount
    )
    for unencoded, oldrl in sorted(manifests.items()):
        newrl = _perform_clone(
            ui,
            dstrepo,
            tr,
            oldrl,
            unencoded,
            upgrade_op,
            sidedatacompanion,
            oncopiedrevision,
        )

    if progress:
        progress.complete()
    progress = srcrepo.ui.makeprogress(
        _(b'changelog revisions'), total=crevcount
    )
    for unencoded, oldrl in sorted(changelogs.items()):
        newrl = _perform_clone(
            ui,
            dstrepo,
            tr,
            oldrl,
            unencoded,
            upgrade_op,
            sidedatacompanion,
            oncopiedrevision,
        )

46896:cf49e54ef965

    metadata,
    pycompat,
    requirements,
    revlog,
    scmutil,
    store,
    util,
    vfs as vfsmod,
)
from ..revlogutils import nodemap


def _revlogfrompath(repo, rl_type, path):
    """Obtain a revlog from a repo path.

    An instance of the appropriate class is returned.
    """
    if rl_type & store.FILEFLAGS_CHANGELOG:
        return changelog.changelog(repo.svfs)
    elif rl_type & store.FILEFLAGS_MANIFESTLOG:
        mandir = b''
        if b'/' in path:
            mandir = path.rsplit(b'/', 1)[0]
        return manifest.manifestrevlog(
            repo.nodeconstants, repo.svfs, tree=mandir
        )
    else:
        # drop the extension and the `data/` prefix
        path = path.rsplit(b'.', 1)[0].split(b'/', 1)[1]
        return filelog.filelog(repo.svfs, path)


def _copyrevlog(tr, destrepo, oldrl, rl_type, unencodedname):
    """copy all relevant files for `oldrl` into `destrepo` store

    Files are copied "as is" without any transformation. The copy is performed
    without extra checks. Callers are responsible for making sure the copied
    content is compatible with format of the destination repository.
    """
    oldrl = getattr(oldrl, '_revlog', oldrl)
    newrl = _revlogfrompath(destrepo, rl_type, unencodedname)
    newrl = getattr(newrl, '_revlog', newrl)

    oldvfs = oldrl.opener
    newvfs = newrl.opener
    oldindex = oldvfs.join(oldrl.indexfile)

    util.copyfile(oldindex, newindex)
    copydata = oldrl.opener.exists(oldrl.datafile)
    if copydata:
        util.copyfile(olddata, newdata)

    if rl_type & store.FILEFLAGS_FILELOG:
        destrepo.svfs.fncache.add(unencodedname)
        if copydata:
            destrepo.svfs.fncache.add(unencodedname[:-2] + b'.d')

    elif requirements.COPIESSDC_REQUIREMENT in removedreqs:
        sidedatacompanion = metadata.getsidedataremover(srcrepo, dstrepo)
    return sidedatacompanion


def matchrevlog(revlogfilter, rl_type):
    """check if a revlog is selected for cloning.

    In other words, are there any updates which need to be done on revlog
    or it can be blindly copied.

    The store entry is checked against the passed filter"""
    if rl_type & store.FILEFLAGS_CHANGELOG:
        return UPGRADE_CHANGELOG in revlogfilter
    elif rl_type & store.FILEFLAGS_MANIFESTLOG:
        return UPGRADE_MANIFEST in revlogfilter
    assert rl_type & store.FILEFLAGS_FILELOG
    return UPGRADE_FILELOGS in revlogfilter


def _perform_clone(
    ui,
    dstrepo,
    tr,
    old_revlog,
    rl_type,
    unencoded,
    upgrade_op,
    sidedatacompanion,
    oncopiedrevision,
):
    """ returns the new revlog object created"""
    newrl = None
    if matchrevlog(upgrade_op.revlogs_to_process, rl_type):
        ui.note(
            _(b'cloning %d revisions from %s\n') % (len(old_revlog), unencoded)
        )
        newrl = _revlogfrompath(dstrepo, rl_type, unencoded)
        old_revlog.clone(
            tr,
            newrl,
            addrevisioncb=oncopiedrevision,
            deltareuse=upgrade_op.delta_reuse_mode,

            sidedatacompanion=sidedatacompanion,
        )
    else:
        msg = _(b'blindly copying %s containing %i revisions\n')
        ui.note(msg % (unencoded, len(old_revlog)))
        _copyrevlog(tr, dstrepo, old_revlog, rl_type, unencoded)

        newrl = _revlogfrompath(dstrepo, rl_type, unencoded)
    return newrl


def _clonerevlogs(
    ui,

    changelogs = {}
    filelogs = {}

    # Perform a pass to collect metadata. This validates we can open all
    # source files and allows a unified progress bar to be displayed.
    for rl_type, unencoded, encoded, size in alldatafiles:
        if not rl_type & store.FILEFLAGS_REVLOG_MAIN:
            continue

        rl = _revlogfrompath(srcrepo, rl_type, unencoded)

        info = rl.storageinfo(
            exclusivefiles=True,
            revisionscount=True,
            trackedsize=True,

        srcsize += datasize
        srcrawsize += rawsize

        # This is for the separate progress bars.
        if rl_type & store.FILEFLAGS_CHANGELOG:
            changelogs[unencoded] = (rl_type, rl)
            crevcount += len(rl)
            csrcsize += datasize
            crawsize += rawsize
        elif rl_type & store.FILEFLAGS_MANIFESTLOG:
            manifests[unencoded] = (rl_type, rl)
            mcount += 1
            mrevcount += len(rl)
            msrcsize += datasize
            mrawsize += rawsize
        elif rl_type & store.FILEFLAGS_FILELOG:
            filelogs[unencoded] = (rl_type, rl)
            fcount += 1
            frevcount += len(rl)
            fsrcsize += datasize
            frawsize += rawsize
        else:

            util.bytecount(fsrcsize),
            util.bytecount(frawsize),
        )
    )
    progress = srcrepo.ui.makeprogress(_(b'file revisions'), total=frevcount)
    for unencoded, (rl_type, oldrl) in sorted(filelogs.items()):
        newrl = _perform_clone(
            ui,
            dstrepo,
            tr,
            oldrl,
            rl_type,
            unencoded,
            upgrade_op,
            sidedatacompanion,
            oncopiedrevision,
        )

    if progress:
        progress.complete()
    progress = srcrepo.ui.makeprogress(
        _(b'manifest revisions'), total=mrevcount
    )
    for unencoded, (rl_type, oldrl) in sorted(manifests.items()):
        newrl = _perform_clone(
            ui,
            dstrepo,
            tr,
            oldrl,
            rl_type,
            unencoded,
            upgrade_op,
            sidedatacompanion,
            oncopiedrevision,
        )

    if progress:
        progress.complete()
    progress = srcrepo.ui.makeprogress(
        _(b'changelog revisions'), total=crevcount
    )
    for unencoded, (rl_type, oldrl) in sorted(changelogs.items()):
        newrl = _perform_clone(
            ui,
            dstrepo,
            tr,
            oldrl,
            rl_type,
            unencoded,
            upgrade_op,
            sidedatacompanion,
            oncopiedrevision,
        )