Mercurial > hg-stable
changeset 25325:fcd2f9b06629
largefiles: use the convert extension for 'lfconvert --to-normal'
The logic in the convert extension is more advanced, supporting extra features
like converting revision IDs in 'extras' (e.g. 'amend_source'), supports
updating hashes in commit messages, and outputs an SHA map file. Rather than
try to duplicate all of that, just use the existing code.
Even though the convert extension supports user supplied options like filemap,
etc, those features aren't available on the lfconvert interface. Therefore, it
is safe to use the filemap mechanism (in memory) to handle the standin -> file
rename. The convert extension handles the destination locking for this path.
There was a comment in test-lfconvert.t about the hash on rev 5 being different
because it was doing a better job than "hg remove" + "hg merge" + "hg commit".
It isn't clear to me what was happening or why, but now the hashes match the
original repo exactly after a roundtrip, which seems like a good idea. If there
really was something beneficial about the previous behavior, perhaps merge can
be changed so that everyone benefits.
Converting to a largefiles repo still uses the original (limited) lfconvert
logic.
| author | Matt Harbison <matt_harbison@yahoo.com> |
|---|---|
| date | Thu, 28 May 2015 13:34:37 -0400 |
| parents | deed9c6b12d2 |
| children | 238e5cd94bbc |
| files | hgext/largefiles/lfcommands.py tests/test-lfconvert.t |
| diffstat | 2 files changed, 107 insertions(+), 35 deletions(-) [+] |
line wrap: on
line diff
--- a/hgext/largefiles/lfcommands.py Wed May 27 00:22:29 2015 -0700 +++ b/hgext/largefiles/lfcommands.py Thu May 28 13:34:37 2015 -0400 @@ -16,6 +16,9 @@ from mercurial.i18n import _ from mercurial.lock import release +from hgext.convert import convcmd +from hgext.convert import filemap + import lfutil import basestore @@ -70,12 +73,6 @@ success = False dstwlock = dstlock = None try: - # Lock destination to prevent modification while it is converted to. - # Don't need to lock src because we are just reading from its history - # which can't change. - dstwlock = rdst.wlock() - dstlock = rdst.lock() - # Get a list of all changesets in the source. The easy way to do this # is to simply walk the changelog, using changelog.nodesbetween(). # Take a look at mercurial/revlog.py:639 for more details. @@ -84,6 +81,12 @@ rsrc.heads())[0]) revmap = {node.nullid: node.nullid} if tolfile: + # Lock destination to prevent modification while it is converted to. + # Don't need to lock src because we are just reading from its + # history which can't change. + dstwlock = rdst.wlock() + dstlock = rdst.lock() + lfiles = set() normalfiles = set() if not pats: @@ -118,16 +121,51 @@ rdst.requirements.add('largefiles') rdst._writerequirements() else: - for ctx in ctxs: - ui.progress(_('converting revisions'), ctx.rev(), - unit=_('revision'), total=rsrc['tip'].rev()) - _addchangeset(ui, rsrc, rdst, ctx, revmap) + class lfsource(filemap.filemap_source): + def __init__(self, ui, source): + super(lfsource, self).__init__(ui, source, None) + self.filemapper.rename[lfutil.shortname] = '.' + + def getfile(self, name, rev): + realname, realrev = rev + f = super(lfsource, self).getfile(name, rev) + + if (not realname.startswith(lfutil.shortnameslash) + or f[0] is None): + return f + + # Substitute in the largefile data for the hash + hash = f[0].strip() + path = lfutil.findfile(rsrc, hash) - ui.progress(_('converting revisions'), None) + if path is None: + raise util.Abort(_("missing largefile for \'%s\' in %s") + % (realname, realrev)) + fp = open(path, 'rb') + + try: + return (fp.read(), f[1]) + finally: + fp.close() + + class converter(convcmd.converter): + def __init__(self, ui, source, dest, revmapfile, opts): + src = lfsource(ui, source) + + super(converter, self).__init__(ui, src, dest, revmapfile, + opts) + + found, missing = downloadlfiles(ui, rsrc) + if missing != 0: + raise util.Abort(_("all largefiles must be present locally")) + + convcmd.converter = converter + convcmd.convert(ui, src, dest) success = True finally: - rdst.dirstate.clear() - release(dstlock, dstwlock) + if tolfile: + rdst.dirstate.clear() + release(dstlock, dstwlock) if not success: # we failed, remove the new directory shutil.rmtree(rdst.root)
--- a/tests/test-lfconvert.t Wed May 27 00:22:29 2015 -0700 +++ b/tests/test-lfconvert.t Thu May 28 13:34:37 2015 -0400 @@ -226,6 +226,7 @@ $ hg commit -m "add anotherlarge (should be a largefile)" $ cat .hglf/anotherlarge 3b71f43ff30f4b15b5cd85dd9e95ebc7e84eb5a3 + $ hg tag mytag $ cd .. round-trip: converting back to a normal (non-largefiles) repo with @@ -233,25 +234,30 @@ $ cd largefiles-repo $ hg lfconvert --to-normal . ../normal-repo initializing destination ../normal-repo + 0 additional largefiles cached + scanning source... + sorting... + converting... + 7 add large, normal1 + 6 add sub/* + 5 rename sub/ to stuff/ + 4 add normal3, modify sub/* + 3 remove large, normal3 + 2 merge + 1 add anotherlarge (should be a largefile) + 0 Added tag mytag for changeset abacddda7028 $ cd ../normal-repo $ cat >> .hg/hgrc <<EOF > [extensions] > largefiles = ! > EOF -# Hmmm: the changeset ID for rev 5 is different from the original -# normal repo (../bigfile-repo), because the changelog filelist -# differs between the two incarnations of rev 5: this repo includes -# 'large' in the list, but ../bigfile-repo does not. Since rev 5 -# removes 'large' relative to the first parent in both repos, it seems -# to me that lfconvert is doing a *better* job than -# "hg remove" + "hg merge" + "hg commit". -# $ hg -R ../bigfile-repo debugdata -c 5 -# $ hg debugdata -c 5 $ hg log -G --template "{rev}:{node|short} {desc|firstline}\n" - o 6:1635824e6f59 add anotherlarge (should be a largefile) + o 7:b5fedc110b9d Added tag mytag for changeset 867ab992ecf4 | - o 5:7215f8deeaaf merge + o 6:867ab992ecf4 add anotherlarge (should be a largefile) + | + o 5:4884f215abda merge |\ | o 4:7285f817b77e remove large, normal3 | | @@ -264,8 +270,9 @@ o 0:117b8328f97a add large, normal1 $ hg update - 4 files updated, 0 files merged, 0 files removed, 0 files unresolved + 5 files updated, 0 files merged, 0 files removed, 0 files unresolved $ hg locate + .hgtags anotherlarge normal1 stuff/maybelarge.dat @@ -284,15 +291,18 @@ scanning source... sorting... converting... - 6 add large, normal1 - 5 add sub/* - 4 rename sub/ to stuff/ - 3 add normal3, modify sub/* - 2 remove large, normal3 - 1 merge - 0 add anotherlarge (should be a largefile) + 7 add large, normal1 + 6 add sub/* + 5 rename sub/ to stuff/ + 4 add normal3, modify sub/* + 3 remove large, normal3 + 2 merge + 1 add anotherlarge (should be a largefile) + 0 Added tag mytag for changeset abacddda7028 $ hg -R largefiles-repo-hg log -G --template "{rev}:{node|short} {desc|firstline}\n" + o 7:2f08f66459b7 Added tag mytag for changeset 17126745edfd + | o 6:17126745edfd add anotherlarge (should be a largefile) | o 5:9cc5aa7204f0 merge @@ -315,8 +325,8 @@ checking manifests crosschecking files in changesets and manifests checking files - 8 files, 7 changesets, 12 total revisions - searching 7 changesets for largefiles + 9 files, 8 changesets, 13 total revisions + searching 8 changesets for largefiles changeset 0:d4892ec57ce2: large references missing $TESTTMP/largefiles-repo-hg/.hg/largefiles/2e000fa7e85759c7f4c254d4d9c33ef481e459a7 (glob) changeset 1:334e5237836d: sub/maybelarge.dat references missing $TESTTMP/largefiles-repo-hg/.hg/largefiles/34e163be8e43c5631d8b92e9c43ab0bf0fa62b9c (glob) changeset 2:261ad3f3f037: stuff/maybelarge.dat references missing $TESTTMP/largefiles-repo-hg/.hg/largefiles/34e163be8e43c5631d8b92e9c43ab0bf0fa62b9c (glob) @@ -336,6 +346,18 @@ $ rm -f "${USERCACHE}"/* $ hg lfconvert --to-normal issue3519 normalized3519 initializing destination normalized3519 + 4 additional largefiles cached + scanning source... + sorting... + converting... + 7 add large, normal1 + 6 add sub/* + 5 rename sub/ to stuff/ + 4 add normal3, modify sub/* + 3 remove large, normal3 + 2 merge + 1 add anotherlarge (should be a largefile) + 0 Added tag mytag for changeset abacddda7028 Ensure the abort message is useful if a largefile is entirely unavailable $ rm -rf normalized3519 @@ -344,8 +366,20 @@ $ rm largefiles-repo/.hg/largefiles/* $ hg lfconvert --to-normal issue3519 normalized3519 initializing destination normalized3519 + anotherlarge: largefile 3b71f43ff30f4b15b5cd85dd9e95ebc7e84eb5a3 not available from file:/*/$TESTTMP/largefiles-repo (glob) + stuff/maybelarge.dat: largefile 76236b6a2c6102826c61af4297dd738fb3b1de38 not available from file:/*/$TESTTMP/largefiles-repo (glob) + stuff/maybelarge.dat: largefile 76236b6a2c6102826c61af4297dd738fb3b1de38 not available from file:/*/$TESTTMP/largefiles-repo (glob) + sub/maybelarge.dat: largefile 76236b6a2c6102826c61af4297dd738fb3b1de38 not available from file:/*/$TESTTMP/largefiles-repo (glob) large: largefile 2e000fa7e85759c7f4c254d4d9c33ef481e459a7 not available from file:/*/$TESTTMP/largefiles-repo (glob) - abort: missing largefile 'large' from revision d4892ec57ce212905215fad1d9018f56b99202ad + sub/maybelarge.dat: largefile 76236b6a2c6102826c61af4297dd738fb3b1de38 not available from file:/*/$TESTTMP/largefiles-repo (glob) + large: largefile 2e000fa7e85759c7f4c254d4d9c33ef481e459a7 not available from file:/*/$TESTTMP/largefiles-repo (glob) + stuff/maybelarge.dat: largefile 34e163be8e43c5631d8b92e9c43ab0bf0fa62b9c not available from file:/*/$TESTTMP/largefiles-repo (glob) + large: largefile 2e000fa7e85759c7f4c254d4d9c33ef481e459a7 not available from file:/*/$TESTTMP/largefiles-repo (glob) + sub/maybelarge.dat: largefile 34e163be8e43c5631d8b92e9c43ab0bf0fa62b9c not available from file:/*/$TESTTMP/largefiles-repo (glob) + large: largefile 2e000fa7e85759c7f4c254d4d9c33ef481e459a7 not available from file:/*/$TESTTMP/largefiles-repo (glob) + 0 additional largefiles cached + 11 largefiles failed to download + abort: all largefiles must be present locally [255]