Mercurial > hg
changeset 18641:6204e4d4dd6d
Merge crew and main.
| author | Augie Fackler <raf@durin42.com> |
|---|---|
| date | Sun, 10 Feb 2013 04:04:22 -0600 |
| parents | 0027a5cec9d0 (current diff) a8648f32b8ed (diff) |
| children | a40d608e2a7b |
| files | |
| diffstat | 13 files changed, 238 insertions(+), 69 deletions(-) [+] |
line wrap: on
line diff
--- a/mercurial/help/config.txt Sat Feb 09 22:54:34 2013 +0000 +++ b/mercurial/help/config.txt Sun Feb 10 04:04:22 2013 -0600 @@ -1463,3 +1463,15 @@ ``templates`` Where to find the HTML templates. Default is install path. + +``worker`` +---------- + +Parallel master/worker configuration. We currently perform working +directory updates in parallel on Unix-like systems, which greatly +helps performance. + +``numcpus`` + Number of CPUs to use for parallel operations. Default is 4 or the + number of CPUs on the system, whichever is larger. A zero or + negative value is treated as ``use the default``.
--- a/mercurial/merge.py Sat Feb 09 22:54:34 2013 +0000 +++ b/mercurial/merge.py Sun Feb 10 04:04:22 2013 -0600 @@ -7,7 +7,7 @@ from node import nullid, nullrev, hex, bin from i18n import _ -import error, util, filemerge, copies, subrepo +import error, util, filemerge, copies, subrepo, worker import errno, os, shutil class mergestate(object): @@ -342,6 +342,42 @@ def actionkey(a): return a[1] == "r" and -1 or 0, a +def getremove(repo, mctx, overwrite, args): + """apply usually-non-interactive updates to the working directory + + mctx is the context to be merged into the working copy + + yields tuples for progress updates + """ + verbose = repo.ui.verbose + unlink = util.unlinkpath + wjoin = repo.wjoin + fctx = mctx.filectx + wwrite = repo.wwrite + audit = repo.wopener.audit + i = 0 + for arg in args: + f = arg[0] + if arg[1] == 'r': + if verbose: + repo.ui.note(_("removing %s\n") % f) + audit(f) + try: + unlink(wjoin(f), ignoremissing=True) + except OSError, inst: + repo.ui.warn(_("update failed to remove %s: %s!\n") % + (f, inst.strerror)) + else: + if verbose: + repo.ui.note(_("getting %s\n") % f) + wwrite(f, fctx(f).data(), arg[2][0]) + if i == 100: + yield i, f + i = 0 + i += 1 + if i > 0: + yield i, f + def applyupdates(repo, actions, wctx, mctx, actx, overwrite): """apply the merge action list to the working directory @@ -393,22 +429,34 @@ util.unlinkpath(repo.wjoin(f)) numupdates = len(actions) + workeractions = [a for a in actions if a[1] in 'gr'] + updated = len([a for a in workeractions if a[1] == 'g']) + removed = len([a for a in workeractions if a[1] == 'r']) + actions = [a for a in actions if a[1] not in 'gr'] + + hgsub = [a[1] for a in workeractions if a[0] == '.hgsubstate'] + if hgsub and hgsub[0] == 'r': + subrepo.submerge(repo, wctx, mctx, wctx, overwrite) + + z = 0 + prog = worker.worker(repo.ui, 0.001, getremove, (repo, mctx, overwrite), + workeractions) + for i, item in prog: + z += i + repo.ui.progress(_('updating'), z, item=item, total=numupdates, + unit=_('files')) + + if hgsub and hgsub[0] == 'g': + subrepo.submerge(repo, wctx, mctx, wctx, overwrite) + + _updating = _('updating') + _files = _('files') + progress = repo.ui.progress + for i, a in enumerate(actions): f, m, args, msg = a - repo.ui.progress(_('updating'), i + 1, item=f, total=numupdates, - unit=_('files')) - if m == "r": # remove - repo.ui.note(_("removing %s\n") % f) - audit(f) - if f == '.hgsubstate': # subrepo states need updating - subrepo.submerge(repo, wctx, mctx, wctx, overwrite) - try: - util.unlinkpath(repo.wjoin(f), ignoremissing=True) - except OSError, inst: - repo.ui.warn(_("update failed to remove %s: %s!\n") % - (f, inst.strerror)) - removed += 1 - elif m == "m": # merge + progress(_updating, z + i + 1, item=f, total=numupdates, unit=_files) + if m == "m": # merge if fd == '.hgsubstate': # subrepo states need updating subrepo.submerge(repo, wctx, mctx, wctx.ancestor(mctx), overwrite) @@ -423,13 +471,6 @@ updated += 1 else: merged += 1 - elif m == "g": # get - flags, = args - repo.ui.note(_("getting %s\n") % f) - repo.wwrite(f, mctx.filectx(f).data(), flags) - updated += 1 - if f == '.hgsubstate': # subrepo states need updating - subrepo.submerge(repo, wctx, mctx, wctx, overwrite) elif m == "d": # directory rename f2, fd, flags = args if f: @@ -459,7 +500,7 @@ util.setflags(repo.wjoin(f), 'l' in flags, 'x' in flags) updated += 1 ms.commit() - repo.ui.progress(_('updating'), None, total=numupdates, unit=_('files')) + progress(_updating, None, total=numupdates, unit=_files) return updated, merged, removed, unresolved
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/mercurial/worker.py Sun Feb 10 04:04:22 2013 -0600 @@ -0,0 +1,123 @@ +# worker.py - master-slave parallelism support +# +# Copyright 2013 Facebook, Inc. +# +# This software may be used and distributed according to the terms of the +# GNU General Public License version 2 or any later version. + +from i18n import _ +import os, signal, sys, util + +def countcpus(): + '''try to count the number of CPUs on the system''' + + # posix + try: + n = int(os.sysconf('SC_NPROCESSORS_ONLN')) + if n > 0: + return n + except (AttributeError, ValueError): + pass + + # windows + try: + n = int(os.environ['NUMBER_OF_PROCESSORS']) + if n > 0: + return n + except (KeyError, ValueError): + pass + + return 1 + +def _numworkers(ui): + s = ui.config('worker', 'numcpus') + if s: + try: + n = int(s) + if n >= 1: + return n + except ValueError: + raise util.Abort(_('number of cpus must be an integer')) + return min(max(countcpus(), 4), 32) + +if os.name == 'posix': + _startupcost = 0.01 +else: + _startupcost = 1e30 + +def worthwhile(ui, costperop, nops): + '''try to determine whether the benefit of multiple processes can + outweigh the cost of starting them''' + linear = costperop * nops + workers = _numworkers(ui) + benefit = linear - (_startupcost * workers + linear / workers) + return benefit >= 0.15 + +def worker(ui, costperarg, func, staticargs, args): + '''run a function, possibly in parallel in multiple worker + processes. + + returns a progress iterator + + costperarg - cost of a single task + + func - function to run + + staticargs - arguments to pass to every invocation of the function + + args - arguments to split into chunks, to pass to individual + workers + ''' + if worthwhile(ui, costperarg, len(args)): + return _platformworker(ui, func, staticargs, args) + return func(*staticargs + (args,)) + +def _posixworker(ui, func, staticargs, args): + rfd, wfd = os.pipe() + workers = _numworkers(ui) + for pargs in partition(args, workers): + pid = os.fork() + if pid == 0: + try: + os.close(rfd) + for i, item in func(*(staticargs + (pargs,))): + os.write(wfd, '%d %s\n' % (i, item)) + os._exit(0) + except KeyboardInterrupt: + os._exit(255) + os.close(wfd) + fp = os.fdopen(rfd, 'rb', 0) + oldhandler = signal.getsignal(signal.SIGINT) + signal.signal(signal.SIGINT, signal.SIG_IGN) + def cleanup(): + # python 2.4 is too dumb for try/yield/finally + signal.signal(signal.SIGINT, oldhandler) + problems = 0 + for i in xrange(workers): + problems |= os.wait()[1] + if problems: + sys.exit(1) + try: + for line in fp: + l = line.split(' ', 1) + yield int(l[0]), l[1][:-1] + except: # re-raises + cleanup() + raise + cleanup() + +if os.name != 'nt': + _platformworker = _posixworker + +def partition(lst, nslices): + '''partition a list into N slices of equal size''' + n = len(lst) + chunk, slop = n / nslices, n % nslices + end = 0 + for i in xrange(nslices): + start = end + end = start + chunk + if slop: + end += 1 + slop -= 1 + yield lst[start:end]
--- a/tests/test-graft.t Sat Feb 09 22:54:34 2013 +0000 +++ b/tests/test-graft.t Sun Feb 10 04:04:22 2013 -0600 @@ -150,8 +150,8 @@ branchmerge: True, force: True, partial: False ancestor: 4c60f11aa304, local: 6b9e5368ca4e+, remote: 97f8bfe72746 e: remote is newer -> g + getting e updating: e 1/1 files (100.00%) - getting e e grafting revision 4 searching for copies back to rev 1 @@ -161,8 +161,8 @@ d: remote is newer -> g e: versions differ -> m preserving e for resolve of e + getting d updating: d 1/2 files (50.00%) - getting d updating: e 2/2 files (100.00%) picked tool 'internal:merge' for e (binary False symlink False) merging e
--- a/tests/test-issue522.t Sat Feb 09 22:54:34 2013 +0000 +++ b/tests/test-issue522.t Sun Feb 10 04:04:22 2013 -0600 @@ -32,8 +32,8 @@ branchmerge: True, force: False, partial: False ancestor: bbd179dfa0a7, local: 71766447bdbb+, remote: 4d9e78aaceee foo: remote is newer -> g + getting foo updating: foo 1/1 files (100.00%) - getting foo 1 files updated, 0 files merged, 0 files removed, 0 files unresolved (branch merge, don't forget to commit)
--- a/tests/test-issue672.t Sat Feb 09 22:54:34 2013 +0000 +++ b/tests/test-issue672.t Sun Feb 10 04:04:22 2013 -0600 @@ -36,10 +36,9 @@ ancestor: 81f4b099af3d, local: c64f439569a9+, remote: c12dcd37c90a 1: other deleted -> r 1a: remote created -> g - updating: 1 1/2 files (50.00%) removing 1 + getting 1a updating: 1a 2/2 files (100.00%) - getting 1a 1 files updated, 0 files merged, 1 files removed, 0 files unresolved (branch merge, don't forget to commit)
--- a/tests/test-largefiles.t Sat Feb 09 22:54:34 2013 +0000 +++ b/tests/test-largefiles.t Sun Feb 10 04:04:22 2013 -0600 @@ -1679,8 +1679,8 @@ branchmerge: False, force: False, partial: False ancestor: 000000000000, local: 000000000000+, remote: cf03e5bb9936 .hglf/f1: remote created -> g + getting .hglf/f1 updating: .hglf/f1 1/1 files (100.00%) - getting .hglf/f1 getting changed largefiles using http://localhost:$HGPORT2/ sending capabilities command
--- a/tests/test-rename-dir-merge.t Sat Feb 09 22:54:34 2013 +0000 +++ b/tests/test-rename-dir-merge.t Sun Feb 10 04:04:22 2013 -0600 @@ -44,16 +44,13 @@ a/c: remote renamed directory to b/c -> d b/a: remote created -> g b/b: remote created -> g - updating: a/a 1/5 files (20.00%) removing a/a - updating: a/b 2/5 files (40.00%) removing a/b - updating: a/c 3/5 files (60.00%) + getting b/a + getting b/b + updating: b/b 4/5 files (80.00%) + updating: a/c 5/5 files (100.00%) moving a/c to b/c - updating: b/a 4/5 files (80.00%) - getting b/a - updating: b/b 5/5 files (100.00%) - getting b/b 3 files updated, 0 files merged, 2 files removed, 0 files unresolved (branch merge, don't forget to commit)
--- a/tests/test-rename-merge1.t Sat Feb 09 22:54:34 2013 +0000 +++ b/tests/test-rename-merge1.t Sun Feb 10 04:04:22 2013 -0600 @@ -41,17 +41,17 @@ a2: divergent renames -> dr b2: remote created -> g removing a - updating: a 1/3 files (33.33%) + getting b2 + updating: b2 1/3 files (33.33%) + updating: a 2/3 files (66.67%) picked tool 'internal:merge' for b (binary False symlink False) merging a and b to b my b@044f8520aeeb+ other b@85c198ef2f6c ancestor a@af1939970a1c premerge successful - updating: a2 2/3 files (66.67%) + updating: a2 3/3 files (100.00%) note: possible conflict - a2 was renamed multiple times to: c2 b2 - updating: b2 3/3 files (100.00%) - getting b2 1 files updated, 1 files merged, 0 files removed, 0 files unresolved (branch merge, don't forget to commit) @@ -183,11 +183,11 @@ ancestor: 19d7f95df299, local: 0084274f6b67+, remote: 5d32493049f0 file: rename and delete -> rd newfile: remote created -> g - updating: file 1/2 files (50.00%) + getting newfile + updating: newfile 1/2 files (50.00%) + updating: file 2/2 files (100.00%) note: possible conflict - file was deleted and renamed to: newfile - updating: newfile 2/2 files (100.00%) - getting newfile 1 files updated, 0 files merged, 0 files removed, 0 files unresolved (branch merge, don't forget to commit) $ hg status
--- a/tests/test-rename-merge2.t Sat Feb 09 22:54:34 2013 +0000 +++ b/tests/test-rename-merge2.t Sun Feb 10 04:04:22 2013 -0600 @@ -126,8 +126,8 @@ preserving b for resolve of b rev: versions differ -> m preserving rev for resolve of rev + getting a updating: a 1/3 files (33.33%) - getting a updating: b 2/3 files (66.67%) picked tool 'python ../merge' for b (binary False symlink False) merging b and a to b @@ -231,8 +231,8 @@ b: remote created -> g rev: versions differ -> m preserving rev for resolve of rev + getting b updating: b 1/2 files (50.00%) - getting b updating: rev 2/2 files (100.00%) picked tool 'python ../merge' for rev (binary False symlink False) merging rev @@ -289,10 +289,9 @@ b: remote created -> g rev: versions differ -> m preserving rev for resolve of rev - updating: a 1/3 files (33.33%) removing a + getting b updating: b 2/3 files (66.67%) - getting b updating: rev 3/3 files (100.00%) picked tool 'python ../merge' for rev (binary False symlink False) merging rev @@ -380,12 +379,12 @@ c: remote created -> g rev: versions differ -> m preserving rev for resolve of rev - updating: a 1/3 files (33.33%) + getting c + updating: c 1/3 files (33.33%) + updating: a 2/3 files (66.67%) note: possible conflict - a was renamed multiple times to: b c - updating: c 2/3 files (66.67%) - getting c updating: rev 3/3 files (100.00%) picked tool 'python ../merge' for rev (binary False symlink False) merging rev @@ -439,8 +438,8 @@ preserving b for resolve of b rev: versions differ -> m preserving rev for resolve of rev + removing a updating: a 1/3 files (33.33%) - removing a updating: b 2/3 files (66.67%) picked tool 'python ../merge' for b (binary False symlink False) merging b @@ -469,8 +468,8 @@ preserving b for resolve of b rev: versions differ -> m preserving rev for resolve of rev + getting a updating: a 1/3 files (33.33%) - getting a updating: b 2/3 files (66.67%) picked tool 'python ../merge' for b (binary False symlink False) merging b @@ -500,8 +499,8 @@ preserving b for resolve of b rev: versions differ -> m preserving rev for resolve of rev + removing a updating: a 1/3 files (33.33%) - removing a updating: b 2/3 files (66.67%) picked tool 'python ../merge' for b (binary False symlink False) merging b @@ -530,8 +529,8 @@ preserving b for resolve of b rev: versions differ -> m preserving rev for resolve of rev + getting a updating: a 1/3 files (33.33%) - getting a updating: b 2/3 files (66.67%) picked tool 'python ../merge' for b (binary False symlink False) merging b @@ -591,8 +590,8 @@ preserving b for resolve of b rev: versions differ -> m preserving rev for resolve of rev + getting a updating: a 1/3 files (33.33%) - getting a updating: b 2/3 files (66.67%) picked tool 'python ../merge' for b (binary False symlink False) merging b @@ -731,13 +730,13 @@ c: remote created -> g rev: versions differ -> m preserving rev for resolve of rev - updating: b 1/3 files (33.33%) + getting c + updating: c 1/3 files (33.33%) + updating: b 2/3 files (66.67%) picked tool 'python ../merge' for b (binary False symlink False) merging b and a to b my b@02963e448370+ other a@2b958612230f ancestor a@924404dff337 premerge successful - updating: c 2/3 files (66.67%) - getting c updating: rev 3/3 files (100.00%) picked tool 'python ../merge' for rev (binary False symlink False) merging rev
--- a/tests/test-subrepo.t Sat Feb 09 22:54:34 2013 +0000 +++ b/tests/test-subrepo.t Sun Feb 10 04:04:22 2013 -0600 @@ -214,8 +214,8 @@ branchmerge: False, force: False, partial: False ancestor: 60ca1237c194, local: 60ca1237c194+, remote: 6747d179aa9a t: remote is newer -> g + getting t updating: t 1/1 files (100.00%) - getting t 0 files updated, 0 files merged, 0 files removed, 0 files unresolved (branch merge, don't forget to commit) $ hg debugsub
--- a/tests/test-up-local-change.t Sat Feb 09 22:54:34 2013 +0000 +++ b/tests/test-up-local-change.t Sun Feb 10 04:04:22 2013 -0600 @@ -49,12 +49,12 @@ a: versions differ -> m preserving a for resolve of a b: remote created -> g - updating: a 1/2 files (50.00%) + getting b + updating: b 1/2 files (50.00%) + updating: a 2/2 files (100.00%) picked tool 'true' for a (binary False symlink False) merging a my a@c19d34741b0a+ other a@1e71731e6fbb ancestor a@c19d34741b0a - updating: b 2/2 files (100.00%) - getting b 1 files updated, 1 files merged, 0 files removed, 0 files unresolved $ hg parents changeset: 1:1e71731e6fbb @@ -70,8 +70,8 @@ b: other deleted -> r a: versions differ -> m preserving a for resolve of a + removing b updating: b 1/2 files (50.00%) - removing b updating: a 2/2 files (100.00%) picked tool 'true' for a (binary False symlink False) merging a @@ -103,12 +103,12 @@ a: versions differ -> m preserving a for resolve of a b: remote created -> g - updating: a 1/2 files (50.00%) + getting b + updating: b 1/2 files (50.00%) + updating: a 2/2 files (100.00%) picked tool 'true' for a (binary False symlink False) merging a my a@c19d34741b0a+ other a@1e71731e6fbb ancestor a@c19d34741b0a - updating: b 2/2 files (100.00%) - getting b 1 files updated, 1 files merged, 0 files removed, 0 files unresolved $ hg parents changeset: 1:1e71731e6fbb
--- a/tests/test-update-reverse.t Sat Feb 09 22:54:34 2013 +0000 +++ b/tests/test-update-reverse.t Sun Feb 10 04:04:22 2013 -0600 @@ -71,12 +71,10 @@ side1: other deleted -> r side2: other deleted -> r main: remote created -> g - updating: side1 1/3 files (33.33%) removing side1 - updating: side2 2/3 files (66.67%) removing side2 + getting main updating: main 3/3 files (100.00%) - getting main 1 files updated, 0 files merged, 2 files removed, 0 files unresolved $ ls