# HG changeset patch # User Martin von Zweigbergk # Date 1576259023 28800 # Node ID 5606e1cb468524dad793d2ea56b93cd100ab2c16 # Parent aac921f54554eb1cb4d4416c8237773ceba7de9b# Parent 612b4b63fb2267320f18655c0fa8a675f8fe3939 merge with stable diff -r aac921f54554 -r 5606e1cb4685 hgext/remotefilelog/__init__.py --- a/hgext/remotefilelog/__init__.py Mon Dec 09 22:24:58 2019 -0800 +++ b/hgext/remotefilelog/__init__.py Fri Dec 13 09:43:43 2019 -0800 @@ -229,7 +229,7 @@ configitem(b'packs', b'maxpacksize', default=0) configitem(b'packs', b'maxchainlen', default=1000) -configitem(b'devel', b'remotefilelog.ensurestart', default=False) +configitem(b'devel', b'remotefilelog.bg-wait', default=False) # default TTL limit is 30 days _defaultlimit = 60 * 60 * 24 * 30 @@ -1080,26 +1080,19 @@ prefetchrevset = ui.config(b'remotefilelog', b'pullprefetch') bgrepack = repo.ui.configbool(b'remotefilelog', b'backgroundrepack') bgprefetch = repo.ui.configbool(b'remotefilelog', b'backgroundprefetch') - ensurestart = repo.ui.configbool(b'devel', b'remotefilelog.ensurestart') if prefetchrevset: ui.status(_(b"prefetching file contents\n")) revs = scmutil.revrange(repo, [prefetchrevset]) base = repo[b'.'].rev() if bgprefetch: - repo.backgroundprefetch( - prefetchrevset, repack=bgrepack, ensurestart=ensurestart - ) + repo.backgroundprefetch(prefetchrevset, repack=bgrepack) else: repo.prefetch(revs, base=base) if bgrepack: - repackmod.backgroundrepack( - repo, incremental=True, ensurestart=ensurestart - ) + repackmod.backgroundrepack(repo, incremental=True) elif bgrepack: - repackmod.backgroundrepack( - repo, incremental=True, ensurestart=ensurestart - ) + repackmod.backgroundrepack(repo, incremental=True) return result @@ -1250,13 +1243,9 @@ revs = scmutil.revrange(repo, opts.get(b'rev')) repo.prefetch(revs, opts.get(b'base'), pats, opts) - ensurestart = repo.ui.configbool(b'devel', b'remotefilelog.ensurestart') - # Run repack in background if opts.get(b'repack'): - repackmod.backgroundrepack( - repo, incremental=True, ensurestart=ensurestart - ) + repackmod.backgroundrepack(repo, incremental=True) @command( @@ -1276,12 +1265,10 @@ ) def repack_(ui, repo, *pats, **opts): if opts.get('background'): - ensurestart = repo.ui.configbool(b'devel', b'remotefilelog.ensurestart') repackmod.backgroundrepack( repo, incremental=opts.get('incremental'), packsonly=opts.get('packsonly', False), - ensurestart=ensurestart, ) return diff -r aac921f54554 -r 5606e1cb4685 hgext/remotefilelog/repack.py --- a/hgext/remotefilelog/repack.py Mon Dec 09 22:24:58 2019 -0800 +++ b/hgext/remotefilelog/repack.py Fri Dec 13 09:43:43 2019 -0800 @@ -36,9 +36,7 @@ pass -def backgroundrepack( - repo, incremental=True, packsonly=False, ensurestart=False -): +def backgroundrepack(repo, incremental=True, packsonly=False): cmd = [procutil.hgexecutable(), b'-R', repo.origroot, b'repack'] msg = _(b"(running background repack)\n") if incremental: @@ -48,7 +46,11 @@ cmd.append(b'--packsonly') repo.ui.warn(msg) # We know this command will find a binary, so don't block on it starting. - procutil.runbgcommand(cmd, encoding.environ, ensurestart=ensurestart) + kwargs = {} + if repo.ui.configbool(b'devel', b'remotefilelog.bg-wait'): + kwargs['record_wait'] = repo.ui.atexit + + procutil.runbgcommand(cmd, encoding.environ, ensurestart=False, **kwargs) def fullrepack(repo, options=None): diff -r aac921f54554 -r 5606e1cb4685 hgext/remotefilelog/shallowrepo.py --- a/hgext/remotefilelog/shallowrepo.py Mon Dec 09 22:24:58 2019 -0800 +++ b/hgext/remotefilelog/shallowrepo.py Fri Dec 13 09:43:43 2019 -0800 @@ -215,13 +215,7 @@ ) def backgroundprefetch( - self, - revs, - base=None, - repack=False, - pats=None, - opts=None, - ensurestart=False, + self, revs, base=None, repack=False, pats=None, opts=None ): """Runs prefetch in background with optional repack """ @@ -232,8 +226,12 @@ cmd += [b'-r', revs] # We know this command will find a binary, so don't block # on it starting. + kwargs = {} + if repo.ui.configbool(b'devel', b'remotefilelog.bg-wait'): + kwargs['record_wait'] = repo.ui.atexit + procutil.runbgcommand( - cmd, encoding.environ, ensurestart=ensurestart + cmd, encoding.environ, ensurestart=False, **kwargs ) def prefetch(self, revs, base=None, pats=None, opts=None): diff -r aac921f54554 -r 5606e1cb4685 mercurial/utils/procutil.py --- a/mercurial/utils/procutil.py Mon Dec 09 22:24:58 2019 -0800 +++ b/mercurial/utils/procutil.py Fri Dec 13 09:43:43 2019 -0800 @@ -540,12 +540,18 @@ ) def runbgcommand( - script, env, shell=False, stdout=None, stderr=None, ensurestart=True + script, + env, + shell=False, + stdout=None, + stderr=None, + ensurestart=True, + record_wait=None, ): '''Spawn a command without waiting for it to finish.''' # we can't use close_fds *and* redirect stdin. I'm not sure that we # need to because the detached process has no console connection. - subprocess.Popen( + p = subprocess.Popen( tonativestr(script), shell=shell, env=tonativeenv(env), @@ -554,46 +560,64 @@ stdout=stdout, stderr=stderr, ) + if record_wait is not None: + record_wait(p.wait) else: def runbgcommand( - cmd, env, shell=False, stdout=None, stderr=None, ensurestart=True + cmd, + env, + shell=False, + stdout=None, + stderr=None, + ensurestart=True, + record_wait=None, ): - '''Spawn a command without waiting for it to finish.''' + '''Spawn a command without waiting for it to finish. + + + When `record_wait` is not None, the spawned process will not be fully + detached and the `record_wait` argument will be called with a the + `Subprocess.wait` function for the spawned process. This is mostly + useful for developers that need to make sure the spawned process + finished before a certain point. (eg: writing test)''' # double-fork to completely detach from the parent process # based on http://code.activestate.com/recipes/278731 - pid = os.fork() - if pid: - if not ensurestart: + if record_wait is None: + pid = os.fork() + if pid: + if not ensurestart: + return + # Parent process + (_pid, status) = os.waitpid(pid, 0) + if os.WIFEXITED(status): + returncode = os.WEXITSTATUS(status) + else: + returncode = -(os.WTERMSIG(status)) + if returncode != 0: + # The child process's return code is 0 on success, an errno + # value on failure, or 255 if we don't have a valid errno + # value. + # + # (It would be slightly nicer to return the full exception info + # over a pipe as the subprocess module does. For now it + # doesn't seem worth adding that complexity here, though.) + if returncode == 255: + returncode = errno.EINVAL + raise OSError( + returncode, + b'error running %r: %s' + % (cmd, os.strerror(returncode)), + ) return - # Parent process - (_pid, status) = os.waitpid(pid, 0) - if os.WIFEXITED(status): - returncode = os.WEXITSTATUS(status) - else: - returncode = -(os.WTERMSIG(status)) - if returncode != 0: - # The child process's return code is 0 on success, an errno - # value on failure, or 255 if we don't have a valid errno - # value. - # - # (It would be slightly nicer to return the full exception info - # over a pipe as the subprocess module does. For now it - # doesn't seem worth adding that complexity here, though.) - if returncode == 255: - returncode = errno.EINVAL - raise OSError( - returncode, - b'error running %r: %s' % (cmd, os.strerror(returncode)), - ) - return returncode = 255 try: - # Start a new session - os.setsid() + if record_wait is None: + # Start a new session + os.setsid() stdin = open(os.devnull, b'r') if stdout is None: @@ -603,7 +627,7 @@ # connect stdin to devnull to make sure the subprocess can't # muck up that stream for mercurial. - subprocess.Popen( + p = subprocess.Popen( cmd, shell=shell, env=env, @@ -612,6 +636,8 @@ stdout=stdout, stderr=stderr, ) + if record_wait is not None: + record_wait(p.wait) returncode = 0 except EnvironmentError as ex: returncode = ex.errno & 0xFF @@ -624,4 +650,5 @@ finally: # mission accomplished, this child needs to exit and not # continue the hg process here. - os._exit(returncode) + if record_wait is None: + os._exit(returncode) diff -r aac921f54554 -r 5606e1cb4685 mercurial/utils/resourceutil.py diff -r aac921f54554 -r 5606e1cb4685 tests/test-remotefilelog-bgprefetch.t --- a/tests/test-remotefilelog-bgprefetch.t Mon Dec 09 22:24:58 2019 -0800 +++ b/tests/test-remotefilelog-bgprefetch.t Fri Dec 13 09:43:43 2019 -0800 @@ -1,11 +1,9 @@ #require no-windows $ . "$TESTDIR/remotefilelog-library.sh" -# devel.remotefilelog.ensurestart: reduce race condition with -# waiton{repack/prefetch} $ cat >> $HGRCPATH < [devel] - > remotefilelog.ensurestart=True + > remotefilelog.bg-wait=True > EOF $ hg init master @@ -78,8 +76,6 @@ new changesets 6b4b6f66ef8c (run 'hg update' to get a working copy) prefetching file contents - $ sleep 0.5 - $ hg debugwaitonprefetch >/dev/null 2>%1 $ find $CACHEDIR -type f | sort $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/ef95c5376f34698742fe34f315fd82136f8f68c0 $TESTTMP/hgcache/master/95/cb0bfd2977c761298d9624e4b4d4c72a39974a/076f5e2225b3ff0400b98c92aa6cdf403ee24cca @@ -107,11 +103,6 @@ new changesets 6b4b6f66ef8c (run 'hg update' to get a working copy) prefetching file contents - $ sleep 0.5 - $ hg debugwaitonprefetch >/dev/null 2>%1 - $ sleep 0.5 - $ hg debugwaitonrepack >/dev/null 2>%1 - $ sleep 0.5 $ find $CACHEDIR -type f | sort $TESTTMP/hgcache/master/packs/6e8633deba6e544e5f8edbd7b996d6e31a2c42ae.histidx $TESTTMP/hgcache/master/packs/6e8633deba6e544e5f8edbd7b996d6e31a2c42ae.histpack @@ -143,11 +134,6 @@ $ hg up -r 0 1 files updated, 0 files merged, 1 files removed, 0 files unresolved * files fetched over * fetches - (* misses, 0.00% hit ratio) over *s (glob) - $ sleep 1 - $ hg debugwaitonprefetch >/dev/null 2>%1 - $ sleep 1 - $ hg debugwaitonrepack >/dev/null 2>%1 - $ sleep 1 $ find $CACHEDIR -type f | sort $TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histidx $TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histpack @@ -195,11 +181,6 @@ $ hg commit -qAm b * files fetched over 1 fetches - (* misses, 0.00% hit ratio) over *s (glob) $ hg bookmark temporary - $ sleep 1 - $ hg debugwaitonprefetch >/dev/null 2>%1 - $ sleep 1 - $ hg debugwaitonrepack >/dev/null 2>%1 - $ sleep 1 $ find $CACHEDIR -type f | sort $TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histidx $TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histpack @@ -237,19 +218,14 @@ # background prefetch with repack on rebase when wcprevset configured $ hg up -r 2 - 3 files updated, 0 files merged, 3 files removed, 0 files unresolved + 3 files updated, 0 files merged, 2 files removed, 0 files unresolved (leaving bookmark temporary) $ clearcache $ find $CACHEDIR -type f | sort $ hg rebase -s temporary -d foo - rebasing 3:58147a5b5242 "b" (temporary tip) - saved backup bundle to $TESTTMP/shallow/.hg/strip-backup/58147a5b5242-c3678817-rebase.hg + rebasing 3:d9cf06e3b5b6 "b" (temporary tip) + saved backup bundle to $TESTTMP/shallow/.hg/strip-backup/d9cf06e3b5b6-e5c3dc63-rebase.hg 3 files fetched over 1 fetches - (3 misses, 0.00% hit ratio) over *s (glob) - $ sleep 1 - $ hg debugwaitonprefetch >/dev/null 2>%1 - $ sleep 1 - $ hg debugwaitonrepack >/dev/null 2>%1 - $ sleep 1 # Ensure that file 'y' was prefetched - it was not part of the rebase operation and therefore # could only be downloaded by the background prefetch @@ -281,7 +257,7 @@ # Check that foregound prefetch with no arguments blocks until background prefetches finish $ hg up -r 3 - 2 files updated, 0 files merged, 0 files removed, 0 files unresolved + 1 files updated, 0 files merged, 0 files removed, 0 files unresolved $ clearcache $ hg prefetch --repack waiting for lock on prefetching in $TESTTMP/shallow held by process * on host * (glob) (?) @@ -289,10 +265,6 @@ (running background incremental repack) * files fetched over 1 fetches - (* misses, 0.00% hit ratio) over *s (glob) (?) - $ sleep 0.5 - $ hg debugwaitonrepack >/dev/null 2>%1 - $ sleep 0.5 - $ find $CACHEDIR -type f | sort $TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histidx $TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histpack @@ -333,9 +305,6 @@ got lock after * seconds (glob) (?) (running background incremental repack) * files fetched over 1 fetches - (* misses, 0.00% hit ratio) over *s (glob) (?) - $ sleep 0.5 - $ hg debugwaitonrepack >/dev/null 2>%1 - $ sleep 0.5 $ find $CACHEDIR -type f | sort $TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histidx diff -r aac921f54554 -r 5606e1cb4685 tests/test-remotefilelog-gcrepack.t --- a/tests/test-remotefilelog-gcrepack.t Mon Dec 09 22:24:58 2019 -0800 +++ b/tests/test-remotefilelog-gcrepack.t Fri Dec 13 09:43:43 2019 -0800 @@ -40,8 +40,6 @@ $ hg prefetch 2 files fetched over 1 fetches - (2 misses, 0.00% hit ratio) over *s (glob) $ hg repack - $ sleep 0.5 - $ hg debugwaitonrepack >/dev/null 2>%1 $ find $CACHEDIR | sort | egrep ".datapack|.histpack" $TESTTMP/hgcache/master/packs/7bcd2d90b99395ca43172a0dd24e18860b2902f9.histpack @@ -73,8 +71,6 @@ > EOF $ hg repack - $ sleep 0.5 - $ hg debugwaitonrepack >/dev/null 2>%1 $ find $CACHEDIR | sort | egrep ".datapack|.histpack" $TESTTMP/hgcache/master/packs/7bcd2d90b99395ca43172a0dd24e18860b2902f9.histpack @@ -101,8 +97,6 @@ $ hg prefetch 2 files fetched over 1 fetches - (2 misses, 0.00% hit ratio) over *s (glob) $ hg repack - $ sleep 0.5 - $ hg debugwaitonrepack >/dev/null 2>%1 $ find $CACHEDIR | sort | egrep ".datapack|.histpack" $TESTTMP/hgcache/master/packs/7bcd2d90b99395ca43172a0dd24e18860b2902f9.histpack @@ -135,8 +129,6 @@ > EOF $ hg repack - $ sleep 0.5 - $ hg debugwaitonrepack >/dev/null 2>%1 $ find $CACHEDIR | sort | egrep ".datapack|.histpack" $TESTTMP/hgcache/master/packs/7bcd2d90b99395ca43172a0dd24e18860b2902f9.histpack diff -r aac921f54554 -r 5606e1cb4685 tests/test-remotefilelog-repack-fast.t --- a/tests/test-remotefilelog-repack-fast.t Mon Dec 09 22:24:58 2019 -0800 +++ b/tests/test-remotefilelog-repack-fast.t Fri Dec 13 09:43:43 2019 -0800 @@ -1,13 +1,12 @@ #require no-windows $ . "$TESTDIR/remotefilelog-library.sh" -# devel.remotefilelog.ensurestart: reduce race condition with -# waiton{repack/prefetch} + $ cat >> $HGRCPATH < [remotefilelog] > fastdatapack=True > [devel] - > remotefilelog.ensurestart=True + > remotefilelog.bg-wait=True > EOF $ hg init master @@ -142,8 +141,6 @@ $ hg repack --background (running background repack) - $ sleep 0.5 - $ hg debugwaitonrepack >/dev/null 2>&1 $ find $CACHEDIR -type f | sort $TESTTMP/hgcache/master/packs/39443fa1064182e93d968b5cba292eb5283260d0.dataidx $TESTTMP/hgcache/master/packs/39443fa1064182e93d968b5cba292eb5283260d0.datapack @@ -342,8 +339,6 @@ searching for changes no changes found (running background incremental repack) - $ sleep 0.5 - $ hg debugwaitonrepack >/dev/null 2>&1 $ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack -r--r--r-- 303 156a6c1c83aeb69422d7936e0a46ba9bc06a71c0.datapack $ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack diff -r aac921f54554 -r 5606e1cb4685 tests/test-remotefilelog-repack.t --- a/tests/test-remotefilelog-repack.t Mon Dec 09 22:24:58 2019 -0800 +++ b/tests/test-remotefilelog-repack.t Fri Dec 13 09:43:43 2019 -0800 @@ -1,11 +1,9 @@ #require no-windows $ . "$TESTDIR/remotefilelog-library.sh" -# devel.remotefilelog.ensurestart: reduce race condition with -# waiton{repack/prefetch} $ cat >> $HGRCPATH < [devel] - > remotefilelog.ensurestart=True + > remotefilelog.bg-wait=True > EOF $ hg init master @@ -152,8 +150,6 @@ $ hg repack --background (running background repack) - $ sleep 0.5 - $ hg debugwaitonrepack >/dev/null 2>&1 $ find $CACHEDIR -type f | sort $TESTTMP/hgcache/master/packs/39443fa1064182e93d968b5cba292eb5283260d0.dataidx $TESTTMP/hgcache/master/packs/39443fa1064182e93d968b5cba292eb5283260d0.datapack @@ -371,8 +367,6 @@ searching for changes no changes found (running background incremental repack) - $ sleep 0.5 - $ hg debugwaitonrepack >/dev/null 2>&1 $ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack -r--r--r-- 303 156a6c1c83aeb69422d7936e0a46ba9bc06a71c0.datapack $ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack