setdiscovery: make progress on most connected groups each roundtrip
Consider history like this:
o
| o
| |
| o
| |
| o
|/
o
| o
| |
| o
| |
| o
|/
o
| o
| |
| o
| |
| o
|/
o
~
Assume the left mainline is available in the remote repo and the other
commits are only in the local repo. Also imagine that instead of 3
local branches with 3 commits on each, there are 1000 branches (the
number of commits on each doesn't matter much here). In such a
scenario, the current setdiscovery code will pick a sample size of 200
among these branches and ask the remote which of them it has. However,
the discovery for each such branch is completely independent of the
discovery for the others -- knowing whether the remote has a commit in
one branch doesn't give us any information about the other
branches. The discovery will therefore take at least 5 roundtrips
(maybe more depending on which commit in each linear chain was
sampled). Since the discovery for each branch is independent, there is
no reason to let one branch wait for another, so this patch makes it
so we sample at least as many commits as there are branches. It may
still happen (it's very likely, even) that we get multiple samples
from one branch and none from another, but that will even out over a
few rounds and I think this is still a big improvement.
Because of http header size limits, we still use the old behavior
unless experimental.httppostargs=true.
I've timed this by running `hg debugdiscovery mozilla-unified --debug` in the
mozilla-try repo. Both repos were local. Before this patch, last part
of the output was:
2249 total queries in 5276.4859s
elapsed time: 5276.652634 seconds
heads summary:
total common heads: 13
also local heads: 4
also remote heads: 8
both: 4
local heads: 28317
common: 4
missing: 28313
remote heads: 12
common: 8
unknown: 4
local changesets: 2014901
common: 530373
missing: 1484528
common heads:
1dad417c28ad 4a108e94d3e2 4d7ef530fffb 5350524bb654 777e60ca8853 7d97fafba271 9cd2ab4d0029 a55ce37217da d38398e5144e dcc6d7a0dc00 e09297892ada e24ec6070d7b fd559328eaf3
After this patch, the output was (including all the samples, since
there were so few now):
taking initial sample
query 2; still undecided: 1599476, sample size is: 108195
sampling from both directions
query 3; still undecided: 810922, sample size is: 194158
sampling from both directions
query 4; still undecided: 325882, sample size is: 137302
sampling from both directions
query 5; still undecided: 111459, sample size is: 74586
sampling from both directions
query 6; still undecided: 26805, sample size is: 23960
sampling from both directions
query 7; still undecided: 2549, sample size is: 2528
sampling from both directions
query 8; still undecided: 21, sample size is: 21
8 total queries in 24.5064s
elapsed time: 24.670051 seconds
heads summary:
total common heads: 13
also local heads: 4
also remote heads: 8
both: 4
local heads: 28317
common: 4
missing: 28313
remote heads: 12
common: 8
unknown: 4
local changesets: 2014901
common: 530373
missing: 1484528
common heads:
1dad417c28ad 4a108e94d3e2 4d7ef530fffb 5350524bb654 777e60ca8853 7d97fafba271 9cd2ab4d0029 a55ce37217da d38398e5144e dcc6d7a0dc00 e09297892ada e24ec6070d7b fd559328eaf3
Differential Revision: https://phab.mercurial-scm.org/D2647
#!/usr/bin/env python
#
# check-translation.py - check Mercurial specific translation problems
from __future__ import absolute_import
import re
import polib
scanners = []
checkers = []
def scanner():
def decorator(func):
scanners.append(func)
return func
return decorator
def levelchecker(level, msgidpat):
def decorator(func):
if msgidpat:
match = re.compile(msgidpat).search
else:
match = lambda msgid: True
checkers.append((func, level))
func.match = match
return func
return decorator
def match(checker, pe):
"""Examine whether POEntry "pe" is target of specified checker or not
"""
if not checker.match(pe.msgid):
return
# examine suppression by translator comment
nochecker = 'no-%s-check' % checker.__name__
for tc in pe.tcomment.split():
if nochecker == tc:
return
return True
####################
def fatalchecker(msgidpat=None):
return levelchecker('fatal', msgidpat)
@fatalchecker(r'\$\$')
def promptchoice(pe):
"""Check translation of the string given to "ui.promptchoice()"
>>> pe = polib.POEntry(
... msgid ='prompt$$missing &sep$$missing &$$followed by &none',
... msgstr='prompt missing &sep$$missing amp$$followed by none&')
>>> match(promptchoice, pe)
True
>>> for e in promptchoice(pe): print(e)
number of choices differs between msgid and msgstr
msgstr has invalid choice missing '&'
msgstr has invalid '&' followed by none
"""
idchoices = [c.rstrip(' ') for c in pe.msgid.split('$$')[1:]]
strchoices = [c.rstrip(' ') for c in pe.msgstr.split('$$')[1:]]
if len(idchoices) != len(strchoices):
yield "number of choices differs between msgid and msgstr"
indices = [(c, c.find('&')) for c in strchoices]
if [c for c, i in indices if i == -1]:
yield "msgstr has invalid choice missing '&'"
if [c for c, i in indices if len(c) == i + 1]:
yield "msgstr has invalid '&' followed by none"
deprecatedpe = None
@scanner()
def deprecatedsetup(pofile):
pes = [p for p in pofile if p.msgid == '(DEPRECATED)' and p.msgstr]
if len(pes):
global deprecatedpe
deprecatedpe = pes[0]
@fatalchecker(r'\(DEPRECATED\)')
def deprecated(pe):
"""Check for DEPRECATED
>>> ped = polib.POEntry(
... msgid = '(DEPRECATED)',
... msgstr= '(DETACERPED)')
>>> deprecatedsetup([ped])
>>> pe = polib.POEntry(
... msgid = 'Something (DEPRECATED)',
... msgstr= 'something (DEPRECATED)')
>>> match(deprecated, pe)
True
>>> for e in deprecated(pe): print(e)
>>> pe = polib.POEntry(
... msgid = 'Something (DEPRECATED)',
... msgstr= 'something (DETACERPED)')
>>> match(deprecated, pe)
True
>>> for e in deprecated(pe): print(e)
>>> pe = polib.POEntry(
... msgid = 'Something (DEPRECATED)',
... msgstr= 'something')
>>> match(deprecated, pe)
True
>>> for e in deprecated(pe): print(e)
msgstr inconsistently translated (DEPRECATED)
>>> pe = polib.POEntry(
... msgid = 'Something (DEPRECATED, foo bar)',
... msgstr= 'something (DETACERPED, foo bar)')
>>> match(deprecated, pe)
"""
if not ('(DEPRECATED)' in pe.msgstr or
(deprecatedpe and
deprecatedpe.msgstr in pe.msgstr)):
yield "msgstr inconsistently translated (DEPRECATED)"
####################
def warningchecker(msgidpat=None):
return levelchecker('warning', msgidpat)
@warningchecker()
def taildoublecolons(pe):
"""Check equality of tail '::'-ness between msgid and msgstr
>>> pe = polib.POEntry(
... msgid ='ends with ::',
... msgstr='ends with ::')
>>> for e in taildoublecolons(pe): print(e)
>>> pe = polib.POEntry(
... msgid ='ends with ::',
... msgstr='ends without double-colons')
>>> for e in taildoublecolons(pe): print(e)
tail '::'-ness differs between msgid and msgstr
>>> pe = polib.POEntry(
... msgid ='ends without double-colons',
... msgstr='ends with ::')
>>> for e in taildoublecolons(pe): print(e)
tail '::'-ness differs between msgid and msgstr
"""
if pe.msgid.endswith('::') != pe.msgstr.endswith('::'):
yield "tail '::'-ness differs between msgid and msgstr"
@warningchecker()
def indentation(pe):
"""Check equality of initial indentation between msgid and msgstr
This may report unexpected warning, because this doesn't aware
the syntax of rst document and the context of msgstr.
>>> pe = polib.POEntry(
... msgid =' indented text',
... msgstr=' narrowed indentation')
>>> for e in indentation(pe): print(e)
initial indentation width differs betweeen msgid and msgstr
"""
idindent = len(pe.msgid) - len(pe.msgid.lstrip())
strindent = len(pe.msgstr) - len(pe.msgstr.lstrip())
if idindent != strindent:
yield "initial indentation width differs betweeen msgid and msgstr"
####################
def check(pofile, fatal=True, warning=False):
targetlevel = { 'fatal': fatal, 'warning': warning }
targetcheckers = [(checker, level)
for checker, level in checkers
if targetlevel[level]]
if not targetcheckers:
return []
detected = []
for checker in scanners:
checker(pofile)
for pe in pofile.translated_entries():
errors = []
for checker, level in targetcheckers:
if match(checker, pe):
errors.extend((level, checker.__name__, error)
for error in checker(pe))
if errors:
detected.append((pe, errors))
return detected
########################################
if __name__ == "__main__":
import sys
import optparse
optparser = optparse.OptionParser("""%prog [options] pofile ...
This checks Mercurial specific translation problems in specified
'*.po' files.
Each detected problems are shown in the format below::
filename:linenum:type(checker): problem detail .....
"type" is "fatal" or "warning". "checker" is the name of the function
detecting corresponded error.
Checking by checker "foo" on the specific msgstr can be suppressed by
the "translator comment" like below. Multiple "no-xxxx-check" should
be separated by whitespaces::
# no-foo-check
msgid = "....."
msgstr = "....."
""")
optparser.add_option("", "--warning",
help="show also warning level problems",
action="store_true")
optparser.add_option("", "--doctest",
help="run doctest of this tool, instead of check",
action="store_true")
(options, args) = optparser.parse_args()
if options.doctest:
import os
if 'TERM' in os.environ:
del os.environ['TERM']
import doctest
failures, tests = doctest.testmod()
sys.exit(failures and 1 or 0)
detected = []
warning = options.warning
for f in args:
detected.extend((f, pe, errors)
for pe, errors in check(polib.pofile(f),
warning=warning))
if detected:
for f, pe, errors in detected:
for level, checker, error in errors:
sys.stderr.write('%s:%d:%s(%s): %s\n'
% (f, pe.linenum, level, checker, error))
sys.exit(1)