Mercurial > hg
comparison mercurial/verify.py @ 39872:733db72f0f54
revlog: move revision verification out of verify
File revision verification is performing low-level checks of file
storage, namely that flags are appropriate and revision data can
be resolved.
Since these checks are somewhat revlog-specific and may not
be appropriate for alternate storage backends, this commit moves
those checks from verify.py to revlog.py.
Because we're now emitting warnings/errors that apply to specific
revisions, we taught the iverifyproblem interface to expose the
problematic node and to report this node in verify output. This
was necessary to prevent unwanted test changes.
After this change, revlog.verifyintegrity() and file verify code
in verify.py both iterate over revisions and resolve their fulltext.
But they do so in separate loops. (verify.py needs to resolve
fulltexts as part of calling renamed() - at least when using revlogs.)
This should add overhead.
But on the mozilla-unified repo:
$ hg verify
before: time: real 700.640 secs (user 585.520+0.000 sys 23.480+0.000)
after: time: real 682.380 secs (user 570.370+0.000 sys 22.240+0.000)
I'm not sure what's going on. Maybe avoiding the filelog attribute
proxies shaved off enough time to offset the losses? Maybe fulltext
resolution has less overhead than I thought?
I've left a comment indicating the potential for optimization. But
because it doesn't produce a performance regression on a large
repository, I'm not going to worry about it.
Differential Revision: https://phab.mercurial-scm.org/D4745
author | Gregory Szorc <gregory.szorc@gmail.com> |
---|---|
date | Mon, 24 Sep 2018 11:27:47 -0700 |
parents | e6d3d39cc1c7 |
children | fec944719324 |
comparison
equal
deleted
inserted
replaced
39871:01c0f01b562b | 39872:733db72f0f54 |
---|---|
341 elif (size > 0 or not revlogv1) and f.startswith('data/'): | 341 elif (size > 0 or not revlogv1) and f.startswith('data/'): |
342 storefiles.add(_normpath(f)) | 342 storefiles.add(_normpath(f)) |
343 | 343 |
344 state = { | 344 state = { |
345 # TODO this assumes revlog storage for changelog. | 345 # TODO this assumes revlog storage for changelog. |
346 'expectedversion': self.repo.changelog.version & 0xFFFF | 346 'expectedversion': self.repo.changelog.version & 0xFFFF, |
347 'skipflags': self.skipflags, | |
348 # experimental config: censor.policy | |
349 'erroroncensored': ui.config('censor', 'policy') == 'abort', | |
347 } | 350 } |
348 | 351 |
349 files = sorted(set(filenodes) | set(filelinkrevs)) | 352 files = sorted(set(filenodes) | set(filelinkrevs)) |
350 revisions = 0 | 353 revisions = 0 |
351 progress = ui.makeprogress(_('checking'), unit=_('files'), | 354 progress = ui.makeprogress(_('checking'), unit=_('files'), |
379 self.fncachewarned = True | 382 self.fncachewarned = True |
380 | 383 |
381 if not len(fl) and (self.havecl or self.havemf): | 384 if not len(fl) and (self.havecl or self.havemf): |
382 self.err(lr, _("empty or missing %s") % f) | 385 self.err(lr, _("empty or missing %s") % f) |
383 else: | 386 else: |
387 # Guard against implementations not setting this. | |
388 state['skipread'] = set() | |
384 for problem in fl.verifyintegrity(state): | 389 for problem in fl.verifyintegrity(state): |
390 if problem.node is not None: | |
391 linkrev = fl.linkrev(fl.rev(problem.node)) | |
392 else: | |
393 linkrev = None | |
394 | |
385 if problem.warning: | 395 if problem.warning: |
386 self.warn(problem.warning) | 396 self.warn(problem.warning) |
387 elif problem.error: | 397 elif problem.error: |
388 self.err(lr, problem.error, f) | 398 self.err(linkrev if linkrev is not None else lr, |
399 problem.error, f) | |
389 else: | 400 else: |
390 raise error.ProgrammingError( | 401 raise error.ProgrammingError( |
391 'problem instance does not set warning or error ' | 402 'problem instance does not set warning or error ' |
392 'attribute: %s' % problem.msg) | 403 'attribute: %s' % problem.msg) |
393 | 404 |
394 seen = {} | 405 seen = {} |
395 rp = None | |
396 for i in fl: | 406 for i in fl: |
397 revisions += 1 | 407 revisions += 1 |
398 n = fl.node(i) | 408 n = fl.node(i) |
399 lr = self.checkentry(fl, i, n, seen, linkrevs, f) | 409 lr = self.checkentry(fl, i, n, seen, linkrevs, f) |
400 if f in filenodes: | 410 if f in filenodes: |
401 if havemf and n not in filenodes[f]: | 411 if havemf and n not in filenodes[f]: |
402 self.err(lr, _("%s not in manifests") % (short(n)), f) | 412 self.err(lr, _("%s not in manifests") % (short(n)), f) |
403 else: | 413 else: |
404 del filenodes[f][n] | 414 del filenodes[f][n] |
405 | 415 |
406 # Verify contents. 4 cases to care about: | 416 if n in state['skipread']: |
407 # | 417 continue |
408 # common: the most common case | |
409 # rename: with a rename | |
410 # meta: file content starts with b'\1\n', the metadata | |
411 # header defined in filelog.py, but without a rename | |
412 # ext: content stored externally | |
413 # | |
414 # More formally, their differences are shown below: | |
415 # | |
416 # | common | rename | meta | ext | |
417 # ------------------------------------------------------- | |
418 # flags() | 0 | 0 | 0 | not 0 | |
419 # renamed() | False | True | False | ? | |
420 # rawtext[0:2]=='\1\n'| False | True | True | ? | |
421 # | |
422 # "rawtext" means the raw text stored in revlog data, which | |
423 # could be retrieved by "revision(rev, raw=True)". "text" | |
424 # mentioned below is "revision(rev, raw=False)". | |
425 # | |
426 # There are 3 different lengths stored physically: | |
427 # 1. L1: rawsize, stored in revlog index | |
428 # 2. L2: len(rawtext), stored in revlog data | |
429 # 3. L3: len(text), stored in revlog data if flags==0, or | |
430 # possibly somewhere else if flags!=0 | |
431 # | |
432 # L1 should be equal to L2. L3 could be different from them. | |
433 # "text" may or may not affect commit hash depending on flag | |
434 # processors (see revlog.addflagprocessor). | |
435 # | |
436 # | common | rename | meta | ext | |
437 # ------------------------------------------------- | |
438 # rawsize() | L1 | L1 | L1 | L1 | |
439 # size() | L1 | L2-LM | L1(*) | L1 (?) | |
440 # len(rawtext) | L2 | L2 | L2 | L2 | |
441 # len(text) | L2 | L2 | L2 | L3 | |
442 # len(read()) | L2 | L2-LM | L2-LM | L3 (?) | |
443 # | |
444 # LM: length of metadata, depending on rawtext | |
445 # (*): not ideal, see comment in filelog.size | |
446 # (?): could be "- len(meta)" if the resolved content has | |
447 # rename metadata | |
448 # | |
449 # Checks needed to be done: | |
450 # 1. length check: L1 == L2, in all cases. | |
451 # 2. hash check: depending on flag processor, we may need to | |
452 # use either "text" (external), or "rawtext" (in revlog). | |
453 try: | |
454 skipflags = self.skipflags | |
455 if skipflags: | |
456 skipflags &= fl.flags(i) | |
457 if not skipflags: | |
458 fl.read(n) # side effect: read content and do checkhash | |
459 rp = fl.renamed(n) | |
460 # the "L1 == L2" check | |
461 l1 = fl.rawsize(i) | |
462 l2 = len(fl.revision(n, raw=True)) | |
463 if l1 != l2: | |
464 self.err(lr, _("unpacked size is %s, %s expected") % | |
465 (l2, l1), f) | |
466 except error.CensoredNodeError: | |
467 # experimental config: censor.policy | |
468 if ui.config("censor", "policy") == "abort": | |
469 self.err(lr, _("censored file data"), f) | |
470 except Exception as inst: | |
471 self.exc(lr, _("unpacking %s") % short(n), inst, f) | |
472 | 418 |
473 # check renames | 419 # check renames |
474 try: | 420 try: |
421 # This requires resolving fulltext (at least on revlogs). We | |
422 # may want ``verifyintegrity()`` to pass a set of nodes with | |
423 # rename metadata as an optimization. | |
424 rp = fl.renamed(n) | |
475 if rp: | 425 if rp: |
476 if lr is not None and ui.verbose: | 426 if lr is not None and ui.verbose: |
477 ctx = lrugetctx(lr) | 427 ctx = lrugetctx(lr) |
478 if not any(rp[0] in pctx for pctx in ctx.parents()): | 428 if not any(rp[0] in pctx for pctx in ctx.parents()): |
479 self.warn(_("warning: copy source of '%s' not" | 429 self.warn(_("warning: copy source of '%s' not" |