mercurial/verify.py
author Pierre-Yves David <pierre-yves.david@octobus.net>
Tue, 01 Jun 2021 08:54:53 +0200
changeset 47353 5ed2aaab58b0
parent 47139 f58a13c52726
child 47354 03dea8553114
permissions -rw-r--r--
verify: align a comment line This argument description is back with it comrade. Differential Revision: https://phab.mercurial-scm.org/D10805
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
2778
fdc232d8a193 Move repo.verify
Matt Mackall <mpm@selenic.com>
parents:
diff changeset
     1
# verify.py - repository integrity checking for Mercurial
fdc232d8a193 Move repo.verify
Matt Mackall <mpm@selenic.com>
parents:
diff changeset
     2
#
46819
d4ba4d51f85f contributor: change mentions of mpm to olivia
Raphaël Gomès <rgomes@octobus.net>
parents: 46790
diff changeset
     3
# Copyright 2006, 2007 Olivia Mackall <olivia@selenic.com>
2778
fdc232d8a193 Move repo.verify
Matt Mackall <mpm@selenic.com>
parents:
diff changeset
     4
#
8225
46293a0c7e9f updated license to be explicit about GPL version 2
Martin Geisler <mg@lazybytes.net>
parents: 8209
diff changeset
     5
# This software may be used and distributed according to the terms of the
10263
25e572394f5c Update license to GPLv2+
Matt Mackall <mpm@selenic.com>
parents: 9690
diff changeset
     6
# GNU General Public License version 2 or any later version.
2778
fdc232d8a193 Move repo.verify
Matt Mackall <mpm@selenic.com>
parents:
diff changeset
     7
25991
d21d1774c73b verify: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25846
diff changeset
     8
from __future__ import absolute_import
d21d1774c73b verify: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25846
diff changeset
     9
17860
a45b33f12627 verify: fix all doubled-slash sites (issue3665)
Bryan O'Sullivan <bryano@fb.com>
parents: 17851
diff changeset
    10
import os
25991
d21d1774c73b verify: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25846
diff changeset
    11
d21d1774c73b verify: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25846
diff changeset
    12
from .i18n import _
47012
d55b71393907 node: replace nullid and friends with nodeconstants class
Joerg Sonnenberger <joerg@bec.de>
parents: 46895
diff changeset
    13
from .node import short
d55b71393907 node: replace nullid and friends with nodeconstants class
Joerg Sonnenberger <joerg@bec.de>
parents: 46895
diff changeset
    14
from .utils import stringutil
25991
d21d1774c73b verify: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25846
diff changeset
    15
d21d1774c73b verify: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25846
diff changeset
    16
from . import (
d21d1774c73b verify: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25846
diff changeset
    17
    error,
35585
35fb3367f72d py3: use pycompat.bytestr() instead of str()
Pulkit Goyal <7895pulkit@gmail.com>
parents: 33499
diff changeset
    18
    pycompat,
25991
d21d1774c73b verify: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25846
diff changeset
    19
    revlog,
d21d1774c73b verify: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25846
diff changeset
    20
    util,
d21d1774c73b verify: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25846
diff changeset
    21
)
2778
fdc232d8a193 Move repo.verify
Matt Mackall <mpm@selenic.com>
parents:
diff changeset
    22
42155
57539e5ea2e0 verify: introduce a notion of "level"
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41877
diff changeset
    23
VERIFY_DEFAULT = 0
42156
496ac8a02380 verify: introduce an experimental --full flag
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 42155
diff changeset
    24
VERIFY_FULL = 1
42155
57539e5ea2e0 verify: introduce a notion of "level"
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41877
diff changeset
    25
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
    26
42155
57539e5ea2e0 verify: introduce a notion of "level"
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41877
diff changeset
    27
def verify(repo, level=None):
27849
900d36a3e4dd with: use context manager in verify
Bryan O'Sullivan <bryano@fb.com>
parents: 27695
diff changeset
    28
    with repo.lock():
42155
57539e5ea2e0 verify: introduce a notion of "level"
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41877
diff changeset
    29
        v = verifier(repo, level)
57539e5ea2e0 verify: introduce a notion of "level"
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41877
diff changeset
    30
        return v.verify()
4915
97b734fb9c6f Use try/finally pattern to cleanup locks and transactions
Matt Mackall <mpm@selenic.com>
parents: 4635
diff changeset
    31
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
    32
17860
a45b33f12627 verify: fix all doubled-slash sites (issue3665)
Bryan O'Sullivan <bryano@fb.com>
parents: 17851
diff changeset
    33
def _normpath(f):
a45b33f12627 verify: fix all doubled-slash sites (issue3665)
Bryan O'Sullivan <bryano@fb.com>
parents: 17851
diff changeset
    34
    # under hg < 2.4, convert didn't sanitize paths properly, so a
a45b33f12627 verify: fix all doubled-slash sites (issue3665)
Bryan O'Sullivan <bryano@fb.com>
parents: 17851
diff changeset
    35
    # converted repo may contain repeated slashes
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
    36
    while b'//' in f:
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
    37
        f = f.replace(b'//', b'/')
17860
a45b33f12627 verify: fix all doubled-slash sites (issue3665)
Bryan O'Sullivan <bryano@fb.com>
parents: 17851
diff changeset
    38
    return f
a45b33f12627 verify: fix all doubled-slash sites (issue3665)
Bryan O'Sullivan <bryano@fb.com>
parents: 17851
diff changeset
    39
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
    40
27443
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
    41
class verifier(object):
42155
57539e5ea2e0 verify: introduce a notion of "level"
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41877
diff changeset
    42
    def __init__(self, repo, level=None):
27444
6647401858ab verify: move widely used variables into class members
Durham Goode <durham@fb.com>
parents: 27443
diff changeset
    43
        self.repo = repo.unfiltered()
6647401858ab verify: move widely used variables into class members
Durham Goode <durham@fb.com>
parents: 27443
diff changeset
    44
        self.ui = repo.ui
39938
fec944719324 narrow: move support for `hg verify` into core
Martin von Zweigbergk <martinvonz@google.com>
parents: 39872
diff changeset
    45
        self.match = repo.narrowmatch()
42155
57539e5ea2e0 verify: introduce a notion of "level"
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41877
diff changeset
    46
        if level is None:
57539e5ea2e0 verify: introduce a notion of "level"
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41877
diff changeset
    47
            level = VERIFY_DEFAULT
57539e5ea2e0 verify: introduce a notion of "level"
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41877
diff changeset
    48
        self._level = level
27444
6647401858ab verify: move widely used variables into class members
Durham Goode <durham@fb.com>
parents: 27443
diff changeset
    49
        self.badrevs = set()
27453
8462d7f2c4fe verify: clean up weird error/warning lists
Matt Mackall <mpm@selenic.com>
parents: 27450
diff changeset
    50
        self.errors = 0
8462d7f2c4fe verify: clean up weird error/warning lists
Matt Mackall <mpm@selenic.com>
parents: 27450
diff changeset
    51
        self.warnings = 0
27444
6647401858ab verify: move widely used variables into class members
Durham Goode <durham@fb.com>
parents: 27443
diff changeset
    52
        self.havecl = len(repo.changelog) > 0
39244
73cf21b2e8a6 manifest: add getstorage() to manifestlog and use it globally
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38397
diff changeset
    53
        self.havemf = len(repo.manifestlog.getstorage(b'')) > 0
47139
f58a13c52726 revlog: split the `version` attribute into its two components
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47138
diff changeset
    54
        self.revlogv1 = repo.changelog._format_version != revlog.REVLOGV0
44113
e77b57e09bfa verify: avoid spurious integrity warnings in verbose mode (issue6172)
Matt Harbison <matt_harbison@yahoo.com>
parents: 44073
diff changeset
    55
        self.lrugetctx = util.lrucachefunc(repo.unfiltered().__getitem__)
27444
6647401858ab verify: move widely used variables into class members
Durham Goode <durham@fb.com>
parents: 27443
diff changeset
    56
        self.refersmf = False
27445
cc178057ab49 verify: move fncachewarned up to a class variable
Durham Goode <durham@fb.com>
parents: 27444
diff changeset
    57
        self.fncachewarned = False
32288
a2ab9ebcd85b verify: add a config option to skip certain flag processors
Jun Wu <quark@fb.com>
parents: 32250
diff changeset
    58
        # developer config: verify.skipflags
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
    59
        self.skipflags = repo.ui.configint(b'verify', b'skipflags')
37417
76d2115cb817 verify: allow suppressing warnings about extra files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 37410
diff changeset
    60
        self.warnorphanstorefiles = True
27444
6647401858ab verify: move widely used variables into class members
Durham Goode <durham@fb.com>
parents: 27443
diff changeset
    61
41862
c66037fb1bc5 verify: make the `warn` method private
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41861
diff changeset
    62
    def _warn(self, msg):
41861
a58748300e61 verify: document the `warn` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 40616
diff changeset
    63
        """record a "warning" level issue"""
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
    64
        self.ui.warn(msg + b"\n")
27453
8462d7f2c4fe verify: clean up weird error/warning lists
Matt Mackall <mpm@selenic.com>
parents: 27450
diff changeset
    65
        self.warnings += 1
27446
6b2c1a1871a6 verify: move warn() to a class level function
Durham Goode <durham@fb.com>
parents: 27445
diff changeset
    66
41864
7eaf4b1ac2a3 verify: make `err` a private method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41863
diff changeset
    67
    def _err(self, linkrev, msg, filename=None):
41863
9534f3cb6b6c verify: document the `err` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41862
diff changeset
    68
        """record a "error" level issue"""
27447
d1b91c10ce70 verify: move err() to be a class function
Durham Goode <durham@fb.com>
parents: 27446
diff changeset
    69
        if linkrev is not None:
d1b91c10ce70 verify: move err() to be a class function
Durham Goode <durham@fb.com>
parents: 27446
diff changeset
    70
            self.badrevs.add(linkrev)
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
    71
            linkrev = b"%d" % linkrev
27447
d1b91c10ce70 verify: move err() to be a class function
Durham Goode <durham@fb.com>
parents: 27446
diff changeset
    72
        else:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
    73
            linkrev = b'?'
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
    74
        msg = b"%s: %s" % (linkrev, msg)
27447
d1b91c10ce70 verify: move err() to be a class function
Durham Goode <durham@fb.com>
parents: 27446
diff changeset
    75
        if filename:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
    76
            msg = b"%s@%s" % (filename, msg)
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
    77
        self.ui.warn(b" " + msg + b"\n")
27453
8462d7f2c4fe verify: clean up weird error/warning lists
Matt Mackall <mpm@selenic.com>
parents: 27450
diff changeset
    78
        self.errors += 1
27447
d1b91c10ce70 verify: move err() to be a class function
Durham Goode <durham@fb.com>
parents: 27446
diff changeset
    79
41866
cfe08588d711 verify: make the `exc` method private
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41865
diff changeset
    80
    def _exc(self, linkrev, msg, inst, filename=None):
41865
5df8475c5343 verify: document the `exc` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41864
diff changeset
    81
        """record exception raised during the verify process"""
46696
ed0899e01628 verify: convert an exception to bytes before logging
Matt Harbison <matt_harbison@yahoo.com>
parents: 44113
diff changeset
    82
        fmsg = stringutil.forcebytestr(inst)
36577
d85ef895d5f6 verify: fix exception formatting bug in Python 3
Augie Fackler <augie@google.com>
parents: 36339
diff changeset
    83
        if not fmsg:
d85ef895d5f6 verify: fix exception formatting bug in Python 3
Augie Fackler <augie@google.com>
parents: 36339
diff changeset
    84
            fmsg = pycompat.byterepr(inst)
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
    85
        self._err(linkrev, b"%s: %s" % (msg, fmsg), filename)
27448
f4f2179077cb verify: move exc() function onto class
Durham Goode <durham@fb.com>
parents: 27447
diff changeset
    86
41874
1f412223f5bb verify: rename the `checklog` to `_checkrevlog`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41873
diff changeset
    87
    def _checkrevlog(self, obj, name, linkrev):
41873
08d977451f26 verify: document the `checklog` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41871
diff changeset
    88
        """verify high level property of a revlog
08d977451f26 verify: document the `checklog` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41871
diff changeset
    89
08d977451f26 verify: document the `checklog` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41871
diff changeset
    90
        - revlog is present,
08d977451f26 verify: document the `checklog` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41871
diff changeset
    91
        - revlog is non-empty,
08d977451f26 verify: document the `checklog` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41871
diff changeset
    92
        - sizes (index and data) are correct,
08d977451f26 verify: document the `checklog` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41871
diff changeset
    93
        - revlog's format version is correct.
08d977451f26 verify: document the `checklog` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41871
diff changeset
    94
        """
27642
f6457349985b verify: move checklog() onto class
Durham Goode <durham@fb.com>
parents: 27453
diff changeset
    95
        if not len(obj) and (self.havecl or self.havemf):
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
    96
            self._err(linkrev, _(b"empty or missing %s") % name)
27642
f6457349985b verify: move checklog() onto class
Durham Goode <durham@fb.com>
parents: 27453
diff changeset
    97
            return
f6457349985b verify: move checklog() onto class
Durham Goode <durham@fb.com>
parents: 27453
diff changeset
    98
f6457349985b verify: move checklog() onto class
Durham Goode <durham@fb.com>
parents: 27453
diff changeset
    99
        d = obj.checksize()
f6457349985b verify: move checklog() onto class
Durham Goode <durham@fb.com>
parents: 27453
diff changeset
   100
        if d[0]:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   101
            self._err(None, _(b"data length off by %d bytes") % d[0], name)
27642
f6457349985b verify: move checklog() onto class
Durham Goode <durham@fb.com>
parents: 27453
diff changeset
   102
        if d[1]:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   103
            self._err(None, _(b"index contains %d extra bytes") % d[1], name)
27642
f6457349985b verify: move checklog() onto class
Durham Goode <durham@fb.com>
parents: 27453
diff changeset
   104
47139
f58a13c52726 revlog: split the `version` attribute into its two components
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47138
diff changeset
   105
        if obj._format_version != revlog.REVLOGV0:
27642
f6457349985b verify: move checklog() onto class
Durham Goode <durham@fb.com>
parents: 27453
diff changeset
   106
            if not self.revlogv1:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   107
                self._warn(_(b"warning: `%s' uses revlog format 1") % name)
27642
f6457349985b verify: move checklog() onto class
Durham Goode <durham@fb.com>
parents: 27453
diff changeset
   108
        elif self.revlogv1:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   109
            self._warn(_(b"warning: `%s' uses revlog format 0") % name)
27642
f6457349985b verify: move checklog() onto class
Durham Goode <durham@fb.com>
parents: 27453
diff changeset
   110
41871
9e737ca539f6 verify: make `checkentry` a private method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41870
diff changeset
   111
    def _checkentry(self, obj, i, node, seen, linkrevs, f):
41870
00c9fde75c1a verify: document the `checkentry` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41869
diff changeset
   112
        """verify a single revlog entry
00c9fde75c1a verify: document the `checkentry` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41869
diff changeset
   113
00c9fde75c1a verify: document the `checkentry` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41869
diff changeset
   114
        arguments are:
00c9fde75c1a verify: document the `checkentry` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41869
diff changeset
   115
        - obj:      the source revlog
00c9fde75c1a verify: document the `checkentry` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41869
diff changeset
   116
        - i:        the revision number
47353
5ed2aaab58b0 verify: align a comment line
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47139
diff changeset
   117
        - node:     the revision node id
41870
00c9fde75c1a verify: document the `checkentry` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41869
diff changeset
   118
        - seen:     nodes previously seen for this revlog
00c9fde75c1a verify: document the `checkentry` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41869
diff changeset
   119
        - linkrevs: [changelog-revisions] introducing "node"
00c9fde75c1a verify: document the `checkentry` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41869
diff changeset
   120
        - f:        string label ("changelog", "manifest", or filename)
00c9fde75c1a verify: document the `checkentry` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41869
diff changeset
   121
00c9fde75c1a verify: document the `checkentry` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41869
diff changeset
   122
        Performs the following checks:
00c9fde75c1a verify: document the `checkentry` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41869
diff changeset
   123
        - linkrev points to an existing changelog revision,
00c9fde75c1a verify: document the `checkentry` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41869
diff changeset
   124
        - linkrev points to a changelog revision that introduces this revision,
00c9fde75c1a verify: document the `checkentry` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41869
diff changeset
   125
        - linkrev points to the lowest of these changesets,
00c9fde75c1a verify: document the `checkentry` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41869
diff changeset
   126
        - both parents exist in the revlog,
00c9fde75c1a verify: document the `checkentry` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41869
diff changeset
   127
        - the revision is not duplicated.
00c9fde75c1a verify: document the `checkentry` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41869
diff changeset
   128
00c9fde75c1a verify: document the `checkentry` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41869
diff changeset
   129
        Return the linkrev of the revision (or None for changelog's revisions).
00c9fde75c1a verify: document the `checkentry` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41869
diff changeset
   130
        """
27643
62ce86fcfd06 verify: move checkentry() to be a class function
Durham Goode <durham@fb.com>
parents: 27642
diff changeset
   131
        lr = obj.linkrev(obj.rev(node))
62ce86fcfd06 verify: move checkentry() to be a class function
Durham Goode <durham@fb.com>
parents: 27642
diff changeset
   132
        if lr < 0 or (self.havecl and lr not in linkrevs):
62ce86fcfd06 verify: move checkentry() to be a class function
Durham Goode <durham@fb.com>
parents: 27642
diff changeset
   133
            if lr < 0 or lr >= len(self.repo.changelog):
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   134
                msg = _(b"rev %d points to nonexistent changeset %d")
27643
62ce86fcfd06 verify: move checkentry() to be a class function
Durham Goode <durham@fb.com>
parents: 27642
diff changeset
   135
            else:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   136
                msg = _(b"rev %d points to unexpected changeset %d")
41864
7eaf4b1ac2a3 verify: make `err` a private method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41863
diff changeset
   137
            self._err(None, msg % (i, lr), f)
27643
62ce86fcfd06 verify: move checkentry() to be a class function
Durham Goode <durham@fb.com>
parents: 27642
diff changeset
   138
            if linkrevs:
62ce86fcfd06 verify: move checkentry() to be a class function
Durham Goode <durham@fb.com>
parents: 27642
diff changeset
   139
                if f and len(linkrevs) > 1:
62ce86fcfd06 verify: move checkentry() to be a class function
Durham Goode <durham@fb.com>
parents: 27642
diff changeset
   140
                    try:
62ce86fcfd06 verify: move checkentry() to be a class function
Durham Goode <durham@fb.com>
parents: 27642
diff changeset
   141
                        # attempt to filter down to real linkrevs
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   142
                        linkrevs = [
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   143
                            l
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   144
                            for l in linkrevs
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   145
                            if self.lrugetctx(l)[f].filenode() == node
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   146
                        ]
27643
62ce86fcfd06 verify: move checkentry() to be a class function
Durham Goode <durham@fb.com>
parents: 27642
diff changeset
   147
                    except Exception:
62ce86fcfd06 verify: move checkentry() to be a class function
Durham Goode <durham@fb.com>
parents: 27642
diff changeset
   148
                        pass
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   149
                self._warn(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   150
                    _(b" (expected %s)")
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   151
                    % b" ".join(map(pycompat.bytestr, linkrevs))
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   152
                )
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   153
            lr = None  # can't be trusted
27643
62ce86fcfd06 verify: move checkentry() to be a class function
Durham Goode <durham@fb.com>
parents: 27642
diff changeset
   154
62ce86fcfd06 verify: move checkentry() to be a class function
Durham Goode <durham@fb.com>
parents: 27642
diff changeset
   155
        try:
62ce86fcfd06 verify: move checkentry() to be a class function
Durham Goode <durham@fb.com>
parents: 27642
diff changeset
   156
            p1, p2 = obj.parents(node)
47012
d55b71393907 node: replace nullid and friends with nodeconstants class
Joerg Sonnenberger <joerg@bec.de>
parents: 46895
diff changeset
   157
            if p1 not in seen and p1 != self.repo.nullid:
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   158
                self._err(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   159
                    lr,
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   160
                    _(b"unknown parent 1 %s of %s") % (short(p1), short(node)),
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   161
                    f,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   162
                )
47012
d55b71393907 node: replace nullid and friends with nodeconstants class
Joerg Sonnenberger <joerg@bec.de>
parents: 46895
diff changeset
   163
            if p2 not in seen and p2 != self.repo.nullid:
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   164
                self._err(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   165
                    lr,
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   166
                    _(b"unknown parent 2 %s of %s") % (short(p2), short(node)),
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   167
                    f,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   168
                )
27643
62ce86fcfd06 verify: move checkentry() to be a class function
Durham Goode <durham@fb.com>
parents: 27642
diff changeset
   169
        except Exception as inst:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   170
            self._exc(lr, _(b"checking parents of %s") % short(node), inst, f)
27643
62ce86fcfd06 verify: move checkentry() to be a class function
Durham Goode <durham@fb.com>
parents: 27642
diff changeset
   171
62ce86fcfd06 verify: move checkentry() to be a class function
Durham Goode <durham@fb.com>
parents: 27642
diff changeset
   172
        if node in seen:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   173
            self._err(lr, _(b"duplicate revision %d (%d)") % (i, seen[node]), f)
27643
62ce86fcfd06 verify: move checkentry() to be a class function
Durham Goode <durham@fb.com>
parents: 27642
diff changeset
   174
        seen[node] = i
62ce86fcfd06 verify: move checkentry() to be a class function
Durham Goode <durham@fb.com>
parents: 27642
diff changeset
   175
        return lr
62ce86fcfd06 verify: move checkentry() to be a class function
Durham Goode <durham@fb.com>
parents: 27642
diff changeset
   176
27444
6647401858ab verify: move widely used variables into class members
Durham Goode <durham@fb.com>
parents: 27443
diff changeset
   177
    def verify(self):
41867
e8c4a9f5b986 verify: minimal documentation for `verifier.verify`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41866
diff changeset
   178
        """verify the content of the Mercurial repository
e8c4a9f5b986 verify: minimal documentation for `verifier.verify`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41866
diff changeset
   179
e8c4a9f5b986 verify: minimal documentation for `verifier.verify`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41866
diff changeset
   180
        This method run all verifications, displaying issues as they are found.
e8c4a9f5b986 verify: minimal documentation for `verifier.verify`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41866
diff changeset
   181
41868
567892b4306c verify: explicitly return 0 if no error are encountered
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41867
diff changeset
   182
        return 1 if any error have been encountered, 0 otherwise."""
41869
4da2261e949b verify: add some inline documentation to the top level `verify` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41868
diff changeset
   183
        # initial validation and generic report
27444
6647401858ab verify: move widely used variables into class members
Durham Goode <durham@fb.com>
parents: 27443
diff changeset
   184
        repo = self.repo
27443
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   185
        ui = repo.ui
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   186
        if not repo.url().startswith(b'file:'):
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   187
            raise error.Abort(_(b"cannot verify bundle or remote repos"))
6752
e79a8f36c2a5 verify: lots of refactoring
Matt Mackall <mpm@selenic.com>
parents: 6751
diff changeset
   188
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   189
        if os.path.exists(repo.sjoin(b"journal")):
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   190
            ui.warn(_(b"abandoned transaction found - run hg recover\n"))
27443
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   191
27648
e72e669dd51f verify: get rid of some unnecessary local variables
Durham Goode <durham@fb.com>
parents: 27647
diff changeset
   192
        if ui.verbose or not self.revlogv1:
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   193
            ui.status(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   194
                _(b"repository uses revlog format %d\n")
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   195
                % (self.revlogv1 and 1 or 0)
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   196
            )
27443
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   197
41869
4da2261e949b verify: add some inline documentation to the top level `verify` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41868
diff changeset
   198
        # data verification
27695
fb0cc863d172 verify: replace "output parameters" by return values
Martin von Zweigbergk <martinvonz@google.com>
parents: 27648
diff changeset
   199
        mflinkrevs, filelinkrevs = self._verifychangelog()
fb0cc863d172 verify: replace "output parameters" by return values
Martin von Zweigbergk <martinvonz@google.com>
parents: 27648
diff changeset
   200
        filenodes = self._verifymanifest(mflinkrevs)
28111
06205989264b verify: move cross-checking of changeset/manifest out of _crosscheckfiles()
Martin von Zweigbergk <martinvonz@google.com>
parents: 28007
diff changeset
   201
        del mflinkrevs
06205989264b verify: move cross-checking of changeset/manifest out of _crosscheckfiles()
Martin von Zweigbergk <martinvonz@google.com>
parents: 28007
diff changeset
   202
        self._crosscheckfiles(filelinkrevs, filenodes)
27647
2c2858f3c1bb verify: move changelog verificaiton to its own function
Durham Goode <durham@fb.com>
parents: 27646
diff changeset
   203
        totalfiles, filerevisions = self._verifyfiles(filenodes, filelinkrevs)
2c2858f3c1bb verify: move changelog verificaiton to its own function
Durham Goode <durham@fb.com>
parents: 27646
diff changeset
   204
41869
4da2261e949b verify: add some inline documentation to the top level `verify` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41868
diff changeset
   205
        # final report
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   206
        ui.status(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   207
            _(b"checked %d changesets with %d changes to %d files\n")
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   208
            % (len(repo.changelog), filerevisions, totalfiles)
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   209
        )
27647
2c2858f3c1bb verify: move changelog verificaiton to its own function
Durham Goode <durham@fb.com>
parents: 27646
diff changeset
   210
        if self.warnings:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   211
            ui.warn(_(b"%d warnings encountered!\n") % self.warnings)
27647
2c2858f3c1bb verify: move changelog verificaiton to its own function
Durham Goode <durham@fb.com>
parents: 27646
diff changeset
   212
        if self.fncachewarned:
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   213
            ui.warn(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   214
                _(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   215
                    b'hint: run "hg debugrebuildfncache" to recover from '
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   216
                    b'corrupt fncache\n'
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   217
                )
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   218
            )
27647
2c2858f3c1bb verify: move changelog verificaiton to its own function
Durham Goode <durham@fb.com>
parents: 27646
diff changeset
   219
        if self.errors:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   220
            ui.warn(_(b"%d integrity errors encountered!\n") % self.errors)
27648
e72e669dd51f verify: get rid of some unnecessary local variables
Durham Goode <durham@fb.com>
parents: 27647
diff changeset
   221
            if self.badrevs:
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   222
                ui.warn(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   223
                    _(b"(first damaged changeset appears to be %d)\n")
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   224
                    % min(self.badrevs)
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   225
                )
27647
2c2858f3c1bb verify: move changelog verificaiton to its own function
Durham Goode <durham@fb.com>
parents: 27646
diff changeset
   226
            return 1
41868
567892b4306c verify: explicitly return 0 if no error are encountered
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41867
diff changeset
   227
        return 0
27647
2c2858f3c1bb verify: move changelog verificaiton to its own function
Durham Goode <durham@fb.com>
parents: 27646
diff changeset
   228
27695
fb0cc863d172 verify: replace "output parameters" by return values
Martin von Zweigbergk <martinvonz@google.com>
parents: 27648
diff changeset
   229
    def _verifychangelog(self):
41875
2f1f475e9646 verify: document `_verifychangelog`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41874
diff changeset
   230
        """verify the changelog of a repository
2f1f475e9646 verify: document `_verifychangelog`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41874
diff changeset
   231
2f1f475e9646 verify: document `_verifychangelog`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41874
diff changeset
   232
        The following checks are performed:
2f1f475e9646 verify: document `_verifychangelog`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41874
diff changeset
   233
        - all of `_checkrevlog` checks,
2f1f475e9646 verify: document `_verifychangelog`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41874
diff changeset
   234
        - all of `_checkentry` checks (for each revisions),
2f1f475e9646 verify: document `_verifychangelog`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41874
diff changeset
   235
        - each revision can be read.
2f1f475e9646 verify: document `_verifychangelog`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41874
diff changeset
   236
2f1f475e9646 verify: document `_verifychangelog`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41874
diff changeset
   237
        The function returns some of the data observed in the changesets as a
2f1f475e9646 verify: document `_verifychangelog`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41874
diff changeset
   238
        (mflinkrevs, filelinkrevs) tuples:
2f1f475e9646 verify: document `_verifychangelog`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41874
diff changeset
   239
        - mflinkrevs:   is a { manifest-node -> [changelog-rev] } mapping
2f1f475e9646 verify: document `_verifychangelog`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41874
diff changeset
   240
        - filelinkrevs: is a { file-path -> [changelog-rev] } mapping
2f1f475e9646 verify: document `_verifychangelog`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41874
diff changeset
   241
2f1f475e9646 verify: document `_verifychangelog`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41874
diff changeset
   242
        If a matcher was specified, filelinkrevs will only contains matched
2f1f475e9646 verify: document `_verifychangelog`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41874
diff changeset
   243
        files.
2f1f475e9646 verify: document `_verifychangelog`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41874
diff changeset
   244
        """
27647
2c2858f3c1bb verify: move changelog verificaiton to its own function
Durham Goode <durham@fb.com>
parents: 27646
diff changeset
   245
        ui = self.ui
2c2858f3c1bb verify: move changelog verificaiton to its own function
Durham Goode <durham@fb.com>
parents: 27646
diff changeset
   246
        repo = self.repo
30866
5249b6470de9 verify: replace _validpath() by matcher
Martin von Zweigbergk <martinvonz@google.com>
parents: 30393
diff changeset
   247
        match = self.match
27647
2c2858f3c1bb verify: move changelog verificaiton to its own function
Durham Goode <durham@fb.com>
parents: 27646
diff changeset
   248
        cl = repo.changelog
2c2858f3c1bb verify: move changelog verificaiton to its own function
Durham Goode <durham@fb.com>
parents: 27646
diff changeset
   249
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   250
        ui.status(_(b"checking changesets\n"))
27695
fb0cc863d172 verify: replace "output parameters" by return values
Martin von Zweigbergk <martinvonz@google.com>
parents: 27648
diff changeset
   251
        mflinkrevs = {}
fb0cc863d172 verify: replace "output parameters" by return values
Martin von Zweigbergk <martinvonz@google.com>
parents: 27648
diff changeset
   252
        filelinkrevs = {}
27443
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   253
        seen = {}
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   254
        self._checkrevlog(cl, b"changelog", 0)
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   255
        progress = ui.makeprogress(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   256
            _(b'checking'), unit=_(b'changesets'), total=len(repo)
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   257
        )
27443
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   258
        for i in repo:
38397
1249475f0bd6 verify: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents: 38396
diff changeset
   259
            progress.update(i)
27443
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   260
            n = cl.node(i)
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   261
            self._checkentry(cl, i, n, seen, [i], b"changelog")
2778
fdc232d8a193 Move repo.verify
Matt Mackall <mpm@selenic.com>
parents:
diff changeset
   262
27443
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   263
            try:
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   264
                changes = cl.read(n)
47012
d55b71393907 node: replace nullid and friends with nodeconstants class
Joerg Sonnenberger <joerg@bec.de>
parents: 46895
diff changeset
   265
                if changes[0] != self.repo.nullid:
27443
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   266
                    mflinkrevs.setdefault(changes[0], []).append(i)
27444
6647401858ab verify: move widely used variables into class members
Durham Goode <durham@fb.com>
parents: 27443
diff changeset
   267
                    self.refersmf = True
27443
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   268
                for f in changes[3]:
30866
5249b6470de9 verify: replace _validpath() by matcher
Martin von Zweigbergk <martinvonz@google.com>
parents: 30393
diff changeset
   269
                    if match(f):
27443
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   270
                        filelinkrevs.setdefault(_normpath(f), []).append(i)
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   271
            except Exception as inst:
27444
6647401858ab verify: move widely used variables into class members
Durham Goode <durham@fb.com>
parents: 27443
diff changeset
   272
                self.refersmf = True
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   273
                self._exc(i, _(b"unpacking changeset %s") % short(n), inst)
38397
1249475f0bd6 verify: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents: 38396
diff changeset
   274
        progress.complete()
27695
fb0cc863d172 verify: replace "output parameters" by return values
Martin von Zweigbergk <martinvonz@google.com>
parents: 27648
diff changeset
   275
        return mflinkrevs, filelinkrevs
2778
fdc232d8a193 Move repo.verify
Matt Mackall <mpm@selenic.com>
parents:
diff changeset
   276
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   277
    def _verifymanifest(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   278
        self, mflinkrevs, dir=b"", storefiles=None, subdirprogress=None
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   279
    ):
41876
5ad5a70df2f7 verify: document the `_verifymanifest` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41875
diff changeset
   280
        """verify the manifestlog content
5ad5a70df2f7 verify: document the `_verifymanifest` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41875
diff changeset
   281
5ad5a70df2f7 verify: document the `_verifymanifest` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41875
diff changeset
   282
        Inputs:
5ad5a70df2f7 verify: document the `_verifymanifest` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41875
diff changeset
   283
        - mflinkrevs:     a {manifest-node -> [changelog-revisions]} mapping
5ad5a70df2f7 verify: document the `_verifymanifest` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41875
diff changeset
   284
        - dir:            a subdirectory to check (for tree manifest repo)
5ad5a70df2f7 verify: document the `_verifymanifest` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41875
diff changeset
   285
        - storefiles:     set of currently "orphan" files.
5ad5a70df2f7 verify: document the `_verifymanifest` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41875
diff changeset
   286
        - subdirprogress: a progress object
5ad5a70df2f7 verify: document the `_verifymanifest` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41875
diff changeset
   287
5ad5a70df2f7 verify: document the `_verifymanifest` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41875
diff changeset
   288
        This function checks:
5ad5a70df2f7 verify: document the `_verifymanifest` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41875
diff changeset
   289
        * all of `_checkrevlog` checks (for all manifest related revlogs)
5ad5a70df2f7 verify: document the `_verifymanifest` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41875
diff changeset
   290
        * all of `_checkentry` checks (for all manifest related revisions)
5ad5a70df2f7 verify: document the `_verifymanifest` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41875
diff changeset
   291
        * nodes for subdirectory exists in the sub-directory manifest
5ad5a70df2f7 verify: document the `_verifymanifest` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41875
diff changeset
   292
        * each manifest entries have a file path
5ad5a70df2f7 verify: document the `_verifymanifest` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41875
diff changeset
   293
        * each manifest node refered in mflinkrevs exist in the manifest log
5ad5a70df2f7 verify: document the `_verifymanifest` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41875
diff changeset
   294
5ad5a70df2f7 verify: document the `_verifymanifest` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41875
diff changeset
   295
        If tree manifest is in use and a matchers is specified, only the
5ad5a70df2f7 verify: document the `_verifymanifest` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41875
diff changeset
   296
        sub-directories matching it will be verified.
5ad5a70df2f7 verify: document the `_verifymanifest` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41875
diff changeset
   297
5ad5a70df2f7 verify: document the `_verifymanifest` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41875
diff changeset
   298
        return a two level mapping:
5ad5a70df2f7 verify: document the `_verifymanifest` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41875
diff changeset
   299
            {"path" -> { filenode -> changelog-revision}}
5ad5a70df2f7 verify: document the `_verifymanifest` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41875
diff changeset
   300
5ad5a70df2f7 verify: document the `_verifymanifest` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41875
diff changeset
   301
        This mapping primarily contains entries for every files in the
5ad5a70df2f7 verify: document the `_verifymanifest` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41875
diff changeset
   302
        repository. In addition, when tree-manifest is used, it also contains
5ad5a70df2f7 verify: document the `_verifymanifest` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41875
diff changeset
   303
        sub-directory entries.
5ad5a70df2f7 verify: document the `_verifymanifest` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41875
diff changeset
   304
5ad5a70df2f7 verify: document the `_verifymanifest` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41875
diff changeset
   305
        If a matcher is provided, only matching paths will be included.
5ad5a70df2f7 verify: document the `_verifymanifest` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41875
diff changeset
   306
        """
27646
8f43793382c6 verify: move manifest verification to its own function
Durham Goode <durham@fb.com>
parents: 27645
diff changeset
   307
        repo = self.repo
8f43793382c6 verify: move manifest verification to its own function
Durham Goode <durham@fb.com>
parents: 27645
diff changeset
   308
        ui = self.ui
30866
5249b6470de9 verify: replace _validpath() by matcher
Martin von Zweigbergk <martinvonz@google.com>
parents: 30393
diff changeset
   309
        match = self.match
30295
f65faa4422c8 manifest: remove manifest.readshallowdelta
Durham Goode <durham@fb.com>
parents: 28467
diff changeset
   310
        mfl = self.repo.manifestlog
39244
73cf21b2e8a6 manifest: add getstorage() to manifestlog and use it globally
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38397
diff changeset
   311
        mf = mfl.getstorage(dir)
27646
8f43793382c6 verify: move manifest verification to its own function
Durham Goode <durham@fb.com>
parents: 27645
diff changeset
   312
28203
7297e9e13a8a verify: check directory manifests
Martin von Zweigbergk <martinvonz@google.com>
parents: 28115
diff changeset
   313
        if not dir:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   314
            self.ui.status(_(b"checking manifests\n"))
28203
7297e9e13a8a verify: check directory manifests
Martin von Zweigbergk <martinvonz@google.com>
parents: 28115
diff changeset
   315
27695
fb0cc863d172 verify: replace "output parameters" by return values
Martin von Zweigbergk <martinvonz@google.com>
parents: 27648
diff changeset
   316
        filenodes = {}
28203
7297e9e13a8a verify: check directory manifests
Martin von Zweigbergk <martinvonz@google.com>
parents: 28115
diff changeset
   317
        subdirnodes = {}
27443
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   318
        seen = {}
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   319
        label = b"manifest"
28203
7297e9e13a8a verify: check directory manifests
Martin von Zweigbergk <martinvonz@google.com>
parents: 28115
diff changeset
   320
        if dir:
7297e9e13a8a verify: check directory manifests
Martin von Zweigbergk <martinvonz@google.com>
parents: 28115
diff changeset
   321
            label = dir
28204
962921c330b0 verify: check for orphaned dirlogs
Martin von Zweigbergk <martinvonz@google.com>
parents: 28203
diff changeset
   322
            revlogfiles = mf.files()
962921c330b0 verify: check for orphaned dirlogs
Martin von Zweigbergk <martinvonz@google.com>
parents: 28203
diff changeset
   323
            storefiles.difference_update(revlogfiles)
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   324
            if subdirprogress:  # should be true since we're in a subdirectory
38396
0ddbe03c5aaa verify: use progress helper for subdirectory progress
Martin von Zweigbergk <martinvonz@google.com>
parents: 37417
diff changeset
   325
                subdirprogress.increment()
27444
6647401858ab verify: move widely used variables into class members
Durham Goode <durham@fb.com>
parents: 27443
diff changeset
   326
        if self.refersmf:
27443
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   327
            # Do not check manifest if there are only changelog entries with
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   328
            # null manifests.
47138
bc138f2a2e47 verify: pass a revlog to `_checkrevlog` in `_verifymanifest`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47012
diff changeset
   329
            self._checkrevlog(mf._revlog, label, 0)
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   330
        progress = ui.makeprogress(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   331
            _(b'checking'), unit=_(b'manifests'), total=len(mf)
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   332
        )
27443
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   333
        for i in mf:
28203
7297e9e13a8a verify: check directory manifests
Martin von Zweigbergk <martinvonz@google.com>
parents: 28115
diff changeset
   334
            if not dir:
38397
1249475f0bd6 verify: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents: 38396
diff changeset
   335
                progress.update(i)
27443
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   336
            n = mf.node(i)
41871
9e737ca539f6 verify: make `checkentry` a private method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41870
diff changeset
   337
            lr = self._checkentry(mf, i, n, seen, mflinkrevs.get(n, []), label)
27443
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   338
            if n in mflinkrevs:
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   339
                del mflinkrevs[n]
28203
7297e9e13a8a verify: check directory manifests
Martin von Zweigbergk <martinvonz@google.com>
parents: 28115
diff changeset
   340
            elif dir:
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   341
                self._err(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   342
                    lr,
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   343
                    _(b"%s not in parent-directory manifest") % short(n),
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   344
                    label,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   345
                )
27443
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   346
            else:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   347
                self._err(lr, _(b"%s not in changesets") % short(n), label)
2778
fdc232d8a193 Move repo.verify
Matt Mackall <mpm@selenic.com>
parents:
diff changeset
   348
27443
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   349
            try:
30295
f65faa4422c8 manifest: remove manifest.readshallowdelta
Durham Goode <durham@fb.com>
parents: 28467
diff changeset
   350
                mfdelta = mfl.get(dir, n).readdelta(shallow=True)
f65faa4422c8 manifest: remove manifest.readshallowdelta
Durham Goode <durham@fb.com>
parents: 28467
diff changeset
   351
                for f, fn, fl in mfdelta.iterentries():
27443
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   352
                    if not f:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   353
                        self._err(lr, _(b"entry without name in manifest"))
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   354
                    elif f == b"/dev/null":  # ignore this in very old repos
28203
7297e9e13a8a verify: check directory manifests
Martin von Zweigbergk <martinvonz@google.com>
parents: 28115
diff changeset
   355
                        continue
7297e9e13a8a verify: check directory manifests
Martin von Zweigbergk <martinvonz@google.com>
parents: 28115
diff changeset
   356
                    fullpath = dir + _normpath(f)
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   357
                    if fl == b't':
30866
5249b6470de9 verify: replace _validpath() by matcher
Martin von Zweigbergk <martinvonz@google.com>
parents: 30393
diff changeset
   358
                        if not match.visitdir(fullpath):
5249b6470de9 verify: replace _validpath() by matcher
Martin von Zweigbergk <martinvonz@google.com>
parents: 30393
diff changeset
   359
                            continue
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   360
                        subdirnodes.setdefault(fullpath + b'/', {}).setdefault(
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   361
                            fn, []
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   362
                        ).append(lr)
28203
7297e9e13a8a verify: check directory manifests
Martin von Zweigbergk <martinvonz@google.com>
parents: 28115
diff changeset
   363
                    else:
30866
5249b6470de9 verify: replace _validpath() by matcher
Martin von Zweigbergk <martinvonz@google.com>
parents: 30393
diff changeset
   364
                        if not match(fullpath):
5249b6470de9 verify: replace _validpath() by matcher
Martin von Zweigbergk <martinvonz@google.com>
parents: 30393
diff changeset
   365
                            continue
28203
7297e9e13a8a verify: check directory manifests
Martin von Zweigbergk <martinvonz@google.com>
parents: 28115
diff changeset
   366
                        filenodes.setdefault(fullpath, {}).setdefault(fn, lr)
27443
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   367
            except Exception as inst:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   368
                self._exc(lr, _(b"reading delta %s") % short(n), inst, label)
42157
7755b89cadaf verify: also check full manifest validity during verify runs
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 42156
diff changeset
   369
            if self._level >= VERIFY_FULL:
7755b89cadaf verify: also check full manifest validity during verify runs
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 42156
diff changeset
   370
                try:
7755b89cadaf verify: also check full manifest validity during verify runs
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 42156
diff changeset
   371
                    # Various issues can affect manifest. So we read each full
7755b89cadaf verify: also check full manifest validity during verify runs
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 42156
diff changeset
   372
                    # text from storage. This triggers the checks from the core
7755b89cadaf verify: also check full manifest validity during verify runs
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 42156
diff changeset
   373
                    # code (eg: hash verification, filename are ordered, etc.)
7755b89cadaf verify: also check full manifest validity during verify runs
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 42156
diff changeset
   374
                    mfdelta = mfl.get(dir, n).read()
7755b89cadaf verify: also check full manifest validity during verify runs
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 42156
diff changeset
   375
                except Exception as inst:
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   376
                    self._exc(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   377
                        lr,
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   378
                        _(b"reading full manifest %s") % short(n),
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   379
                        inst,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   380
                        label,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   381
                    )
42157
7755b89cadaf verify: also check full manifest validity during verify runs
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 42156
diff changeset
   382
28203
7297e9e13a8a verify: check directory manifests
Martin von Zweigbergk <martinvonz@google.com>
parents: 28115
diff changeset
   383
        if not dir:
38397
1249475f0bd6 verify: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents: 38396
diff changeset
   384
            progress.complete()
27443
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   385
28111
06205989264b verify: move cross-checking of changeset/manifest out of _crosscheckfiles()
Martin von Zweigbergk <martinvonz@google.com>
parents: 28007
diff changeset
   386
        if self.havemf:
41877
9c5a6af74afa verify: small refactoring and documentation in `_verifymanifest`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41876
diff changeset
   387
            # since we delete entry in `mflinkrevs` during iteration, any
9c5a6af74afa verify: small refactoring and documentation in `_verifymanifest`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41876
diff changeset
   388
            # remaining entries are "missing". We need to issue errors for them.
9c5a6af74afa verify: small refactoring and documentation in `_verifymanifest`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41876
diff changeset
   389
            changesetpairs = [(c, m) for m in mflinkrevs for c in mflinkrevs[m]]
9c5a6af74afa verify: small refactoring and documentation in `_verifymanifest`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41876
diff changeset
   390
            for c, m in sorted(changesetpairs):
28203
7297e9e13a8a verify: check directory manifests
Martin von Zweigbergk <martinvonz@google.com>
parents: 28115
diff changeset
   391
                if dir:
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   392
                    self._err(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   393
                        c,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   394
                        _(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   395
                            b"parent-directory manifest refers to unknown"
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   396
                            b" revision %s"
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   397
                        )
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   398
                        % short(m),
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   399
                        label,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   400
                    )
28203
7297e9e13a8a verify: check directory manifests
Martin von Zweigbergk <martinvonz@google.com>
parents: 28115
diff changeset
   401
                else:
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   402
                    self._err(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   403
                        c,
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   404
                        _(b"changeset refers to unknown revision %s")
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   405
                        % short(m),
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   406
                        label,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   407
                    )
28203
7297e9e13a8a verify: check directory manifests
Martin von Zweigbergk <martinvonz@google.com>
parents: 28115
diff changeset
   408
7297e9e13a8a verify: check directory manifests
Martin von Zweigbergk <martinvonz@google.com>
parents: 28115
diff changeset
   409
        if not dir and subdirnodes:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   410
            self.ui.status(_(b"checking directory manifests\n"))
28204
962921c330b0 verify: check for orphaned dirlogs
Martin von Zweigbergk <martinvonz@google.com>
parents: 28203
diff changeset
   411
            storefiles = set()
28205
53f42c8d5f71 verify: show progress while verifying dirlogs
Martin von Zweigbergk <martinvonz@google.com>
parents: 28204
diff changeset
   412
            subdirs = set()
28204
962921c330b0 verify: check for orphaned dirlogs
Martin von Zweigbergk <martinvonz@google.com>
parents: 28203
diff changeset
   413
            revlogv1 = self.revlogv1
46895
6085b7f1536d store: also return some information about the type of file `walk` found
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46819
diff changeset
   414
            for t, f, f2, size in repo.store.datafiles():
28204
962921c330b0 verify: check for orphaned dirlogs
Martin von Zweigbergk <martinvonz@google.com>
parents: 28203
diff changeset
   415
                if not f:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   416
                    self._err(None, _(b"cannot decode filename '%s'") % f2)
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   417
                elif (size > 0 or not revlogv1) and f.startswith(b'meta/'):
28204
962921c330b0 verify: check for orphaned dirlogs
Martin von Zweigbergk <martinvonz@google.com>
parents: 28203
diff changeset
   418
                    storefiles.add(_normpath(f))
28205
53f42c8d5f71 verify: show progress while verifying dirlogs
Martin von Zweigbergk <martinvonz@google.com>
parents: 28204
diff changeset
   419
                    subdirs.add(os.path.dirname(f))
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   420
            subdirprogress = ui.makeprogress(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   421
                _(b'checking'), unit=_(b'manifests'), total=len(subdirs)
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   422
            )
28204
962921c330b0 verify: check for orphaned dirlogs
Martin von Zweigbergk <martinvonz@google.com>
parents: 28203
diff changeset
   423
43106
d783f945a701 py3: finish porting iteritems() to pycompat and remove source transformer
Gregory Szorc <gregory.szorc@gmail.com>
parents: 43077
diff changeset
   424
        for subdir, linkrevs in pycompat.iteritems(subdirnodes):
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   425
            subdirfilenodes = self._verifymanifest(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   426
                linkrevs, subdir, storefiles, subdirprogress
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   427
            )
43106
d783f945a701 py3: finish porting iteritems() to pycompat and remove source transformer
Gregory Szorc <gregory.szorc@gmail.com>
parents: 43077
diff changeset
   428
            for f, onefilenodes in pycompat.iteritems(subdirfilenodes):
28203
7297e9e13a8a verify: check directory manifests
Martin von Zweigbergk <martinvonz@google.com>
parents: 28115
diff changeset
   429
                filenodes.setdefault(f, {}).update(onefilenodes)
28111
06205989264b verify: move cross-checking of changeset/manifest out of _crosscheckfiles()
Martin von Zweigbergk <martinvonz@google.com>
parents: 28007
diff changeset
   430
28204
962921c330b0 verify: check for orphaned dirlogs
Martin von Zweigbergk <martinvonz@google.com>
parents: 28203
diff changeset
   431
        if not dir and subdirnodes:
46790
5137896602d9 typing: add an assertion to verify.py to appease pytype
Matt Harbison <matt_harbison@yahoo.com>
parents: 46696
diff changeset
   432
            assert subdirprogress is not None  # help pytype
38396
0ddbe03c5aaa verify: use progress helper for subdirectory progress
Martin von Zweigbergk <martinvonz@google.com>
parents: 37417
diff changeset
   433
            subdirprogress.complete()
37417
76d2115cb817 verify: allow suppressing warnings about extra files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 37410
diff changeset
   434
            if self.warnorphanstorefiles:
76d2115cb817 verify: allow suppressing warnings about extra files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 37410
diff changeset
   435
                for f in sorted(storefiles):
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   436
                    self._warn(_(b"warning: orphan data file '%s'") % f)
28204
962921c330b0 verify: check for orphaned dirlogs
Martin von Zweigbergk <martinvonz@google.com>
parents: 28203
diff changeset
   437
27695
fb0cc863d172 verify: replace "output parameters" by return values
Martin von Zweigbergk <martinvonz@google.com>
parents: 27648
diff changeset
   438
        return filenodes
27645
df8973e1fb45 verify: move file cross checking to its own function
Durham Goode <durham@fb.com>
parents: 27644
diff changeset
   439
28111
06205989264b verify: move cross-checking of changeset/manifest out of _crosscheckfiles()
Martin von Zweigbergk <martinvonz@google.com>
parents: 28007
diff changeset
   440
    def _crosscheckfiles(self, filelinkrevs, filenodes):
27645
df8973e1fb45 verify: move file cross checking to its own function
Durham Goode <durham@fb.com>
parents: 27644
diff changeset
   441
        repo = self.repo
df8973e1fb45 verify: move file cross checking to its own function
Durham Goode <durham@fb.com>
parents: 27644
diff changeset
   442
        ui = self.ui
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   443
        ui.status(_(b"crosschecking files in changesets and manifests\n"))
2778
fdc232d8a193 Move repo.verify
Matt Mackall <mpm@selenic.com>
parents:
diff changeset
   444
28111
06205989264b verify: move cross-checking of changeset/manifest out of _crosscheckfiles()
Martin von Zweigbergk <martinvonz@google.com>
parents: 28007
diff changeset
   445
        total = len(filelinkrevs) + len(filenodes)
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   446
        progress = ui.makeprogress(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   447
            _(b'crosschecking'), unit=_(b'files'), total=total
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   448
        )
27645
df8973e1fb45 verify: move file cross checking to its own function
Durham Goode <durham@fb.com>
parents: 27644
diff changeset
   449
        if self.havemf:
27443
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   450
            for f in sorted(filelinkrevs):
38397
1249475f0bd6 verify: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents: 38396
diff changeset
   451
                progress.increment()
27443
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   452
                if f not in filenodes:
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   453
                    lr = filelinkrevs[f][0]
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   454
                    self._err(lr, _(b"in changeset but not in manifest"), f)
6892
dab95717058d verify: check repo.store
Adrian Buehlmann <adrian@cadifra.com>
parents: 6889
diff changeset
   455
27645
df8973e1fb45 verify: move file cross checking to its own function
Durham Goode <durham@fb.com>
parents: 27644
diff changeset
   456
        if self.havecl:
27443
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   457
            for f in sorted(filenodes):
38397
1249475f0bd6 verify: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents: 38396
diff changeset
   458
                progress.increment()
27443
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   459
                if f not in filelinkrevs:
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   460
                    try:
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   461
                        fl = repo.file(f)
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   462
                        lr = min([fl.linkrev(fl.rev(n)) for n in filenodes[f]])
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   463
                    except Exception:
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   464
                        lr = None
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   465
                    self._err(lr, _(b"in manifest but not in changeset"), f)
27443
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   466
38397
1249475f0bd6 verify: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents: 38396
diff changeset
   467
        progress.complete()
8291
f5c1a9094e41 verify: avoid exception on missing file revlog
Henrik Stuart <hg@hstuart.dk>
parents: 8225
diff changeset
   468
27644
331e5c28f5f0 verify: move filelog verification to its own function
Durham Goode <durham@fb.com>
parents: 27643
diff changeset
   469
    def _verifyfiles(self, filenodes, filelinkrevs):
331e5c28f5f0 verify: move filelog verification to its own function
Durham Goode <durham@fb.com>
parents: 27643
diff changeset
   470
        repo = self.repo
331e5c28f5f0 verify: move filelog verification to its own function
Durham Goode <durham@fb.com>
parents: 27643
diff changeset
   471
        ui = self.ui
331e5c28f5f0 verify: move filelog verification to its own function
Durham Goode <durham@fb.com>
parents: 27643
diff changeset
   472
        lrugetctx = self.lrugetctx
331e5c28f5f0 verify: move filelog verification to its own function
Durham Goode <durham@fb.com>
parents: 27643
diff changeset
   473
        revlogv1 = self.revlogv1
331e5c28f5f0 verify: move filelog verification to its own function
Durham Goode <durham@fb.com>
parents: 27643
diff changeset
   474
        havemf = self.havemf
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   475
        ui.status(_(b"checking files\n"))
8291
f5c1a9094e41 verify: avoid exception on missing file revlog
Henrik Stuart <hg@hstuart.dk>
parents: 8225
diff changeset
   476
27443
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   477
        storefiles = set()
46895
6085b7f1536d store: also return some information about the type of file `walk` found
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46819
diff changeset
   478
        for rl_type, f, f2, size in repo.store.datafiles():
27443
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   479
            if not f:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   480
                self._err(None, _(b"cannot decode filename '%s'") % f2)
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   481
            elif (size > 0 or not revlogv1) and f.startswith(b'data/'):
27443
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   482
                storefiles.add(_normpath(f))
6892
dab95717058d verify: check repo.store
Adrian Buehlmann <adrian@cadifra.com>
parents: 6889
diff changeset
   483
39842
97986c9c69d3 verify: start to abstract file verification
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39777
diff changeset
   484
        state = {
39845
e6d3d39cc1c7 revlog: use proper version comparison during verify
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39842
diff changeset
   485
            # TODO this assumes revlog storage for changelog.
47139
f58a13c52726 revlog: split the `version` attribute into its two components
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47138
diff changeset
   486
            b'expectedversion': self.repo.changelog._format_version,
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   487
            b'skipflags': self.skipflags,
39872
733db72f0f54 revlog: move revision verification out of verify
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39845
diff changeset
   488
            # experimental config: censor.policy
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   489
            b'erroroncensored': ui.config(b'censor', b'policy') == b'abort',
39842
97986c9c69d3 verify: start to abstract file verification
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39777
diff changeset
   490
        }
97986c9c69d3 verify: start to abstract file verification
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39777
diff changeset
   491
27443
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   492
        files = sorted(set(filenodes) | set(filelinkrevs))
27644
331e5c28f5f0 verify: move filelog verification to its own function
Durham Goode <durham@fb.com>
parents: 27643
diff changeset
   493
        revisions = 0
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   494
        progress = ui.makeprogress(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   495
            _(b'checking'), unit=_(b'files'), total=len(files)
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   496
        )
27443
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   497
        for i, f in enumerate(files):
38397
1249475f0bd6 verify: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents: 38396
diff changeset
   498
            progress.update(i, item=f)
6892
dab95717058d verify: check repo.store
Adrian Buehlmann <adrian@cadifra.com>
parents: 6889
diff changeset
   499
            try:
27443
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   500
                linkrevs = filelinkrevs[f]
6892
dab95717058d verify: check repo.store
Adrian Buehlmann <adrian@cadifra.com>
parents: 6889
diff changeset
   501
            except KeyError:
27443
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   502
                # in manifest but not in changelog
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   503
                linkrevs = []
2778
fdc232d8a193 Move repo.verify
Matt Mackall <mpm@selenic.com>
parents:
diff changeset
   504
27443
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   505
            if linkrevs:
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   506
                lr = linkrevs[0]
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   507
            else:
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   508
                lr = None
2778
fdc232d8a193 Move repo.verify
Matt Mackall <mpm@selenic.com>
parents:
diff changeset
   509
3744
d626fc9e3985 verify: add rename link checking
Matt Mackall <mpm@selenic.com>
parents: 3473
diff changeset
   510
            try:
27443
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   511
                fl = repo.file(f)
39777
b63dee7bd0d9 global: replace most uses of RevlogError with StorageError (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39489
diff changeset
   512
            except error.StorageError as e:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   513
                self._err(lr, _(b"broken revlog! (%s)") % e, f)
27443
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   514
                continue
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   515
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   516
            for ff in fl.files():
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   517
                try:
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   518
                    storefiles.remove(ff)
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   519
                except KeyError:
37417
76d2115cb817 verify: allow suppressing warnings about extra files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 37410
diff changeset
   520
                    if self.warnorphanstorefiles:
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   521
                        self._warn(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   522
                            _(b" warning: revlog '%s' not in fncache!") % ff
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   523
                        )
37417
76d2115cb817 verify: allow suppressing warnings about extra files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 37410
diff changeset
   524
                        self.fncachewarned = True
27443
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   525
39842
97986c9c69d3 verify: start to abstract file verification
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39777
diff changeset
   526
            if not len(fl) and (self.havecl or self.havemf):
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   527
                self._err(lr, _(b"empty or missing %s") % f)
39842
97986c9c69d3 verify: start to abstract file verification
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39777
diff changeset
   528
            else:
39872
733db72f0f54 revlog: move revision verification out of verify
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39845
diff changeset
   529
                # Guard against implementations not setting this.
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   530
                state[b'skipread'] = set()
44073
b9e174d4ed11 verify: allow the storage to signal when renames can be tested on `skipread`
Matt Harbison <matt_harbison@yahoo.com>
parents: 43956
diff changeset
   531
                state[b'safe_renamed'] = set()
b9e174d4ed11 verify: allow the storage to signal when renames can be tested on `skipread`
Matt Harbison <matt_harbison@yahoo.com>
parents: 43956
diff changeset
   532
39842
97986c9c69d3 verify: start to abstract file verification
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39777
diff changeset
   533
                for problem in fl.verifyintegrity(state):
39872
733db72f0f54 revlog: move revision verification out of verify
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39845
diff changeset
   534
                    if problem.node is not None:
733db72f0f54 revlog: move revision verification out of verify
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39845
diff changeset
   535
                        linkrev = fl.linkrev(fl.rev(problem.node))
733db72f0f54 revlog: move revision verification out of verify
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39845
diff changeset
   536
                    else:
733db72f0f54 revlog: move revision verification out of verify
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39845
diff changeset
   537
                        linkrev = None
733db72f0f54 revlog: move revision verification out of verify
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39845
diff changeset
   538
39842
97986c9c69d3 verify: start to abstract file verification
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39777
diff changeset
   539
                    if problem.warning:
41862
c66037fb1bc5 verify: make the `warn` method private
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41861
diff changeset
   540
                        self._warn(problem.warning)
39842
97986c9c69d3 verify: start to abstract file verification
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39777
diff changeset
   541
                    elif problem.error:
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   542
                        self._err(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   543
                            linkrev if linkrev is not None else lr,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   544
                            problem.error,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   545
                            f,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   546
                        )
39842
97986c9c69d3 verify: start to abstract file verification
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39777
diff changeset
   547
                    else:
97986c9c69d3 verify: start to abstract file verification
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39777
diff changeset
   548
                        raise error.ProgrammingError(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   549
                            b'problem instance does not set warning or error '
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   550
                            b'attribute: %s' % problem.msg
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   551
                        )
39842
97986c9c69d3 verify: start to abstract file verification
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39777
diff changeset
   552
27443
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   553
            seen = {}
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   554
            for i in fl:
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   555
                revisions += 1
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   556
                n = fl.node(i)
41871
9e737ca539f6 verify: make `checkentry` a private method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 41870
diff changeset
   557
                lr = self._checkentry(fl, i, n, seen, linkrevs, f)
27443
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   558
                if f in filenodes:
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   559
                    if havemf and n not in filenodes[f]:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   560
                        self._err(lr, _(b"%s not in manifests") % (short(n)), f)
6534
9b35a9f34675 verify: check copy source revlog and nodeid
Patrick Mezard <pmezard@gmail.com>
parents: 6211
diff changeset
   561
                    else:
27443
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   562
                        del filenodes[f][n]
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   563
44073
b9e174d4ed11 verify: allow the storage to signal when renames can be tested on `skipread`
Matt Harbison <matt_harbison@yahoo.com>
parents: 43956
diff changeset
   564
                if n in state[b'skipread'] and n not in state[b'safe_renamed']:
39872
733db72f0f54 revlog: move revision verification out of verify
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39845
diff changeset
   565
                    continue
3744
d626fc9e3985 verify: add rename link checking
Matt Mackall <mpm@selenic.com>
parents: 3473
diff changeset
   566
27443
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   567
                # check renames
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   568
                try:
43956
a447efd991b9 verify: update comment to say that lfs doesn't need fulltext to check renames
Matt Harbison <matt_harbison@yahoo.com>
parents: 43106
diff changeset
   569
                    # This requires resolving fulltext (at least on revlogs,
a447efd991b9 verify: update comment to say that lfs doesn't need fulltext to check renames
Matt Harbison <matt_harbison@yahoo.com>
parents: 43106
diff changeset
   570
                    # though not with LFS revisions). We may want
a447efd991b9 verify: update comment to say that lfs doesn't need fulltext to check renames
Matt Harbison <matt_harbison@yahoo.com>
parents: 43106
diff changeset
   571
                    # ``verifyintegrity()`` to pass a set of nodes with
39872
733db72f0f54 revlog: move revision verification out of verify
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39845
diff changeset
   572
                    # rename metadata as an optimization.
733db72f0f54 revlog: move revision verification out of verify
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39845
diff changeset
   573
                    rp = fl.renamed(n)
27443
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   574
                    if rp:
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   575
                        if lr is not None and ui.verbose:
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   576
                            ctx = lrugetctx(lr)
36339
a4d41ba4ad23 verify: don't reimplement any()
Martin von Zweigbergk <martinvonz@google.com>
parents: 36185
diff changeset
   577
                            if not any(rp[0] in pctx for pctx in ctx.parents()):
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   578
                                self._warn(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   579
                                    _(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   580
                                        b"warning: copy source of '%s' not"
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   581
                                        b" in parents of %s"
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   582
                                    )
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   583
                                    % (f, ctx)
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   584
                                )
27443
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   585
                        fl2 = repo.file(rp[0])
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   586
                        if not len(fl2):
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   587
                            self._err(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   588
                                lr,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   589
                                _(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   590
                                    b"empty or missing copy source revlog "
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   591
                                    b"%s:%s"
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   592
                                )
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   593
                                % (rp[0], short(rp[1])),
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   594
                                f,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   595
                            )
47012
d55b71393907 node: replace nullid and friends with nodeconstants class
Joerg Sonnenberger <joerg@bec.de>
parents: 46895
diff changeset
   596
                        elif rp[1] == self.repo.nullid:
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   597
                            ui.note(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   598
                                _(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   599
                                    b"warning: %s@%s: copy source"
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   600
                                    b" revision is nullid %s:%s\n"
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   601
                                )
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   602
                                % (f, lr, rp[0], short(rp[1]))
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   603
                            )
27443
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   604
                        else:
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   605
                            fl2.rev(rp[1])
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   606
                except Exception as inst:
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   607
                    self._exc(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   608
                        lr, _(b"checking rename of %s") % short(n), inst, f
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   609
                    )
6892
dab95717058d verify: check repo.store
Adrian Buehlmann <adrian@cadifra.com>
parents: 6889
diff changeset
   610
27443
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   611
            # cross-check
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   612
            if f in filenodes:
43106
d783f945a701 py3: finish porting iteritems() to pycompat and remove source transformer
Gregory Szorc <gregory.szorc@gmail.com>
parents: 43077
diff changeset
   613
                fns = [(v, k) for k, v in pycompat.iteritems(filenodes[f])]
27443
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   614
                for lr, node in sorted(fns):
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   615
                    self._err(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   616
                        lr,
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   617
                        _(b"manifest refers to unknown revision %s")
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   618
                        % short(node),
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   619
                        f,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42387
diff changeset
   620
                    )
38397
1249475f0bd6 verify: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents: 38396
diff changeset
   621
        progress.complete()
27443
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   622
37417
76d2115cb817 verify: allow suppressing warnings about extra files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 37410
diff changeset
   623
        if self.warnorphanstorefiles:
76d2115cb817 verify: allow suppressing warnings about extra files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 37410
diff changeset
   624
            for f in sorted(storefiles):
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   625
                self._warn(_(b"warning: orphan data file '%s'") % f)
27443
937e73a6e4ff verify: move verify logic into a class
Durham Goode <durham@fb.com>
parents: 26900
diff changeset
   626
27644
331e5c28f5f0 verify: move filelog verification to its own function
Durham Goode <durham@fb.com>
parents: 27643
diff changeset
   627
        return len(files), revisions