Mercurial > hg
comparison mercurial/verify.py @ 43076:2372284d9457
formatting: blacken the codebase
This is using my patch to black
(https://github.com/psf/black/pull/826) so we don't un-wrap collection
literals.
Done with:
hg files 'set:**.py - mercurial/thirdparty/** - "contrib/python-zstandard/**"' | xargs black -S
# skip-blame mass-reformatting only
# no-check-commit reformats foo_bar functions
Differential Revision: https://phab.mercurial-scm.org/D6971
author | Augie Fackler <augie@google.com> |
---|---|
date | Sun, 06 Oct 2019 09:45:02 -0400 |
parents | 91452f62dd53 |
children | 687b865b95ad |
comparison
equal
deleted
inserted
replaced
43075:57875cf423c9 | 43076:2372284d9457 |
---|---|
23 ) | 23 ) |
24 | 24 |
25 VERIFY_DEFAULT = 0 | 25 VERIFY_DEFAULT = 0 |
26 VERIFY_FULL = 1 | 26 VERIFY_FULL = 1 |
27 | 27 |
28 | |
28 def verify(repo, level=None): | 29 def verify(repo, level=None): |
29 with repo.lock(): | 30 with repo.lock(): |
30 v = verifier(repo, level) | 31 v = verifier(repo, level) |
31 return v.verify() | 32 return v.verify() |
33 | |
32 | 34 |
33 def _normpath(f): | 35 def _normpath(f): |
34 # under hg < 2.4, convert didn't sanitize paths properly, so a | 36 # under hg < 2.4, convert didn't sanitize paths properly, so a |
35 # converted repo may contain repeated slashes | 37 # converted repo may contain repeated slashes |
36 while '//' in f: | 38 while '//' in f: |
37 f = f.replace('//', '/') | 39 f = f.replace('//', '/') |
38 return f | 40 return f |
41 | |
39 | 42 |
40 class verifier(object): | 43 class verifier(object): |
41 def __init__(self, repo, level=None): | 44 def __init__(self, repo, level=None): |
42 self.repo = repo.unfiltered() | 45 self.repo = repo.unfiltered() |
43 self.ui = repo.ui | 46 self.ui = repo.ui |
136 self._err(None, msg % (i, lr), f) | 139 self._err(None, msg % (i, lr), f) |
137 if linkrevs: | 140 if linkrevs: |
138 if f and len(linkrevs) > 1: | 141 if f and len(linkrevs) > 1: |
139 try: | 142 try: |
140 # attempt to filter down to real linkrevs | 143 # attempt to filter down to real linkrevs |
141 linkrevs = [l for l in linkrevs | 144 linkrevs = [ |
142 if self.lrugetctx(l)[f].filenode() == node] | 145 l |
146 for l in linkrevs | |
147 if self.lrugetctx(l)[f].filenode() == node | |
148 ] | |
143 except Exception: | 149 except Exception: |
144 pass | 150 pass |
145 self._warn(_(" (expected %s)") % " ".join | 151 self._warn( |
146 (map(pycompat.bytestr, linkrevs))) | 152 _(" (expected %s)") |
147 lr = None # can't be trusted | 153 % " ".join(map(pycompat.bytestr, linkrevs)) |
154 ) | |
155 lr = None # can't be trusted | |
148 | 156 |
149 try: | 157 try: |
150 p1, p2 = obj.parents(node) | 158 p1, p2 = obj.parents(node) |
151 if p1 not in seen and p1 != nullid: | 159 if p1 not in seen and p1 != nullid: |
152 self._err(lr, _("unknown parent 1 %s of %s") % | 160 self._err( |
153 (short(p1), short(node)), f) | 161 lr, |
162 _("unknown parent 1 %s of %s") % (short(p1), short(node)), | |
163 f, | |
164 ) | |
154 if p2 not in seen and p2 != nullid: | 165 if p2 not in seen and p2 != nullid: |
155 self._err(lr, _("unknown parent 2 %s of %s") % | 166 self._err( |
156 (short(p2), short(node)), f) | 167 lr, |
168 _("unknown parent 2 %s of %s") % (short(p2), short(node)), | |
169 f, | |
170 ) | |
157 except Exception as inst: | 171 except Exception as inst: |
158 self._exc(lr, _("checking parents of %s") % short(node), inst, f) | 172 self._exc(lr, _("checking parents of %s") % short(node), inst, f) |
159 | 173 |
160 if node in seen: | 174 if node in seen: |
161 self._err(lr, _("duplicate revision %d (%d)") % (i, seen[node]), f) | 175 self._err(lr, _("duplicate revision %d (%d)") % (i, seen[node]), f) |
176 | 190 |
177 if os.path.exists(repo.sjoin("journal")): | 191 if os.path.exists(repo.sjoin("journal")): |
178 ui.warn(_("abandoned transaction found - run hg recover\n")) | 192 ui.warn(_("abandoned transaction found - run hg recover\n")) |
179 | 193 |
180 if ui.verbose or not self.revlogv1: | 194 if ui.verbose or not self.revlogv1: |
181 ui.status(_("repository uses revlog format %d\n") % | 195 ui.status( |
182 (self.revlogv1 and 1 or 0)) | 196 _("repository uses revlog format %d\n") |
197 % (self.revlogv1 and 1 or 0) | |
198 ) | |
183 | 199 |
184 # data verification | 200 # data verification |
185 mflinkrevs, filelinkrevs = self._verifychangelog() | 201 mflinkrevs, filelinkrevs = self._verifychangelog() |
186 filenodes = self._verifymanifest(mflinkrevs) | 202 filenodes = self._verifymanifest(mflinkrevs) |
187 del mflinkrevs | 203 del mflinkrevs |
188 self._crosscheckfiles(filelinkrevs, filenodes) | 204 self._crosscheckfiles(filelinkrevs, filenodes) |
189 totalfiles, filerevisions = self._verifyfiles(filenodes, filelinkrevs) | 205 totalfiles, filerevisions = self._verifyfiles(filenodes, filelinkrevs) |
190 | 206 |
191 # final report | 207 # final report |
192 ui.status(_("checked %d changesets with %d changes to %d files\n") % | 208 ui.status( |
193 (len(repo.changelog), filerevisions, totalfiles)) | 209 _("checked %d changesets with %d changes to %d files\n") |
210 % (len(repo.changelog), filerevisions, totalfiles) | |
211 ) | |
194 if self.warnings: | 212 if self.warnings: |
195 ui.warn(_("%d warnings encountered!\n") % self.warnings) | 213 ui.warn(_("%d warnings encountered!\n") % self.warnings) |
196 if self.fncachewarned: | 214 if self.fncachewarned: |
197 ui.warn(_('hint: run "hg debugrebuildfncache" to recover from ' | 215 ui.warn( |
198 'corrupt fncache\n')) | 216 _( |
217 'hint: run "hg debugrebuildfncache" to recover from ' | |
218 'corrupt fncache\n' | |
219 ) | |
220 ) | |
199 if self.errors: | 221 if self.errors: |
200 ui.warn(_("%d integrity errors encountered!\n") % self.errors) | 222 ui.warn(_("%d integrity errors encountered!\n") % self.errors) |
201 if self.badrevs: | 223 if self.badrevs: |
202 ui.warn(_("(first damaged changeset appears to be %d)\n") | 224 ui.warn( |
203 % min(self.badrevs)) | 225 _("(first damaged changeset appears to be %d)\n") |
226 % min(self.badrevs) | |
227 ) | |
204 return 1 | 228 return 1 |
205 return 0 | 229 return 0 |
206 | 230 |
207 def _verifychangelog(self): | 231 def _verifychangelog(self): |
208 """verify the changelog of a repository | 232 """verify the changelog of a repository |
228 ui.status(_("checking changesets\n")) | 252 ui.status(_("checking changesets\n")) |
229 mflinkrevs = {} | 253 mflinkrevs = {} |
230 filelinkrevs = {} | 254 filelinkrevs = {} |
231 seen = {} | 255 seen = {} |
232 self._checkrevlog(cl, "changelog", 0) | 256 self._checkrevlog(cl, "changelog", 0) |
233 progress = ui.makeprogress(_('checking'), unit=_('changesets'), | 257 progress = ui.makeprogress( |
234 total=len(repo)) | 258 _('checking'), unit=_('changesets'), total=len(repo) |
259 ) | |
235 for i in repo: | 260 for i in repo: |
236 progress.update(i) | 261 progress.update(i) |
237 n = cl.node(i) | 262 n = cl.node(i) |
238 self._checkentry(cl, i, n, seen, [i], "changelog") | 263 self._checkentry(cl, i, n, seen, [i], "changelog") |
239 | 264 |
249 self.refersmf = True | 274 self.refersmf = True |
250 self._exc(i, _("unpacking changeset %s") % short(n), inst) | 275 self._exc(i, _("unpacking changeset %s") % short(n), inst) |
251 progress.complete() | 276 progress.complete() |
252 return mflinkrevs, filelinkrevs | 277 return mflinkrevs, filelinkrevs |
253 | 278 |
254 def _verifymanifest(self, mflinkrevs, dir="", storefiles=None, | 279 def _verifymanifest( |
255 subdirprogress=None): | 280 self, mflinkrevs, dir="", storefiles=None, subdirprogress=None |
281 ): | |
256 """verify the manifestlog content | 282 """verify the manifestlog content |
257 | 283 |
258 Inputs: | 284 Inputs: |
259 - mflinkrevs: a {manifest-node -> [changelog-revisions]} mapping | 285 - mflinkrevs: a {manifest-node -> [changelog-revisions]} mapping |
260 - dir: a subdirectory to check (for tree manifest repo) | 286 - dir: a subdirectory to check (for tree manifest repo) |
295 label = "manifest" | 321 label = "manifest" |
296 if dir: | 322 if dir: |
297 label = dir | 323 label = dir |
298 revlogfiles = mf.files() | 324 revlogfiles = mf.files() |
299 storefiles.difference_update(revlogfiles) | 325 storefiles.difference_update(revlogfiles) |
300 if subdirprogress: # should be true since we're in a subdirectory | 326 if subdirprogress: # should be true since we're in a subdirectory |
301 subdirprogress.increment() | 327 subdirprogress.increment() |
302 if self.refersmf: | 328 if self.refersmf: |
303 # Do not check manifest if there are only changelog entries with | 329 # Do not check manifest if there are only changelog entries with |
304 # null manifests. | 330 # null manifests. |
305 self._checkrevlog(mf, label, 0) | 331 self._checkrevlog(mf, label, 0) |
306 progress = ui.makeprogress(_('checking'), unit=_('manifests'), | 332 progress = ui.makeprogress( |
307 total=len(mf)) | 333 _('checking'), unit=_('manifests'), total=len(mf) |
334 ) | |
308 for i in mf: | 335 for i in mf: |
309 if not dir: | 336 if not dir: |
310 progress.update(i) | 337 progress.update(i) |
311 n = mf.node(i) | 338 n = mf.node(i) |
312 lr = self._checkentry(mf, i, n, seen, mflinkrevs.get(n, []), label) | 339 lr = self._checkentry(mf, i, n, seen, mflinkrevs.get(n, []), label) |
313 if n in mflinkrevs: | 340 if n in mflinkrevs: |
314 del mflinkrevs[n] | 341 del mflinkrevs[n] |
315 elif dir: | 342 elif dir: |
316 self._err(lr, _("%s not in parent-directory manifest") % | 343 self._err( |
317 short(n), label) | 344 lr, |
345 _("%s not in parent-directory manifest") % short(n), | |
346 label, | |
347 ) | |
318 else: | 348 else: |
319 self._err(lr, _("%s not in changesets") % short(n), label) | 349 self._err(lr, _("%s not in changesets") % short(n), label) |
320 | 350 |
321 try: | 351 try: |
322 mfdelta = mfl.get(dir, n).readdelta(shallow=True) | 352 mfdelta = mfl.get(dir, n).readdelta(shallow=True) |
328 fullpath = dir + _normpath(f) | 358 fullpath = dir + _normpath(f) |
329 if fl == 't': | 359 if fl == 't': |
330 if not match.visitdir(fullpath): | 360 if not match.visitdir(fullpath): |
331 continue | 361 continue |
332 subdirnodes.setdefault(fullpath + '/', {}).setdefault( | 362 subdirnodes.setdefault(fullpath + '/', {}).setdefault( |
333 fn, []).append(lr) | 363 fn, [] |
364 ).append(lr) | |
334 else: | 365 else: |
335 if not match(fullpath): | 366 if not match(fullpath): |
336 continue | 367 continue |
337 filenodes.setdefault(fullpath, {}).setdefault(fn, lr) | 368 filenodes.setdefault(fullpath, {}).setdefault(fn, lr) |
338 except Exception as inst: | 369 except Exception as inst: |
342 # Various issues can affect manifest. So we read each full | 373 # Various issues can affect manifest. So we read each full |
343 # text from storage. This triggers the checks from the core | 374 # text from storage. This triggers the checks from the core |
344 # code (eg: hash verification, filename are ordered, etc.) | 375 # code (eg: hash verification, filename are ordered, etc.) |
345 mfdelta = mfl.get(dir, n).read() | 376 mfdelta = mfl.get(dir, n).read() |
346 except Exception as inst: | 377 except Exception as inst: |
347 self._exc(lr, _("reading full manifest %s") % short(n), | 378 self._exc( |
348 inst, label) | 379 lr, |
380 _("reading full manifest %s") % short(n), | |
381 inst, | |
382 label, | |
383 ) | |
349 | 384 |
350 if not dir: | 385 if not dir: |
351 progress.complete() | 386 progress.complete() |
352 | 387 |
353 if self.havemf: | 388 if self.havemf: |
354 # since we delete entry in `mflinkrevs` during iteration, any | 389 # since we delete entry in `mflinkrevs` during iteration, any |
355 # remaining entries are "missing". We need to issue errors for them. | 390 # remaining entries are "missing". We need to issue errors for them. |
356 changesetpairs = [(c, m) for m in mflinkrevs for c in mflinkrevs[m]] | 391 changesetpairs = [(c, m) for m in mflinkrevs for c in mflinkrevs[m]] |
357 for c, m in sorted(changesetpairs): | 392 for c, m in sorted(changesetpairs): |
358 if dir: | 393 if dir: |
359 self._err(c, _("parent-directory manifest refers to unknown" | 394 self._err( |
360 " revision %s") % short(m), label) | 395 c, |
396 _( | |
397 "parent-directory manifest refers to unknown" | |
398 " revision %s" | |
399 ) | |
400 % short(m), | |
401 label, | |
402 ) | |
361 else: | 403 else: |
362 self._err(c, _("changeset refers to unknown revision %s") % | 404 self._err( |
363 short(m), label) | 405 c, |
406 _("changeset refers to unknown revision %s") % short(m), | |
407 label, | |
408 ) | |
364 | 409 |
365 if not dir and subdirnodes: | 410 if not dir and subdirnodes: |
366 self.ui.status(_("checking directory manifests\n")) | 411 self.ui.status(_("checking directory manifests\n")) |
367 storefiles = set() | 412 storefiles = set() |
368 subdirs = set() | 413 subdirs = set() |
371 if not f: | 416 if not f: |
372 self._err(None, _("cannot decode filename '%s'") % f2) | 417 self._err(None, _("cannot decode filename '%s'") % f2) |
373 elif (size > 0 or not revlogv1) and f.startswith('meta/'): | 418 elif (size > 0 or not revlogv1) and f.startswith('meta/'): |
374 storefiles.add(_normpath(f)) | 419 storefiles.add(_normpath(f)) |
375 subdirs.add(os.path.dirname(f)) | 420 subdirs.add(os.path.dirname(f)) |
376 subdirprogress = ui.makeprogress(_('checking'), unit=_('manifests'), | 421 subdirprogress = ui.makeprogress( |
377 total=len(subdirs)) | 422 _('checking'), unit=_('manifests'), total=len(subdirs) |
423 ) | |
378 | 424 |
379 for subdir, linkrevs in subdirnodes.iteritems(): | 425 for subdir, linkrevs in subdirnodes.iteritems(): |
380 subdirfilenodes = self._verifymanifest(linkrevs, subdir, storefiles, | 426 subdirfilenodes = self._verifymanifest( |
381 subdirprogress) | 427 linkrevs, subdir, storefiles, subdirprogress |
428 ) | |
382 for f, onefilenodes in subdirfilenodes.iteritems(): | 429 for f, onefilenodes in subdirfilenodes.iteritems(): |
383 filenodes.setdefault(f, {}).update(onefilenodes) | 430 filenodes.setdefault(f, {}).update(onefilenodes) |
384 | 431 |
385 if not dir and subdirnodes: | 432 if not dir and subdirnodes: |
386 subdirprogress.complete() | 433 subdirprogress.complete() |
394 repo = self.repo | 441 repo = self.repo |
395 ui = self.ui | 442 ui = self.ui |
396 ui.status(_("crosschecking files in changesets and manifests\n")) | 443 ui.status(_("crosschecking files in changesets and manifests\n")) |
397 | 444 |
398 total = len(filelinkrevs) + len(filenodes) | 445 total = len(filelinkrevs) + len(filenodes) |
399 progress = ui.makeprogress(_('crosschecking'), unit=_('files'), | 446 progress = ui.makeprogress( |
400 total=total) | 447 _('crosschecking'), unit=_('files'), total=total |
448 ) | |
401 if self.havemf: | 449 if self.havemf: |
402 for f in sorted(filelinkrevs): | 450 for f in sorted(filelinkrevs): |
403 progress.increment() | 451 progress.increment() |
404 if f not in filenodes: | 452 if f not in filenodes: |
405 lr = filelinkrevs[f][0] | 453 lr = filelinkrevs[f][0] |
441 'erroroncensored': ui.config('censor', 'policy') == 'abort', | 489 'erroroncensored': ui.config('censor', 'policy') == 'abort', |
442 } | 490 } |
443 | 491 |
444 files = sorted(set(filenodes) | set(filelinkrevs)) | 492 files = sorted(set(filenodes) | set(filelinkrevs)) |
445 revisions = 0 | 493 revisions = 0 |
446 progress = ui.makeprogress(_('checking'), unit=_('files'), | 494 progress = ui.makeprogress( |
447 total=len(files)) | 495 _('checking'), unit=_('files'), total=len(files) |
496 ) | |
448 for i, f in enumerate(files): | 497 for i, f in enumerate(files): |
449 progress.update(i, item=f) | 498 progress.update(i, item=f) |
450 try: | 499 try: |
451 linkrevs = filelinkrevs[f] | 500 linkrevs = filelinkrevs[f] |
452 except KeyError: | 501 except KeyError: |
467 for ff in fl.files(): | 516 for ff in fl.files(): |
468 try: | 517 try: |
469 storefiles.remove(ff) | 518 storefiles.remove(ff) |
470 except KeyError: | 519 except KeyError: |
471 if self.warnorphanstorefiles: | 520 if self.warnorphanstorefiles: |
472 self._warn(_(" warning: revlog '%s' not in fncache!") % | 521 self._warn( |
473 ff) | 522 _(" warning: revlog '%s' not in fncache!") % ff |
523 ) | |
474 self.fncachewarned = True | 524 self.fncachewarned = True |
475 | 525 |
476 if not len(fl) and (self.havecl or self.havemf): | 526 if not len(fl) and (self.havecl or self.havemf): |
477 self._err(lr, _("empty or missing %s") % f) | 527 self._err(lr, _("empty or missing %s") % f) |
478 else: | 528 else: |
485 linkrev = None | 535 linkrev = None |
486 | 536 |
487 if problem.warning: | 537 if problem.warning: |
488 self._warn(problem.warning) | 538 self._warn(problem.warning) |
489 elif problem.error: | 539 elif problem.error: |
490 self._err(linkrev if linkrev is not None else lr, | 540 self._err( |
491 problem.error, f) | 541 linkrev if linkrev is not None else lr, |
542 problem.error, | |
543 f, | |
544 ) | |
492 else: | 545 else: |
493 raise error.ProgrammingError( | 546 raise error.ProgrammingError( |
494 'problem instance does not set warning or error ' | 547 'problem instance does not set warning or error ' |
495 'attribute: %s' % problem.msg) | 548 'attribute: %s' % problem.msg |
549 ) | |
496 | 550 |
497 seen = {} | 551 seen = {} |
498 for i in fl: | 552 for i in fl: |
499 revisions += 1 | 553 revisions += 1 |
500 n = fl.node(i) | 554 n = fl.node(i) |
516 rp = fl.renamed(n) | 570 rp = fl.renamed(n) |
517 if rp: | 571 if rp: |
518 if lr is not None and ui.verbose: | 572 if lr is not None and ui.verbose: |
519 ctx = lrugetctx(lr) | 573 ctx = lrugetctx(lr) |
520 if not any(rp[0] in pctx for pctx in ctx.parents()): | 574 if not any(rp[0] in pctx for pctx in ctx.parents()): |
521 self._warn(_("warning: copy source of '%s' not" | 575 self._warn( |
522 " in parents of %s") % (f, ctx)) | 576 _( |
577 "warning: copy source of '%s' not" | |
578 " in parents of %s" | |
579 ) | |
580 % (f, ctx) | |
581 ) | |
523 fl2 = repo.file(rp[0]) | 582 fl2 = repo.file(rp[0]) |
524 if not len(fl2): | 583 if not len(fl2): |
525 self._err(lr, | 584 self._err( |
526 _("empty or missing copy source revlog " | 585 lr, |
527 "%s:%s") % (rp[0], | 586 _( |
528 short(rp[1])), | 587 "empty or missing copy source revlog " |
529 f) | 588 "%s:%s" |
589 ) | |
590 % (rp[0], short(rp[1])), | |
591 f, | |
592 ) | |
530 elif rp[1] == nullid: | 593 elif rp[1] == nullid: |
531 ui.note(_("warning: %s@%s: copy source" | 594 ui.note( |
532 " revision is nullid %s:%s\n") | 595 _( |
533 % (f, lr, rp[0], short(rp[1]))) | 596 "warning: %s@%s: copy source" |
597 " revision is nullid %s:%s\n" | |
598 ) | |
599 % (f, lr, rp[0], short(rp[1])) | |
600 ) | |
534 else: | 601 else: |
535 fl2.rev(rp[1]) | 602 fl2.rev(rp[1]) |
536 except Exception as inst: | 603 except Exception as inst: |
537 self._exc(lr, _("checking rename of %s") % short(n), | 604 self._exc( |
538 inst, f) | 605 lr, _("checking rename of %s") % short(n), inst, f |
606 ) | |
539 | 607 |
540 # cross-check | 608 # cross-check |
541 if f in filenodes: | 609 if f in filenodes: |
542 fns = [(v, k) for k, v in filenodes[f].iteritems()] | 610 fns = [(v, k) for k, v in filenodes[f].iteritems()] |
543 for lr, node in sorted(fns): | 611 for lr, node in sorted(fns): |
544 self._err(lr, _("manifest refers to unknown revision %s") % | 612 self._err( |
545 short(node), f) | 613 lr, |
614 _("manifest refers to unknown revision %s") | |
615 % short(node), | |
616 f, | |
617 ) | |
546 progress.complete() | 618 progress.complete() |
547 | 619 |
548 if self.warnorphanstorefiles: | 620 if self.warnorphanstorefiles: |
549 for f in sorted(storefiles): | 621 for f in sorted(storefiles): |
550 self._warn(_("warning: orphan data file '%s'") % f) | 622 self._warn(_("warning: orphan data file '%s'") % f) |