Mercurial > hg
annotate mercurial/policy.py @ 40626:87a872555e90
revlog: detect incomplete revlog reads
_readsegment() is supposed to return N bytes of revlog revision
data starting at a file offset. Surprisingly, its behavior before
this patch never verified that it actually read and returned N
bytes! Instead, it would perform the read(), then return whatever
data was available. And even more surprisingly, nothing in the
call chain appears to have been validating that it received all
the data it was expecting.
This behavior could lead to partial or incomplete revision chunks
being operated on. This could result in e.g. cached deltas being
applied against incomplete base revisions. The delta application
process would happily perform this operation. Only hash
verification would detect the corruption and save us.
This commit changes the behavior of raw revlog reading to validate
that we actually read() the number of bytes that were requested.
We will raise a more specific error faster, rather than possibly
have it go undetected or manifest later in the call stack, at
delta application or hash verification.
Differential Revision: https://phab.mercurial-scm.org/D5266
author | Gregory Szorc <gregory.szorc@gmail.com> |
---|---|
date | Tue, 13 Nov 2018 12:30:59 -0800 |
parents | 481db51c83e9 |
children | f2342483f7a6 |
rev | line source |
---|---|
29266
b3a677c82a35
debuginstall: expose modulepolicy
timeless <timeless@mozdev.org>
parents:
diff
changeset
|
1 # policy.py - module policy logic for Mercurial. |
b3a677c82a35
debuginstall: expose modulepolicy
timeless <timeless@mozdev.org>
parents:
diff
changeset
|
2 # |
b3a677c82a35
debuginstall: expose modulepolicy
timeless <timeless@mozdev.org>
parents:
diff
changeset
|
3 # Copyright 2015 Gregory Szorc <gregory.szorc@gmail.com> |
b3a677c82a35
debuginstall: expose modulepolicy
timeless <timeless@mozdev.org>
parents:
diff
changeset
|
4 # |
b3a677c82a35
debuginstall: expose modulepolicy
timeless <timeless@mozdev.org>
parents:
diff
changeset
|
5 # This software may be used and distributed according to the terms of the |
b3a677c82a35
debuginstall: expose modulepolicy
timeless <timeless@mozdev.org>
parents:
diff
changeset
|
6 # GNU General Public License version 2 or any later version. |
b3a677c82a35
debuginstall: expose modulepolicy
timeless <timeless@mozdev.org>
parents:
diff
changeset
|
7 |
b3a677c82a35
debuginstall: expose modulepolicy
timeless <timeless@mozdev.org>
parents:
diff
changeset
|
8 from __future__ import absolute_import |
b3a677c82a35
debuginstall: expose modulepolicy
timeless <timeless@mozdev.org>
parents:
diff
changeset
|
9 |
b3a677c82a35
debuginstall: expose modulepolicy
timeless <timeless@mozdev.org>
parents:
diff
changeset
|
10 import os |
b3a677c82a35
debuginstall: expose modulepolicy
timeless <timeless@mozdev.org>
parents:
diff
changeset
|
11 import sys |
b3a677c82a35
debuginstall: expose modulepolicy
timeless <timeless@mozdev.org>
parents:
diff
changeset
|
12 |
b3a677c82a35
debuginstall: expose modulepolicy
timeless <timeless@mozdev.org>
parents:
diff
changeset
|
13 # Rules for how modules can be loaded. Values are: |
b3a677c82a35
debuginstall: expose modulepolicy
timeless <timeless@mozdev.org>
parents:
diff
changeset
|
14 # |
b3a677c82a35
debuginstall: expose modulepolicy
timeless <timeless@mozdev.org>
parents:
diff
changeset
|
15 # c - require C extensions |
b3a677c82a35
debuginstall: expose modulepolicy
timeless <timeless@mozdev.org>
parents:
diff
changeset
|
16 # allow - allow pure Python implementation when C loading fails |
29490
b4d117cee636
policy: add cffi policy for PyPy
Maciej Fijalkowski <fijall@gmail.com>
parents:
29266
diff
changeset
|
17 # cffi - required cffi versions (implemented within pure module) |
b4d117cee636
policy: add cffi policy for PyPy
Maciej Fijalkowski <fijall@gmail.com>
parents:
29266
diff
changeset
|
18 # cffi-allow - allow pure Python implementation if cffi version is missing |
29266
b3a677c82a35
debuginstall: expose modulepolicy
timeless <timeless@mozdev.org>
parents:
diff
changeset
|
19 # py - only load pure Python modules |
b3a677c82a35
debuginstall: expose modulepolicy
timeless <timeless@mozdev.org>
parents:
diff
changeset
|
20 # |
32251
a04f5c651e52
policy: relax the default for in-place build
Yuya Nishihara <yuya@tcha.org>
parents:
32210
diff
changeset
|
21 # By default, fall back to the pure modules so the in-place build can |
a04f5c651e52
policy: relax the default for in-place build
Yuya Nishihara <yuya@tcha.org>
parents:
32210
diff
changeset
|
22 # run without recompiling the C extensions. This will be overridden by |
a04f5c651e52
policy: relax the default for in-place build
Yuya Nishihara <yuya@tcha.org>
parents:
32210
diff
changeset
|
23 # __modulepolicy__ generated by setup.py. |
a04f5c651e52
policy: relax the default for in-place build
Yuya Nishihara <yuya@tcha.org>
parents:
32210
diff
changeset
|
24 policy = b'allow' |
32366
8e0327dae3f4
policy: add helper to import cext/pure module
Yuya Nishihara <yuya@tcha.org>
parents:
32251
diff
changeset
|
25 _packageprefs = { |
8e0327dae3f4
policy: add helper to import cext/pure module
Yuya Nishihara <yuya@tcha.org>
parents:
32251
diff
changeset
|
26 # policy: (versioned package, pure package) |
8e0327dae3f4
policy: add helper to import cext/pure module
Yuya Nishihara <yuya@tcha.org>
parents:
32251
diff
changeset
|
27 b'c': (r'cext', None), |
8e0327dae3f4
policy: add helper to import cext/pure module
Yuya Nishihara <yuya@tcha.org>
parents:
32251
diff
changeset
|
28 b'allow': (r'cext', r'pure'), |
32512
0e8b0b9a7acc
cffi: split modules from pure
Yuya Nishihara <yuya@tcha.org>
parents:
32511
diff
changeset
|
29 b'cffi': (r'cffi', None), |
0e8b0b9a7acc
cffi: split modules from pure
Yuya Nishihara <yuya@tcha.org>
parents:
32511
diff
changeset
|
30 b'cffi-allow': (r'cffi', r'pure'), |
32366
8e0327dae3f4
policy: add helper to import cext/pure module
Yuya Nishihara <yuya@tcha.org>
parents:
32251
diff
changeset
|
31 b'py': (None, r'pure'), |
8e0327dae3f4
policy: add helper to import cext/pure module
Yuya Nishihara <yuya@tcha.org>
parents:
32251
diff
changeset
|
32 } |
29490
b4d117cee636
policy: add cffi policy for PyPy
Maciej Fijalkowski <fijall@gmail.com>
parents:
29266
diff
changeset
|
33 |
29266
b3a677c82a35
debuginstall: expose modulepolicy
timeless <timeless@mozdev.org>
parents:
diff
changeset
|
34 try: |
b3a677c82a35
debuginstall: expose modulepolicy
timeless <timeless@mozdev.org>
parents:
diff
changeset
|
35 from . import __modulepolicy__ |
b3a677c82a35
debuginstall: expose modulepolicy
timeless <timeless@mozdev.org>
parents:
diff
changeset
|
36 policy = __modulepolicy__.modulepolicy |
b3a677c82a35
debuginstall: expose modulepolicy
timeless <timeless@mozdev.org>
parents:
diff
changeset
|
37 except ImportError: |
b3a677c82a35
debuginstall: expose modulepolicy
timeless <timeless@mozdev.org>
parents:
diff
changeset
|
38 pass |
b3a677c82a35
debuginstall: expose modulepolicy
timeless <timeless@mozdev.org>
parents:
diff
changeset
|
39 |
b3a677c82a35
debuginstall: expose modulepolicy
timeless <timeless@mozdev.org>
parents:
diff
changeset
|
40 # PyPy doesn't load C extensions. |
b3a677c82a35
debuginstall: expose modulepolicy
timeless <timeless@mozdev.org>
parents:
diff
changeset
|
41 # |
b3a677c82a35
debuginstall: expose modulepolicy
timeless <timeless@mozdev.org>
parents:
diff
changeset
|
42 # The canonical way to do this is to test platform.python_implementation(). |
b3a677c82a35
debuginstall: expose modulepolicy
timeless <timeless@mozdev.org>
parents:
diff
changeset
|
43 # But we don't import platform and don't bloat for it here. |
32210
56148133ef36
policy: mark all string literals as sysstr or bytes
Yuya Nishihara <yuya@tcha.org>
parents:
31361
diff
changeset
|
44 if r'__pypy__' in sys.builtin_module_names: |
56148133ef36
policy: mark all string literals as sysstr or bytes
Yuya Nishihara <yuya@tcha.org>
parents:
31361
diff
changeset
|
45 policy = b'cffi' |
29266
b3a677c82a35
debuginstall: expose modulepolicy
timeless <timeless@mozdev.org>
parents:
diff
changeset
|
46 |
b3a677c82a35
debuginstall: expose modulepolicy
timeless <timeless@mozdev.org>
parents:
diff
changeset
|
47 # Environment variable can always force settings. |
31361
8a17c541177f
py3: add "b" prefix to string literals related to module policy
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
31308
diff
changeset
|
48 if sys.version_info[0] >= 3: |
32210
56148133ef36
policy: mark all string literals as sysstr or bytes
Yuya Nishihara <yuya@tcha.org>
parents:
31361
diff
changeset
|
49 if r'HGMODULEPOLICY' in os.environ: |
56148133ef36
policy: mark all string literals as sysstr or bytes
Yuya Nishihara <yuya@tcha.org>
parents:
31361
diff
changeset
|
50 policy = os.environ[r'HGMODULEPOLICY'].encode(r'utf-8') |
31361
8a17c541177f
py3: add "b" prefix to string literals related to module policy
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
31308
diff
changeset
|
51 else: |
32210
56148133ef36
policy: mark all string literals as sysstr or bytes
Yuya Nishihara <yuya@tcha.org>
parents:
31361
diff
changeset
|
52 policy = os.environ.get(r'HGMODULEPOLICY', policy) |
32366
8e0327dae3f4
policy: add helper to import cext/pure module
Yuya Nishihara <yuya@tcha.org>
parents:
32251
diff
changeset
|
53 |
8e0327dae3f4
policy: add helper to import cext/pure module
Yuya Nishihara <yuya@tcha.org>
parents:
32251
diff
changeset
|
54 def _importfrom(pkgname, modname): |
8e0327dae3f4
policy: add helper to import cext/pure module
Yuya Nishihara <yuya@tcha.org>
parents:
32251
diff
changeset
|
55 # from .<pkgname> import <modname> (where . is looked through this module) |
8e0327dae3f4
policy: add helper to import cext/pure module
Yuya Nishihara <yuya@tcha.org>
parents:
32251
diff
changeset
|
56 fakelocals = {} |
8e0327dae3f4
policy: add helper to import cext/pure module
Yuya Nishihara <yuya@tcha.org>
parents:
32251
diff
changeset
|
57 pkg = __import__(pkgname, globals(), fakelocals, [modname], level=1) |
8e0327dae3f4
policy: add helper to import cext/pure module
Yuya Nishihara <yuya@tcha.org>
parents:
32251
diff
changeset
|
58 try: |
8e0327dae3f4
policy: add helper to import cext/pure module
Yuya Nishihara <yuya@tcha.org>
parents:
32251
diff
changeset
|
59 fakelocals[modname] = mod = getattr(pkg, modname) |
8e0327dae3f4
policy: add helper to import cext/pure module
Yuya Nishihara <yuya@tcha.org>
parents:
32251
diff
changeset
|
60 except AttributeError: |
8e0327dae3f4
policy: add helper to import cext/pure module
Yuya Nishihara <yuya@tcha.org>
parents:
32251
diff
changeset
|
61 raise ImportError(r'cannot import name %s' % modname) |
8e0327dae3f4
policy: add helper to import cext/pure module
Yuya Nishihara <yuya@tcha.org>
parents:
32251
diff
changeset
|
62 # force import; fakelocals[modname] may be replaced with the real module |
8e0327dae3f4
policy: add helper to import cext/pure module
Yuya Nishihara <yuya@tcha.org>
parents:
32251
diff
changeset
|
63 getattr(mod, r'__doc__', None) |
8e0327dae3f4
policy: add helper to import cext/pure module
Yuya Nishihara <yuya@tcha.org>
parents:
32251
diff
changeset
|
64 return fakelocals[modname] |
8e0327dae3f4
policy: add helper to import cext/pure module
Yuya Nishihara <yuya@tcha.org>
parents:
32251
diff
changeset
|
65 |
32428
28b773aa3ff2
policy: define C module versions individually
Jun Wu <quark@fb.com>
parents:
32366
diff
changeset
|
66 # keep in sync with "version" in C modules |
28b773aa3ff2
policy: define C module versions individually
Jun Wu <quark@fb.com>
parents:
32366
diff
changeset
|
67 _cextversions = { |
32511
2e431fb98c6b
policy: extend API version checks for cffi
Yuya Nishihara <yuya@tcha.org>
parents:
32428
diff
changeset
|
68 (r'cext', r'base85'): 1, |
36675 | 69 (r'cext', r'bdiff'): 3, |
32511
2e431fb98c6b
policy: extend API version checks for cffi
Yuya Nishihara <yuya@tcha.org>
parents:
32428
diff
changeset
|
70 (r'cext', r'mpatch'): 1, |
36780
f3c314020beb
osutil: implement minimal __getitem__ compatibility on our custom listdir type
Augie Fackler <augie@google.com>
parents:
36675
diff
changeset
|
71 (r'cext', r'osutil'): 4, |
39452 | 72 (r'cext', r'parsers'): 11, |
32428
28b773aa3ff2
policy: define C module versions individually
Jun Wu <quark@fb.com>
parents:
32366
diff
changeset
|
73 } |
28b773aa3ff2
policy: define C module versions individually
Jun Wu <quark@fb.com>
parents:
32366
diff
changeset
|
74 |
33760
cd2aca0808f8
policy: reroute proxy modules internally
Yuya Nishihara <yuya@tcha.org>
parents:
32514
diff
changeset
|
75 # map import request to other package or module |
cd2aca0808f8
policy: reroute proxy modules internally
Yuya Nishihara <yuya@tcha.org>
parents:
32514
diff
changeset
|
76 _modredirects = { |
33761
f5fc54e7e467
encoding: drop circular import by proxying through '<policy>.charencode'
Yuya Nishihara <yuya@tcha.org>
parents:
33760
diff
changeset
|
77 (r'cext', r'charencode'): (r'cext', r'parsers'), |
33760
cd2aca0808f8
policy: reroute proxy modules internally
Yuya Nishihara <yuya@tcha.org>
parents:
32514
diff
changeset
|
78 (r'cffi', r'base85'): (r'pure', r'base85'), |
33761
f5fc54e7e467
encoding: drop circular import by proxying through '<policy>.charencode'
Yuya Nishihara <yuya@tcha.org>
parents:
33760
diff
changeset
|
79 (r'cffi', r'charencode'): (r'pure', r'charencode'), |
33760
cd2aca0808f8
policy: reroute proxy modules internally
Yuya Nishihara <yuya@tcha.org>
parents:
32514
diff
changeset
|
80 (r'cffi', r'parsers'): (r'pure', r'parsers'), |
cd2aca0808f8
policy: reroute proxy modules internally
Yuya Nishihara <yuya@tcha.org>
parents:
32514
diff
changeset
|
81 } |
cd2aca0808f8
policy: reroute proxy modules internally
Yuya Nishihara <yuya@tcha.org>
parents:
32514
diff
changeset
|
82 |
32366
8e0327dae3f4
policy: add helper to import cext/pure module
Yuya Nishihara <yuya@tcha.org>
parents:
32251
diff
changeset
|
83 def _checkmod(pkgname, modname, mod): |
32511
2e431fb98c6b
policy: extend API version checks for cffi
Yuya Nishihara <yuya@tcha.org>
parents:
32428
diff
changeset
|
84 expected = _cextversions.get((pkgname, modname)) |
32366
8e0327dae3f4
policy: add helper to import cext/pure module
Yuya Nishihara <yuya@tcha.org>
parents:
32251
diff
changeset
|
85 actual = getattr(mod, r'version', None) |
8e0327dae3f4
policy: add helper to import cext/pure module
Yuya Nishihara <yuya@tcha.org>
parents:
32251
diff
changeset
|
86 if actual != expected: |
8e0327dae3f4
policy: add helper to import cext/pure module
Yuya Nishihara <yuya@tcha.org>
parents:
32251
diff
changeset
|
87 raise ImportError(r'cannot import module %s.%s ' |
8e0327dae3f4
policy: add helper to import cext/pure module
Yuya Nishihara <yuya@tcha.org>
parents:
32251
diff
changeset
|
88 r'(expected version: %d, actual: %r)' |
8e0327dae3f4
policy: add helper to import cext/pure module
Yuya Nishihara <yuya@tcha.org>
parents:
32251
diff
changeset
|
89 % (pkgname, modname, expected, actual)) |
8e0327dae3f4
policy: add helper to import cext/pure module
Yuya Nishihara <yuya@tcha.org>
parents:
32251
diff
changeset
|
90 |
8e0327dae3f4
policy: add helper to import cext/pure module
Yuya Nishihara <yuya@tcha.org>
parents:
32251
diff
changeset
|
91 def importmod(modname): |
8e0327dae3f4
policy: add helper to import cext/pure module
Yuya Nishihara <yuya@tcha.org>
parents:
32251
diff
changeset
|
92 """Import module according to policy and check API version""" |
8e0327dae3f4
policy: add helper to import cext/pure module
Yuya Nishihara <yuya@tcha.org>
parents:
32251
diff
changeset
|
93 try: |
8e0327dae3f4
policy: add helper to import cext/pure module
Yuya Nishihara <yuya@tcha.org>
parents:
32251
diff
changeset
|
94 verpkg, purepkg = _packageprefs[policy] |
8e0327dae3f4
policy: add helper to import cext/pure module
Yuya Nishihara <yuya@tcha.org>
parents:
32251
diff
changeset
|
95 except KeyError: |
8e0327dae3f4
policy: add helper to import cext/pure module
Yuya Nishihara <yuya@tcha.org>
parents:
32251
diff
changeset
|
96 raise ImportError(r'invalid HGMODULEPOLICY %r' % policy) |
8e0327dae3f4
policy: add helper to import cext/pure module
Yuya Nishihara <yuya@tcha.org>
parents:
32251
diff
changeset
|
97 assert verpkg or purepkg |
8e0327dae3f4
policy: add helper to import cext/pure module
Yuya Nishihara <yuya@tcha.org>
parents:
32251
diff
changeset
|
98 if verpkg: |
33760
cd2aca0808f8
policy: reroute proxy modules internally
Yuya Nishihara <yuya@tcha.org>
parents:
32514
diff
changeset
|
99 pn, mn = _modredirects.get((verpkg, modname), (verpkg, modname)) |
32366
8e0327dae3f4
policy: add helper to import cext/pure module
Yuya Nishihara <yuya@tcha.org>
parents:
32251
diff
changeset
|
100 try: |
33760
cd2aca0808f8
policy: reroute proxy modules internally
Yuya Nishihara <yuya@tcha.org>
parents:
32514
diff
changeset
|
101 mod = _importfrom(pn, mn) |
cd2aca0808f8
policy: reroute proxy modules internally
Yuya Nishihara <yuya@tcha.org>
parents:
32514
diff
changeset
|
102 if pn == verpkg: |
cd2aca0808f8
policy: reroute proxy modules internally
Yuya Nishihara <yuya@tcha.org>
parents:
32514
diff
changeset
|
103 _checkmod(pn, mn, mod) |
32366
8e0327dae3f4
policy: add helper to import cext/pure module
Yuya Nishihara <yuya@tcha.org>
parents:
32251
diff
changeset
|
104 return mod |
8e0327dae3f4
policy: add helper to import cext/pure module
Yuya Nishihara <yuya@tcha.org>
parents:
32251
diff
changeset
|
105 except ImportError: |
8e0327dae3f4
policy: add helper to import cext/pure module
Yuya Nishihara <yuya@tcha.org>
parents:
32251
diff
changeset
|
106 if not purepkg: |
8e0327dae3f4
policy: add helper to import cext/pure module
Yuya Nishihara <yuya@tcha.org>
parents:
32251
diff
changeset
|
107 raise |
33760
cd2aca0808f8
policy: reroute proxy modules internally
Yuya Nishihara <yuya@tcha.org>
parents:
32514
diff
changeset
|
108 pn, mn = _modredirects.get((purepkg, modname), (purepkg, modname)) |
cd2aca0808f8
policy: reroute proxy modules internally
Yuya Nishihara <yuya@tcha.org>
parents:
32514
diff
changeset
|
109 return _importfrom(pn, mn) |