Mercurial > hg
annotate mercurial/cffi/osutil.py @ 40626:87a872555e90
revlog: detect incomplete revlog reads
_readsegment() is supposed to return N bytes of revlog revision
data starting at a file offset. Surprisingly, its behavior before
this patch never verified that it actually read and returned N
bytes! Instead, it would perform the read(), then return whatever
data was available. And even more surprisingly, nothing in the
call chain appears to have been validating that it received all
the data it was expecting.
This behavior could lead to partial or incomplete revision chunks
being operated on. This could result in e.g. cached deltas being
applied against incomplete base revisions. The delta application
process would happily perform this operation. Only hash
verification would detect the corruption and save us.
This commit changes the behavior of raw revlog reading to validate
that we actually read() the number of bytes that were requested.
We will raise a more specific error faster, rather than possibly
have it go undetected or manifest later in the call stack, at
delta application or hash verification.
Differential Revision: https://phab.mercurial-scm.org/D5266
author | Gregory Szorc <gregory.szorc@gmail.com> |
---|---|
date | Tue, 13 Nov 2018 12:30:59 -0800 |
parents | dacfcdd8b94e |
children | 2372284d9457 |
rev | line source |
---|---|
32512
0e8b0b9a7acc
cffi: split modules from pure
Yuya Nishihara <yuya@tcha.org>
parents:
32506
diff
changeset
|
1 # osutil.py - CFFI version of osutil.c |
8232
823f25b25dea
pure/osutil: add copyright and license header
Martin Geisler <mg@lazybytes.net>
parents:
7704
diff
changeset
|
2 # |
32512
0e8b0b9a7acc
cffi: split modules from pure
Yuya Nishihara <yuya@tcha.org>
parents:
32506
diff
changeset
|
3 # Copyright 2016 Maciej Fijalkowski <fijall@gmail.com> |
8232
823f25b25dea
pure/osutil: add copyright and license header
Martin Geisler <mg@lazybytes.net>
parents:
7704
diff
changeset
|
4 # |
823f25b25dea
pure/osutil: add copyright and license header
Martin Geisler <mg@lazybytes.net>
parents:
7704
diff
changeset
|
5 # This software may be used and distributed according to the terms of the |
10263 | 6 # GNU General Public License version 2 or any later version. |
8232
823f25b25dea
pure/osutil: add copyright and license header
Martin Geisler <mg@lazybytes.net>
parents:
7704
diff
changeset
|
7 |
27338
810337ae1b76
osutil: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents:
25645
diff
changeset
|
8 from __future__ import absolute_import |
810337ae1b76
osutil: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents:
25645
diff
changeset
|
9 |
7704
30d1d313370b
move mercurial.osutil to mercurial.pure.osutil
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
10 import os |
10651
5f091fc1bab7
style: use consistent variable names (*mod) with imports which would shadow
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
10282
diff
changeset
|
11 import stat as statmod |
7704
30d1d313370b
move mercurial.osutil to mercurial.pure.osutil
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
12 |
32512
0e8b0b9a7acc
cffi: split modules from pure
Yuya Nishihara <yuya@tcha.org>
parents:
32506
diff
changeset
|
13 from ..pure.osutil import * |
0e8b0b9a7acc
cffi: split modules from pure
Yuya Nishihara <yuya@tcha.org>
parents:
32506
diff
changeset
|
14 |
32367
a9c71d578a1c
osutil: switch to policy importer
Yuya Nishihara <yuya@tcha.org>
parents:
31644
diff
changeset
|
15 from .. import ( |
30304
ba2c04059317
py3: use pycompat.ossep at certain places
Pulkit Goyal <7895pulkit@gmail.com>
parents:
29821
diff
changeset
|
16 pycompat, |
ba2c04059317
py3: use pycompat.ossep at certain places
Pulkit Goyal <7895pulkit@gmail.com>
parents:
29821
diff
changeset
|
17 ) |
ba2c04059317
py3: use pycompat.ossep at certain places
Pulkit Goyal <7895pulkit@gmail.com>
parents:
29821
diff
changeset
|
18 |
34647 | 19 if pycompat.isdarwin: |
32512
0e8b0b9a7acc
cffi: split modules from pure
Yuya Nishihara <yuya@tcha.org>
parents:
32506
diff
changeset
|
20 from . import _osutil |
7704
30d1d313370b
move mercurial.osutil to mercurial.pure.osutil
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
21 |
32512
0e8b0b9a7acc
cffi: split modules from pure
Yuya Nishihara <yuya@tcha.org>
parents:
32506
diff
changeset
|
22 ffi = _osutil.ffi |
0e8b0b9a7acc
cffi: split modules from pure
Yuya Nishihara <yuya@tcha.org>
parents:
32506
diff
changeset
|
23 lib = _osutil.lib |
7704
30d1d313370b
move mercurial.osutil to mercurial.pure.osutil
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
24 |
29600
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
25 listdir_batch_size = 4096 |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
26 # tweakable number, only affects performance, which chunks |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
27 # of bytes do we get back from getattrlistbulk |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
28 |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
29 attrkinds = [None] * 20 # we need the max no for enum VXXX, 20 is plenty |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
30 |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
31 attrkinds[lib.VREG] = statmod.S_IFREG |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
32 attrkinds[lib.VDIR] = statmod.S_IFDIR |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
33 attrkinds[lib.VLNK] = statmod.S_IFLNK |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
34 attrkinds[lib.VBLK] = statmod.S_IFBLK |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
35 attrkinds[lib.VCHR] = statmod.S_IFCHR |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
36 attrkinds[lib.VFIFO] = statmod.S_IFIFO |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
37 attrkinds[lib.VSOCK] = statmod.S_IFSOCK |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
38 |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
39 class stat_res(object): |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
40 def __init__(self, st_mode, st_mtime, st_size): |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
41 self.st_mode = st_mode |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
42 self.st_mtime = st_mtime |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
43 self.st_size = st_size |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
44 |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
45 tv_sec_ofs = ffi.offsetof("struct timespec", "tv_sec") |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
46 buf = ffi.new("char[]", listdir_batch_size) |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
47 |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
48 def listdirinternal(dfd, req, stat, skip): |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
49 ret = [] |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
50 while True: |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
51 r = lib.getattrlistbulk(dfd, req, buf, listdir_batch_size, 0) |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
52 if r == 0: |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
53 break |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
54 if r == -1: |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
55 raise OSError(ffi.errno, os.strerror(ffi.errno)) |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
56 cur = ffi.cast("val_attrs_t*", buf) |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
57 for i in range(r): |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
58 lgt = cur.length |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
59 assert lgt == ffi.cast('uint32_t*', cur)[0] |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
60 ofs = cur.name_info.attr_dataoffset |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
61 str_lgt = cur.name_info.attr_length |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
62 base_ofs = ffi.offsetof('val_attrs_t', 'name_info') |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
63 name = str(ffi.buffer(ffi.cast("char*", cur) + base_ofs + ofs, |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
64 str_lgt - 1)) |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
65 tp = attrkinds[cur.obj_type] |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
66 if name == "." or name == "..": |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
67 continue |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
68 if skip == name and tp == statmod.S_ISDIR: |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
69 return [] |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
70 if stat: |
29821
8656dcac4ce9
osutil: fix the bug on OS X when we return more in listdir
Maciej Fijalkowski <fijall@gmail.com>
parents:
29698
diff
changeset
|
71 mtime = cur.mtime.tv_sec |
29600
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
72 mode = (cur.accessmask & ~lib.S_IFMT)| tp |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
73 ret.append((name, tp, stat_res(st_mode=mode, st_mtime=mtime, |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
74 st_size=cur.datalength))) |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
75 else: |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
76 ret.append((name, tp)) |
29821
8656dcac4ce9
osutil: fix the bug on OS X when we return more in listdir
Maciej Fijalkowski <fijall@gmail.com>
parents:
29698
diff
changeset
|
77 cur = ffi.cast("val_attrs_t*", int(ffi.cast("intptr_t", cur)) |
8656dcac4ce9
osutil: fix the bug on OS X when we return more in listdir
Maciej Fijalkowski <fijall@gmail.com>
parents:
29698
diff
changeset
|
78 + lgt) |
29600
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
79 return ret |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
80 |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
81 def listdir(path, stat=False, skip=None): |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
82 req = ffi.new("struct attrlist*") |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
83 req.bitmapcount = lib.ATTR_BIT_MAP_COUNT |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
84 req.commonattr = (lib.ATTR_CMN_RETURNED_ATTRS | |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
85 lib.ATTR_CMN_NAME | |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
86 lib.ATTR_CMN_OBJTYPE | |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
87 lib.ATTR_CMN_ACCESSMASK | |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
88 lib.ATTR_CMN_MODTIME) |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
89 req.fileattr = lib.ATTR_FILE_DATALENGTH |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
90 dfd = lib.open(path, lib.O_RDONLY, 0) |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
91 if dfd == -1: |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
92 raise OSError(ffi.errno, os.strerror(ffi.errno)) |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
93 |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
94 try: |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
95 ret = listdirinternal(dfd, req, stat, skip) |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
96 finally: |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
97 try: |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
98 lib.close(dfd) |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
99 except BaseException: |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
100 pass # we ignore all the errors from closing, not |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
101 # much we can do about that |
7a157639b8f2
osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents:
27971
diff
changeset
|
102 return ret |