annotate mercurial/cffi/osutil.py @ 39270:37e56607cbb9

lfs: add a progress bar when searching for blobs to upload The search itself can take an extreme amount of time if there are a lot of revisions involved. I've got a local repo that took 6 minutes to push 1850 commits, and 60% of that time was spent here (there are ~70K files): \ 58.1% wrapper.py: extractpointers line 297: pointers = extractpointers(... | 57.7% wrapper.py: pointersfromctx line 352: for p in pointersfromctx(ct... | 57.4% wrapper.py: pointerfromctx line 397: p = pointerfromctx(ctx, f, ... \ 38.7% context.py: __contains__ line 368: if f not in ctx: | 38.7% util.py: __get__ line 82: return key in self._manifest | 38.7% context.py: _manifest line 1416: result = self.func(obj) | 38.7% manifest.py: read line 472: return self._manifestctx.re... \ 25.6% revlog.py: revision line 1562: text = rl.revision(self._node) \ 12.8% revlog.py: _chunks line 2217: bins = self._chunks(chain, ... | 12.0% revlog.py: decompressline 2112: ladd(decomp(buffer(data, ch... \ 7.8% revlog.py: checkhash line 2232: self.checkhash(text, node, ... | 7.8% revlog.py: hash line 2315: if node != self.hash(text, ... | 7.8% revlog.py: hash line 2242: return hash(text, p1, p2) \ 12.0% manifest.py: __init__ line 1565: self._data = manifestdict(t... \ 16.8% context.py: filenode line 378: if not _islfs(fctx.filelog(... | 15.7% util.py: __get__ line 706: return self._filelog | 14.8% context.py: _filelog line 1416: result = self.func(obj) | 14.8% localrepo.py: file line 629: return self._repo.file(self... | 14.8% filelog.py: __init__ line 1134: return filelog.filelog(self... | 14.5% revlog.py: __init__ line 24: censorable=True)
author Matt Harbison <matt_harbison@yahoo.com>
date Fri, 24 Aug 2018 17:45:46 -0400
parents dacfcdd8b94e
children 2372284d9457
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
32512
0e8b0b9a7acc cffi: split modules from pure
Yuya Nishihara <yuya@tcha.org>
parents: 32506
diff changeset
1 # osutil.py - CFFI version of osutil.c
8232
823f25b25dea pure/osutil: add copyright and license header
Martin Geisler <mg@lazybytes.net>
parents: 7704
diff changeset
2 #
32512
0e8b0b9a7acc cffi: split modules from pure
Yuya Nishihara <yuya@tcha.org>
parents: 32506
diff changeset
3 # Copyright 2016 Maciej Fijalkowski <fijall@gmail.com>
8232
823f25b25dea pure/osutil: add copyright and license header
Martin Geisler <mg@lazybytes.net>
parents: 7704
diff changeset
4 #
823f25b25dea pure/osutil: add copyright and license header
Martin Geisler <mg@lazybytes.net>
parents: 7704
diff changeset
5 # This software may be used and distributed according to the terms of the
10263
25e572394f5c Update license to GPLv2+
Matt Mackall <mpm@selenic.com>
parents: 9031
diff changeset
6 # GNU General Public License version 2 or any later version.
8232
823f25b25dea pure/osutil: add copyright and license header
Martin Geisler <mg@lazybytes.net>
parents: 7704
diff changeset
7
27338
810337ae1b76 osutil: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25645
diff changeset
8 from __future__ import absolute_import
810337ae1b76 osutil: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25645
diff changeset
9
7704
30d1d313370b move mercurial.osutil to mercurial.pure.osutil
Martin Geisler <mg@daimi.au.dk>
parents:
diff changeset
10 import os
10651
5f091fc1bab7 style: use consistent variable names (*mod) with imports which would shadow
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 10282
diff changeset
11 import stat as statmod
7704
30d1d313370b move mercurial.osutil to mercurial.pure.osutil
Martin Geisler <mg@daimi.au.dk>
parents:
diff changeset
12
32512
0e8b0b9a7acc cffi: split modules from pure
Yuya Nishihara <yuya@tcha.org>
parents: 32506
diff changeset
13 from ..pure.osutil import *
0e8b0b9a7acc cffi: split modules from pure
Yuya Nishihara <yuya@tcha.org>
parents: 32506
diff changeset
14
32367
a9c71d578a1c osutil: switch to policy importer
Yuya Nishihara <yuya@tcha.org>
parents: 31644
diff changeset
15 from .. import (
30304
ba2c04059317 py3: use pycompat.ossep at certain places
Pulkit Goyal <7895pulkit@gmail.com>
parents: 29821
diff changeset
16 pycompat,
ba2c04059317 py3: use pycompat.ossep at certain places
Pulkit Goyal <7895pulkit@gmail.com>
parents: 29821
diff changeset
17 )
ba2c04059317 py3: use pycompat.ossep at certain places
Pulkit Goyal <7895pulkit@gmail.com>
parents: 29821
diff changeset
18
34647
dacfcdd8b94e codemod: use pycompat.isdarwin
Jun Wu <quark@fb.com>
parents: 33572
diff changeset
19 if pycompat.isdarwin:
32512
0e8b0b9a7acc cffi: split modules from pure
Yuya Nishihara <yuya@tcha.org>
parents: 32506
diff changeset
20 from . import _osutil
7704
30d1d313370b move mercurial.osutil to mercurial.pure.osutil
Martin Geisler <mg@daimi.au.dk>
parents:
diff changeset
21
32512
0e8b0b9a7acc cffi: split modules from pure
Yuya Nishihara <yuya@tcha.org>
parents: 32506
diff changeset
22 ffi = _osutil.ffi
0e8b0b9a7acc cffi: split modules from pure
Yuya Nishihara <yuya@tcha.org>
parents: 32506
diff changeset
23 lib = _osutil.lib
7704
30d1d313370b move mercurial.osutil to mercurial.pure.osutil
Martin Geisler <mg@daimi.au.dk>
parents:
diff changeset
24
29600
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
25 listdir_batch_size = 4096
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
26 # tweakable number, only affects performance, which chunks
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
27 # of bytes do we get back from getattrlistbulk
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
28
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
29 attrkinds = [None] * 20 # we need the max no for enum VXXX, 20 is plenty
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
30
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
31 attrkinds[lib.VREG] = statmod.S_IFREG
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
32 attrkinds[lib.VDIR] = statmod.S_IFDIR
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
33 attrkinds[lib.VLNK] = statmod.S_IFLNK
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
34 attrkinds[lib.VBLK] = statmod.S_IFBLK
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
35 attrkinds[lib.VCHR] = statmod.S_IFCHR
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
36 attrkinds[lib.VFIFO] = statmod.S_IFIFO
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
37 attrkinds[lib.VSOCK] = statmod.S_IFSOCK
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
38
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
39 class stat_res(object):
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
40 def __init__(self, st_mode, st_mtime, st_size):
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
41 self.st_mode = st_mode
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
42 self.st_mtime = st_mtime
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
43 self.st_size = st_size
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
44
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
45 tv_sec_ofs = ffi.offsetof("struct timespec", "tv_sec")
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
46 buf = ffi.new("char[]", listdir_batch_size)
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
47
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
48 def listdirinternal(dfd, req, stat, skip):
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
49 ret = []
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
50 while True:
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
51 r = lib.getattrlistbulk(dfd, req, buf, listdir_batch_size, 0)
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
52 if r == 0:
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
53 break
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
54 if r == -1:
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
55 raise OSError(ffi.errno, os.strerror(ffi.errno))
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
56 cur = ffi.cast("val_attrs_t*", buf)
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
57 for i in range(r):
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
58 lgt = cur.length
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
59 assert lgt == ffi.cast('uint32_t*', cur)[0]
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
60 ofs = cur.name_info.attr_dataoffset
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
61 str_lgt = cur.name_info.attr_length
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
62 base_ofs = ffi.offsetof('val_attrs_t', 'name_info')
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
63 name = str(ffi.buffer(ffi.cast("char*", cur) + base_ofs + ofs,
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
64 str_lgt - 1))
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
65 tp = attrkinds[cur.obj_type]
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
66 if name == "." or name == "..":
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
67 continue
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
68 if skip == name and tp == statmod.S_ISDIR:
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
69 return []
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
70 if stat:
29821
8656dcac4ce9 osutil: fix the bug on OS X when we return more in listdir
Maciej Fijalkowski <fijall@gmail.com>
parents: 29698
diff changeset
71 mtime = cur.mtime.tv_sec
29600
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
72 mode = (cur.accessmask & ~lib.S_IFMT)| tp
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
73 ret.append((name, tp, stat_res(st_mode=mode, st_mtime=mtime,
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
74 st_size=cur.datalength)))
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
75 else:
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
76 ret.append((name, tp))
29821
8656dcac4ce9 osutil: fix the bug on OS X when we return more in listdir
Maciej Fijalkowski <fijall@gmail.com>
parents: 29698
diff changeset
77 cur = ffi.cast("val_attrs_t*", int(ffi.cast("intptr_t", cur))
8656dcac4ce9 osutil: fix the bug on OS X when we return more in listdir
Maciej Fijalkowski <fijall@gmail.com>
parents: 29698
diff changeset
78 + lgt)
29600
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
79 return ret
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
80
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
81 def listdir(path, stat=False, skip=None):
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
82 req = ffi.new("struct attrlist*")
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
83 req.bitmapcount = lib.ATTR_BIT_MAP_COUNT
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
84 req.commonattr = (lib.ATTR_CMN_RETURNED_ATTRS |
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
85 lib.ATTR_CMN_NAME |
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
86 lib.ATTR_CMN_OBJTYPE |
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
87 lib.ATTR_CMN_ACCESSMASK |
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
88 lib.ATTR_CMN_MODTIME)
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
89 req.fileattr = lib.ATTR_FILE_DATALENGTH
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
90 dfd = lib.open(path, lib.O_RDONLY, 0)
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
91 if dfd == -1:
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
92 raise OSError(ffi.errno, os.strerror(ffi.errno))
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
93
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
94 try:
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
95 ret = listdirinternal(dfd, req, stat, skip)
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
96 finally:
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
97 try:
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
98 lib.close(dfd)
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
99 except BaseException:
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
100 pass # we ignore all the errors from closing, not
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
101 # much we can do about that
7a157639b8f2 osutil: add darwin-only version of os.listdir using cffi
Maciej Fijalkowski <fijall@gmail.com>
parents: 27971
diff changeset
102 return ret