Mercurial > hg
annotate mercurial/store.py @ 43076:2372284d9457
formatting: blacken the codebase
This is using my patch to black
(https://github.com/psf/black/pull/826) so we don't un-wrap collection
literals.
Done with:
hg files 'set:**.py - mercurial/thirdparty/** - "contrib/python-zstandard/**"' | xargs black -S
# skip-blame mass-reformatting only
# no-check-commit reformats foo_bar functions
Differential Revision: https://phab.mercurial-scm.org/D6971
author | Augie Fackler <augie@google.com> |
---|---|
date | Sun, 06 Oct 2019 09:45:02 -0400 |
parents | 3df3b139a43d |
children | 687b865b95ad |
rev | line source |
---|---|
6839
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
1 # store.py - repository store handling for Mercurial |
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
2 # |
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
3 # Copyright 2008 Matt Mackall <mpm@selenic.com> |
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
4 # |
8225
46293a0c7e9f
updated license to be explicit about GPL version 2
Martin Geisler <mg@lazybytes.net>
parents:
8210
diff
changeset
|
5 # This software may be used and distributed according to the terms of the |
10263 | 6 # GNU General Public License version 2 or any later version. |
6839
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
7 |
27480
509159675cdb
store: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26778
diff
changeset
|
8 from __future__ import absolute_import |
509159675cdb
store: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26778
diff
changeset
|
9 |
509159675cdb
store: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26778
diff
changeset
|
10 import errno |
41978
a56487081109
store: don't read the whole fncache in memory
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
41973
diff
changeset
|
11 import functools |
29338
81c38cb9c1a1
store: use hashlib.sha1 directly instead of through util
Augie Fackler <raf@durin42.com>
parents:
29071
diff
changeset
|
12 import hashlib |
27480
509159675cdb
store: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26778
diff
changeset
|
13 import os |
509159675cdb
store: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26778
diff
changeset
|
14 import stat |
509159675cdb
store: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26778
diff
changeset
|
15 |
509159675cdb
store: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26778
diff
changeset
|
16 from .i18n import _ |
509159675cdb
store: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26778
diff
changeset
|
17 from . import ( |
42905
3df3b139a43d
localrepo: push manifestlog and changelog construction code into store
Augie Fackler <augie@google.com>
parents:
42715
diff
changeset
|
18 changelog, |
27480
509159675cdb
store: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26778
diff
changeset
|
19 error, |
42905
3df3b139a43d
localrepo: push manifestlog and changelog construction code into store
Augie Fackler <augie@google.com>
parents:
42715
diff
changeset
|
20 manifest, |
35582
72b91f905065
py3: use node.hex(h.digest()) instead of h.hexdigest()
Pulkit Goyal <7895pulkit@gmail.com>
parents:
34215
diff
changeset
|
21 node, |
32372
df448de7cf3b
parsers: switch to policy importer
Yuya Nishihara <yuya@tcha.org>
parents:
31362
diff
changeset
|
22 policy, |
30077
8f42d8c412c8
py3: make encodefun in store.py compatible with py3k
Mateusz Kwapich <mitrandir@fb.com>
parents:
30076
diff
changeset
|
23 pycompat, |
27480
509159675cdb
store: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26778
diff
changeset
|
24 util, |
31234
9b7a2ef4f27c
vfs: use 'vfs' module directly in 'mercurial.store'
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents:
31219
diff
changeset
|
25 vfs as vfsmod, |
27480
509159675cdb
store: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26778
diff
changeset
|
26 ) |
6840
80e51429cb9a
introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents:
6839
diff
changeset
|
27 |
32372
df448de7cf3b
parsers: switch to policy importer
Yuya Nishihara <yuya@tcha.org>
parents:
31362
diff
changeset
|
28 parsers = policy.importmod(r'parsers') |
41978
a56487081109
store: don't read the whole fncache in memory
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
41973
diff
changeset
|
29 # how much bytes should be read from fncache in one read |
a56487081109
store: don't read the whole fncache in memory
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
41973
diff
changeset
|
30 # It is done to prevent loading large fncache files into memory |
a56487081109
store: don't read the whole fncache in memory
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
41973
diff
changeset
|
31 fncache_chunksize = 10 ** 6 |
32372
df448de7cf3b
parsers: switch to policy importer
Yuya Nishihara <yuya@tcha.org>
parents:
31362
diff
changeset
|
32 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
33 |
40494
9aeb9e2d28a7
store: introduce _matchtrackedpath() and use it to filter store files
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40340
diff
changeset
|
34 def _matchtrackedpath(path, matcher): |
9aeb9e2d28a7
store: introduce _matchtrackedpath() and use it to filter store files
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40340
diff
changeset
|
35 """parses a fncache entry and returns whether the entry is tracking a path |
9aeb9e2d28a7
store: introduce _matchtrackedpath() and use it to filter store files
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40340
diff
changeset
|
36 matched by matcher or not. |
9aeb9e2d28a7
store: introduce _matchtrackedpath() and use it to filter store files
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40340
diff
changeset
|
37 |
9aeb9e2d28a7
store: introduce _matchtrackedpath() and use it to filter store files
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40340
diff
changeset
|
38 If matcher is None, returns True""" |
9aeb9e2d28a7
store: introduce _matchtrackedpath() and use it to filter store files
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40340
diff
changeset
|
39 |
9aeb9e2d28a7
store: introduce _matchtrackedpath() and use it to filter store files
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40340
diff
changeset
|
40 if matcher is None: |
9aeb9e2d28a7
store: introduce _matchtrackedpath() and use it to filter store files
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40340
diff
changeset
|
41 return True |
9aeb9e2d28a7
store: introduce _matchtrackedpath() and use it to filter store files
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40340
diff
changeset
|
42 path = decodedir(path) |
9aeb9e2d28a7
store: introduce _matchtrackedpath() and use it to filter store files
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40340
diff
changeset
|
43 if path.startswith('data/'): |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
44 return matcher(path[len('data/') : -len('.i')]) |
40494
9aeb9e2d28a7
store: introduce _matchtrackedpath() and use it to filter store files
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40340
diff
changeset
|
45 elif path.startswith('meta/'): |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
46 return matcher.visitdir(path[len('meta/') : -len('/00manifest.i')]) |
40494
9aeb9e2d28a7
store: introduce _matchtrackedpath() and use it to filter store files
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40340
diff
changeset
|
47 |
40624
66adfd58cb77
store: raise ProgrammingError if unable to decode a storage path
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40584
diff
changeset
|
48 raise error.ProgrammingError("cannot decode path %s" % path) |
66adfd58cb77
store: raise ProgrammingError if unable to decode a storage path
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40584
diff
changeset
|
49 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
50 |
8531
810387f59696
filelog encoding: move the encoding/decoding into store
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8530
diff
changeset
|
51 # This avoids a collision between a file named foo and a dir named |
810387f59696
filelog encoding: move the encoding/decoding into store
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8530
diff
changeset
|
52 # foo.i or foo.d |
17607
cc58dc47cb5e
store: use fast C implementation of encodedir() if it's available
Adrian Buehlmann <adrian@cadifra.com>
parents:
17605
diff
changeset
|
53 def _encodedir(path): |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
54 ''' |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
33412
diff
changeset
|
55 >>> _encodedir(b'data/foo.i') |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
56 'data/foo.i' |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
33412
diff
changeset
|
57 >>> _encodedir(b'data/foo.i/bla.i') |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
58 'data/foo.i.hg/bla.i' |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
33412
diff
changeset
|
59 >>> _encodedir(b'data/foo.i.hg/bla.i') |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
60 'data/foo.i.hg.hg/bla.i' |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
33412
diff
changeset
|
61 >>> _encodedir(b'data/foo.i\\ndata/foo.i/bla.i\\ndata/foo.i.hg/bla.i\\n') |
17605
e9cc29be3305
store: add multiline doctest case for encodedir()
Adrian Buehlmann <adrian@cadifra.com>
parents:
17604
diff
changeset
|
62 'data/foo.i\\ndata/foo.i.hg/bla.i\\ndata/foo.i.hg.hg/bla.i\\n' |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
63 ''' |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
64 return ( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
65 path.replace(".hg/", ".hg.hg/") |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
66 .replace(".i/", ".i.hg/") |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
67 .replace(".d/", ".d.hg/") |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
68 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
69 |
8531
810387f59696
filelog encoding: move the encoding/decoding into store
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8530
diff
changeset
|
70 |
17607
cc58dc47cb5e
store: use fast C implementation of encodedir() if it's available
Adrian Buehlmann <adrian@cadifra.com>
parents:
17605
diff
changeset
|
71 encodedir = getattr(parsers, 'encodedir', _encodedir) |
cc58dc47cb5e
store: use fast C implementation of encodedir() if it's available
Adrian Buehlmann <adrian@cadifra.com>
parents:
17605
diff
changeset
|
72 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
73 |
8531
810387f59696
filelog encoding: move the encoding/decoding into store
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8530
diff
changeset
|
74 def decodedir(path): |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
75 ''' |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
33412
diff
changeset
|
76 >>> decodedir(b'data/foo.i') |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
77 'data/foo.i' |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
33412
diff
changeset
|
78 >>> decodedir(b'data/foo.i.hg/bla.i') |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
79 'data/foo.i/bla.i' |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
33412
diff
changeset
|
80 >>> decodedir(b'data/foo.i.hg.hg/bla.i') |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
81 'data/foo.i.hg/bla.i' |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
82 ''' |
17586
2f1475da1940
store: remove uneeded startswith('data/') checks in encodedir() and decodedir()
Adrian Buehlmann <adrian@cadifra.com>
parents:
17585
diff
changeset
|
83 if ".hg/" not in path: |
8531
810387f59696
filelog encoding: move the encoding/decoding into store
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8530
diff
changeset
|
84 return path |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
85 return ( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
86 path.replace(".d.hg/", ".d/") |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
87 .replace(".i.hg/", ".i/") |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
88 .replace(".hg.hg/", ".hg/") |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
89 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
90 |
8531
810387f59696
filelog encoding: move the encoding/decoding into store
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8530
diff
changeset
|
91 |
29071
2f58975eb2cb
store: treat range as a generator instead of a list for py3 compat
timeless <timeless@mozdev.org>
parents:
28007
diff
changeset
|
92 def _reserved(): |
2f58975eb2cb
store: treat range as a generator instead of a list for py3 compat
timeless <timeless@mozdev.org>
parents:
28007
diff
changeset
|
93 ''' characters that are problematic for filesystems |
2f58975eb2cb
store: treat range as a generator instead of a list for py3 compat
timeless <timeless@mozdev.org>
parents:
28007
diff
changeset
|
94 |
2f58975eb2cb
store: treat range as a generator instead of a list for py3 compat
timeless <timeless@mozdev.org>
parents:
28007
diff
changeset
|
95 * ascii escapes (0..31) |
2f58975eb2cb
store: treat range as a generator instead of a list for py3 compat
timeless <timeless@mozdev.org>
parents:
28007
diff
changeset
|
96 * ascii hi (126..255) |
2f58975eb2cb
store: treat range as a generator instead of a list for py3 compat
timeless <timeless@mozdev.org>
parents:
28007
diff
changeset
|
97 * windows specials |
2f58975eb2cb
store: treat range as a generator instead of a list for py3 compat
timeless <timeless@mozdev.org>
parents:
28007
diff
changeset
|
98 |
2f58975eb2cb
store: treat range as a generator instead of a list for py3 compat
timeless <timeless@mozdev.org>
parents:
28007
diff
changeset
|
99 these characters will be escaped by encodefunctions |
2f58975eb2cb
store: treat range as a generator instead of a list for py3 compat
timeless <timeless@mozdev.org>
parents:
28007
diff
changeset
|
100 ''' |
30076
400dfded8a29
py3: make the string unicode so its iterable in py3k
Mateusz Kwapich <mitrandir@fb.com>
parents:
29338
diff
changeset
|
101 winreserved = [ord(x) for x in u'\\:*?"<>|'] |
29071
2f58975eb2cb
store: treat range as a generator instead of a list for py3 compat
timeless <timeless@mozdev.org>
parents:
28007
diff
changeset
|
102 for x in range(32): |
2f58975eb2cb
store: treat range as a generator instead of a list for py3 compat
timeless <timeless@mozdev.org>
parents:
28007
diff
changeset
|
103 yield x |
2f58975eb2cb
store: treat range as a generator instead of a list for py3 compat
timeless <timeless@mozdev.org>
parents:
28007
diff
changeset
|
104 for x in range(126, 256): |
2f58975eb2cb
store: treat range as a generator instead of a list for py3 compat
timeless <timeless@mozdev.org>
parents:
28007
diff
changeset
|
105 yield x |
2f58975eb2cb
store: treat range as a generator instead of a list for py3 compat
timeless <timeless@mozdev.org>
parents:
28007
diff
changeset
|
106 for x in winreserved: |
2f58975eb2cb
store: treat range as a generator instead of a list for py3 compat
timeless <timeless@mozdev.org>
parents:
28007
diff
changeset
|
107 yield x |
2f58975eb2cb
store: treat range as a generator instead of a list for py3 compat
timeless <timeless@mozdev.org>
parents:
28007
diff
changeset
|
108 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
109 |
6839
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
110 def _buildencodefun(): |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
111 ''' |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
112 >>> enc, dec = _buildencodefun() |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
113 |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
33412
diff
changeset
|
114 >>> enc(b'nothing/special.txt') |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
115 'nothing/special.txt' |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
33412
diff
changeset
|
116 >>> dec(b'nothing/special.txt') |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
117 'nothing/special.txt' |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
118 |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
33412
diff
changeset
|
119 >>> enc(b'HELLO') |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
120 '_h_e_l_l_o' |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
33412
diff
changeset
|
121 >>> dec(b'_h_e_l_l_o') |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
122 'HELLO' |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
123 |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
33412
diff
changeset
|
124 >>> enc(b'hello:world?') |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
125 'hello~3aworld~3f' |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
33412
diff
changeset
|
126 >>> dec(b'hello~3aworld~3f') |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
127 'hello:world?' |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
128 |
34136
414a3513c2bd
doctest: do not embed non-ascii characters in docstring
Yuya Nishihara <yuya@tcha.org>
parents:
34131
diff
changeset
|
129 >>> enc(b'the\\x07quick\\xADshot') |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
130 'the~07quick~adshot' |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
33412
diff
changeset
|
131 >>> dec(b'the~07quick~adshot') |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
132 'the\\x07quick\\xadshot' |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
133 ''' |
6839
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
134 e = '_' |
31253
64596338ba10
py3: factor out bytechr() function
Yuya Nishihara <yuya@tcha.org>
parents:
31234
diff
changeset
|
135 xchr = pycompat.bytechr |
64596338ba10
py3: factor out bytechr() function
Yuya Nishihara <yuya@tcha.org>
parents:
31234
diff
changeset
|
136 asciistr = list(map(xchr, range(127))) |
30077
8f42d8c412c8
py3: make encodefun in store.py compatible with py3k
Mateusz Kwapich <mitrandir@fb.com>
parents:
30076
diff
changeset
|
137 capitals = list(range(ord("A"), ord("Z") + 1)) |
8f42d8c412c8
py3: make encodefun in store.py compatible with py3k
Mateusz Kwapich <mitrandir@fb.com>
parents:
30076
diff
changeset
|
138 |
30108
e1e7190457cf
store: py26 compat, don't use a dict comprehension
Martijn Pieters <mjpieters@fb.com>
parents:
30077
diff
changeset
|
139 cmap = dict((x, x) for x in asciistr) |
29071
2f58975eb2cb
store: treat range as a generator instead of a list for py3 compat
timeless <timeless@mozdev.org>
parents:
28007
diff
changeset
|
140 for x in _reserved(): |
30077
8f42d8c412c8
py3: make encodefun in store.py compatible with py3k
Mateusz Kwapich <mitrandir@fb.com>
parents:
30076
diff
changeset
|
141 cmap[xchr(x)] = "~%02x" % x |
8f42d8c412c8
py3: make encodefun in store.py compatible with py3k
Mateusz Kwapich <mitrandir@fb.com>
parents:
30076
diff
changeset
|
142 for x in capitals + [ord(e)]: |
8f42d8c412c8
py3: make encodefun in store.py compatible with py3k
Mateusz Kwapich <mitrandir@fb.com>
parents:
30076
diff
changeset
|
143 cmap[xchr(x)] = e + xchr(x).lower() |
8f42d8c412c8
py3: make encodefun in store.py compatible with py3k
Mateusz Kwapich <mitrandir@fb.com>
parents:
30076
diff
changeset
|
144 |
6839
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
145 dmap = {} |
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
146 for k, v in cmap.iteritems(): |
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
147 dmap[v] = k |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
148 |
6839
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
149 def decode(s): |
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
150 i = 0 |
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
151 while i < len(s): |
38783
e7aa113b14f7
global: use pycompat.xrange()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
38696
diff
changeset
|
152 for l in pycompat.xrange(1, 4): |
6839
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
153 try: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
154 yield dmap[s[i : i + l]] |
6839
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
155 i += l |
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
156 break |
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
157 except KeyError: |
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
158 pass |
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
159 else: |
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
160 raise KeyError |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
161 |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
162 return ( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
163 lambda s: ''.join( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
164 [cmap[s[c : c + 1]] for c in pycompat.xrange(len(s))] |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
165 ), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
166 lambda s: ''.join(list(decode(s))), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
167 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
168 |
17608
776240123525
store: extract functions _encodefname and _decodefname
Adrian Buehlmann <adrian@cadifra.com>
parents:
17607
diff
changeset
|
169 |
776240123525
store: extract functions _encodefname and _decodefname
Adrian Buehlmann <adrian@cadifra.com>
parents:
17607
diff
changeset
|
170 _encodefname, _decodefname = _buildencodefun() |
6839
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
171 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
172 |
17608
776240123525
store: extract functions _encodefname and _decodefname
Adrian Buehlmann <adrian@cadifra.com>
parents:
17607
diff
changeset
|
173 def encodefilename(s): |
776240123525
store: extract functions _encodefname and _decodefname
Adrian Buehlmann <adrian@cadifra.com>
parents:
17607
diff
changeset
|
174 ''' |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
33412
diff
changeset
|
175 >>> encodefilename(b'foo.i/bar.d/bla.hg/hi:world?/HELLO') |
17608
776240123525
store: extract functions _encodefname and _decodefname
Adrian Buehlmann <adrian@cadifra.com>
parents:
17607
diff
changeset
|
176 'foo.i.hg/bar.d.hg/bla.hg.hg/hi~3aworld~3f/_h_e_l_l_o' |
776240123525
store: extract functions _encodefname and _decodefname
Adrian Buehlmann <adrian@cadifra.com>
parents:
17607
diff
changeset
|
177 ''' |
776240123525
store: extract functions _encodefname and _decodefname
Adrian Buehlmann <adrian@cadifra.com>
parents:
17607
diff
changeset
|
178 return _encodefname(encodedir(s)) |
776240123525
store: extract functions _encodefname and _decodefname
Adrian Buehlmann <adrian@cadifra.com>
parents:
17607
diff
changeset
|
179 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
180 |
17608
776240123525
store: extract functions _encodefname and _decodefname
Adrian Buehlmann <adrian@cadifra.com>
parents:
17607
diff
changeset
|
181 def decodefilename(s): |
776240123525
store: extract functions _encodefname and _decodefname
Adrian Buehlmann <adrian@cadifra.com>
parents:
17607
diff
changeset
|
182 ''' |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
33412
diff
changeset
|
183 >>> decodefilename(b'foo.i.hg/bar.d.hg/bla.hg.hg/hi~3aworld~3f/_h_e_l_l_o') |
17608
776240123525
store: extract functions _encodefname and _decodefname
Adrian Buehlmann <adrian@cadifra.com>
parents:
17607
diff
changeset
|
184 'foo.i/bar.d/bla.hg/hi:world?/HELLO' |
776240123525
store: extract functions _encodefname and _decodefname
Adrian Buehlmann <adrian@cadifra.com>
parents:
17607
diff
changeset
|
185 ''' |
776240123525
store: extract functions _encodefname and _decodefname
Adrian Buehlmann <adrian@cadifra.com>
parents:
17607
diff
changeset
|
186 return decodedir(_decodefname(s)) |
6839
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
187 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
188 |
14288
00a0ab08f986
store: change names to comply with project coding standards
Adrian Buehlmann <adrian@cadifra.com>
parents:
14194
diff
changeset
|
189 def _buildlowerencodefun(): |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
190 ''' |
14288
00a0ab08f986
store: change names to comply with project coding standards
Adrian Buehlmann <adrian@cadifra.com>
parents:
14194
diff
changeset
|
191 >>> f = _buildlowerencodefun() |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
33412
diff
changeset
|
192 >>> f(b'nothing/special.txt') |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
193 'nothing/special.txt' |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
33412
diff
changeset
|
194 >>> f(b'HELLO') |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
195 'hello' |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
33412
diff
changeset
|
196 >>> f(b'hello:world?') |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
197 'hello~3aworld~3f' |
34136
414a3513c2bd
doctest: do not embed non-ascii characters in docstring
Yuya Nishihara <yuya@tcha.org>
parents:
34131
diff
changeset
|
198 >>> f(b'the\\x07quick\\xADshot') |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
199 'the~07quick~adshot' |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
200 ''' |
34214
7e3f078b6f31
py3: use bytechr() in store._buildlowerencodefun()
Yuya Nishihara <yuya@tcha.org>
parents:
34213
diff
changeset
|
201 xchr = pycompat.bytechr |
38783
e7aa113b14f7
global: use pycompat.xrange()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
38696
diff
changeset
|
202 cmap = dict([(xchr(x), xchr(x)) for x in pycompat.xrange(127)]) |
29071
2f58975eb2cb
store: treat range as a generator instead of a list for py3 compat
timeless <timeless@mozdev.org>
parents:
28007
diff
changeset
|
203 for x in _reserved(): |
34214
7e3f078b6f31
py3: use bytechr() in store._buildlowerencodefun()
Yuya Nishihara <yuya@tcha.org>
parents:
34213
diff
changeset
|
204 cmap[xchr(x)] = "~%02x" % x |
18054
b35e3364f94a
check-code: there must also be whitespace between ')' and operator
Mads Kiilerich <madski@unity3d.com>
parents:
17845
diff
changeset
|
205 for x in range(ord("A"), ord("Z") + 1): |
34214
7e3f078b6f31
py3: use bytechr() in store._buildlowerencodefun()
Yuya Nishihara <yuya@tcha.org>
parents:
34213
diff
changeset
|
206 cmap[xchr(x)] = xchr(x).lower() |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
207 |
34213
96808804b68f
store: give name to lowerencode function
Yuya Nishihara <yuya@tcha.org>
parents:
34136
diff
changeset
|
208 def lowerencode(s): |
34215
b4abc438a8c9
py3: iterate bytes as a byte string in store.lowerencode()
Yuya Nishihara <yuya@tcha.org>
parents:
34214
diff
changeset
|
209 return "".join([cmap[c] for c in pycompat.iterbytestr(s)]) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
210 |
34213
96808804b68f
store: give name to lowerencode function
Yuya Nishihara <yuya@tcha.org>
parents:
34136
diff
changeset
|
211 return lowerencode |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
212 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
213 |
18430
0459c6555f69
store: implement lowerencode in C
Bryan O'Sullivan <bryano@fb.com>
parents:
18054
diff
changeset
|
214 lowerencode = getattr(parsers, 'lowerencode', None) or _buildlowerencodefun() |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
215 |
17570
f53a7b256ca6
store: optimze _auxencode() a bit by grouping the reserved names by length
Adrian Buehlmann <adrian@cadifra.com>
parents:
17569
diff
changeset
|
216 # Windows reserved names: con, prn, aux, nul, com1..com9, lpt1..lpt9 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
217 _winres3 = ('aux', 'con', 'prn', 'nul') # length 3 |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
218 _winres4 = ('com', 'lpt') # length 4 (with trailing 1..9) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
219 |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
220 |
12687
34d8247a4595
store: encode first period or space in filenames (issue1713)
Adrian Buehlmann <adrian@cadifra.com>
parents:
12171
diff
changeset
|
221 def _auxencode(path, dotencode): |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
222 ''' |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
223 Encodes filenames containing names reserved by Windows or which end in |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
224 period or space. Does not touch other single reserved characters c. |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
225 Specifically, c in '\\:*?"<>|' or ord(c) <= 31 are *not* encoded here. |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
226 Additionally encodes space or period at the beginning, if dotencode is |
17569
e9af2134825c
store: explain "aux.foo" versus "foo.aux" in doc of _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents:
17568
diff
changeset
|
227 True. Parameter path is assumed to be all lowercase. |
e9af2134825c
store: explain "aux.foo" versus "foo.aux" in doc of _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents:
17568
diff
changeset
|
228 A segment only needs encoding if a reserved name appears as a |
e9af2134825c
store: explain "aux.foo" versus "foo.aux" in doc of _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents:
17568
diff
changeset
|
229 basename (e.g. "aux", "aux.foo"). A directory or file named "foo.aux" |
e9af2134825c
store: explain "aux.foo" versus "foo.aux" in doc of _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents:
17568
diff
changeset
|
230 doesn't need encoding. |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
231 |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
33412
diff
changeset
|
232 >>> s = b'.foo/aux.txt/txt.aux/con/prn/nul/foo.' |
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
33412
diff
changeset
|
233 >>> _auxencode(s.split(b'/'), True) |
17574
81a033bb29bc
store: let _auxencode() return the list of path segments
Adrian Buehlmann <adrian@cadifra.com>
parents:
17573
diff
changeset
|
234 ['~2efoo', 'au~78.txt', 'txt.aux', 'co~6e', 'pr~6e', 'nu~6c', 'foo~2e'] |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
33412
diff
changeset
|
235 >>> s = b'.com1com2/lpt9.lpt4.lpt1/conprn/com0/lpt0/foo.' |
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
33412
diff
changeset
|
236 >>> _auxencode(s.split(b'/'), False) |
17574
81a033bb29bc
store: let _auxencode() return the list of path segments
Adrian Buehlmann <adrian@cadifra.com>
parents:
17573
diff
changeset
|
237 ['.com1com2', 'lp~749.lpt4.lpt1', 'conprn', 'com0', 'lpt0', 'foo~2e'] |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
33412
diff
changeset
|
238 >>> _auxencode([b'foo. '], True) |
17574
81a033bb29bc
store: let _auxencode() return the list of path segments
Adrian Buehlmann <adrian@cadifra.com>
parents:
17573
diff
changeset
|
239 ['foo.~20'] |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
33412
diff
changeset
|
240 >>> _auxencode([b' .foo'], True) |
17574
81a033bb29bc
store: let _auxencode() return the list of path segments
Adrian Buehlmann <adrian@cadifra.com>
parents:
17573
diff
changeset
|
241 ['~20.foo'] |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
242 ''' |
17589
b11024849db6
store: parameter path of _auxencode is now a list of strings
Adrian Buehlmann <adrian@cadifra.com>
parents:
17588
diff
changeset
|
243 for i, n in enumerate(path): |
17572
b644287e79a8
store: unindent most of the contents of the for loop in _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents:
17571
diff
changeset
|
244 if not n: |
b644287e79a8
store: unindent most of the contents of the for loop in _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents:
17571
diff
changeset
|
245 continue |
b644287e79a8
store: unindent most of the contents of the for loop in _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents:
17571
diff
changeset
|
246 if dotencode and n[0] in '. ': |
31362
50cd81346ad4
store: fix many single-byte ops to use slicing in _auxencode
Augie Fackler <augie@google.com>
parents:
31253
diff
changeset
|
247 n = "~%02x" % ord(n[0:1]) + n[1:] |
17589
b11024849db6
store: parameter path of _auxencode is now a list of strings
Adrian Buehlmann <adrian@cadifra.com>
parents:
17588
diff
changeset
|
248 path[i] = n |
17572
b644287e79a8
store: unindent most of the contents of the for loop in _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents:
17571
diff
changeset
|
249 else: |
b644287e79a8
store: unindent most of the contents of the for loop in _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents:
17571
diff
changeset
|
250 l = n.find('.') |
b644287e79a8
store: unindent most of the contents of the for loop in _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents:
17571
diff
changeset
|
251 if l == -1: |
b644287e79a8
store: unindent most of the contents of the for loop in _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents:
17571
diff
changeset
|
252 l = len(n) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
253 if (l == 3 and n[:3] in _winres3) or ( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
254 l == 4 and n[3:4] <= '9' and n[3:4] >= '1' and n[:3] in _winres4 |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
255 ): |
17572
b644287e79a8
store: unindent most of the contents of the for loop in _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents:
17571
diff
changeset
|
256 # encode third letter ('aux' -> 'au~78') |
31362
50cd81346ad4
store: fix many single-byte ops to use slicing in _auxencode
Augie Fackler <augie@google.com>
parents:
31253
diff
changeset
|
257 ec = "~%02x" % ord(n[2:3]) |
17572
b644287e79a8
store: unindent most of the contents of the for loop in _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents:
17571
diff
changeset
|
258 n = n[0:2] + ec + n[3:] |
17589
b11024849db6
store: parameter path of _auxencode is now a list of strings
Adrian Buehlmann <adrian@cadifra.com>
parents:
17588
diff
changeset
|
259 path[i] = n |
17572
b644287e79a8
store: unindent most of the contents of the for loop in _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents:
17571
diff
changeset
|
260 if n[-1] in '. ': |
b644287e79a8
store: unindent most of the contents of the for loop in _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents:
17571
diff
changeset
|
261 # encode last period or space ('foo...' -> 'foo..~2e') |
31362
50cd81346ad4
store: fix many single-byte ops to use slicing in _auxencode
Augie Fackler <augie@google.com>
parents:
31253
diff
changeset
|
262 path[i] = n[:-1] + "~%02x" % ord(n[-1:]) |
17589
b11024849db6
store: parameter path of _auxencode is now a list of strings
Adrian Buehlmann <adrian@cadifra.com>
parents:
17588
diff
changeset
|
263 return path |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
264 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
265 |
14288
00a0ab08f986
store: change names to comply with project coding standards
Adrian Buehlmann <adrian@cadifra.com>
parents:
14194
diff
changeset
|
266 _maxstorepathlen = 120 |
00a0ab08f986
store: change names to comply with project coding standards
Adrian Buehlmann <adrian@cadifra.com>
parents:
14194
diff
changeset
|
267 _dirprefixlen = 8 |
00a0ab08f986
store: change names to comply with project coding standards
Adrian Buehlmann <adrian@cadifra.com>
parents:
14194
diff
changeset
|
268 _maxshortdirslen = 8 * (_dirprefixlen + 1) - 4 |
17610
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
269 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
270 |
17610
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
271 def _hashencode(path, dotencode): |
35582
72b91f905065
py3: use node.hex(h.digest()) instead of h.hexdigest()
Pulkit Goyal <7895pulkit@gmail.com>
parents:
34215
diff
changeset
|
272 digest = node.hex(hashlib.sha1(path).digest()) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
273 le = lowerencode(path[5:]).split('/') # skips prefix 'data/' or 'meta/' |
17610
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
274 parts = _auxencode(le, dotencode) |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
275 basename = parts[-1] |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
276 _root, ext = os.path.splitext(basename) |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
277 sdirs = [] |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
278 sdirslen = 0 |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
279 for p in parts[:-1]: |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
280 d = p[:_dirprefixlen] |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
281 if d[-1] in '. ': |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
282 # Windows can't access dirs ending in period or space |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
283 d = d[:-1] + '_' |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
284 if sdirslen == 0: |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
285 t = len(d) |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
286 else: |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
287 t = sdirslen + 1 + len(d) |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
288 if t > _maxshortdirslen: |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
289 break |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
290 sdirs.append(d) |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
291 sdirslen = t |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
292 dirs = '/'.join(sdirs) |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
293 if len(dirs) > 0: |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
294 dirs += '/' |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
295 res = 'dh/' + dirs + digest + ext |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
296 spaceleft = _maxstorepathlen - len(res) |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
297 if spaceleft > 0: |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
298 filler = basename[:spaceleft] |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
299 res = 'dh/' + dirs + filler + digest + ext |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
300 return res |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
301 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
302 |
17590
eb0884680f5c
store: eliminate one level of lambda functions on _hybridencode
Adrian Buehlmann <adrian@cadifra.com>
parents:
17589
diff
changeset
|
303 def _hybridencode(path, dotencode): |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
304 '''encodes path with a length limit |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
305 |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
306 Encodes all paths that begin with 'data/', according to the following. |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
307 |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
308 Default encoding (reversible): |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
309 |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
310 Encodes all uppercase letters 'X' as '_x'. All reserved or illegal |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
311 characters are encoded as '~xx', where xx is the two digit hex code |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
312 of the character (see encodefilename). |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
313 Relevant path components consisting of Windows reserved filenames are |
17738
b8424c92ba2b
spelling: fix minor spell checker issues
Mads Kiilerich <mads@kiilerich.com>
parents:
17731
diff
changeset
|
314 masked by encoding the third character ('aux' -> 'au~78', see _auxencode). |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
315 |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
316 Hashed encoding (not reversible): |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
317 |
14288
00a0ab08f986
store: change names to comply with project coding standards
Adrian Buehlmann <adrian@cadifra.com>
parents:
14194
diff
changeset
|
318 If the default-encoded path is longer than _maxstorepathlen, a |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
319 non-reversible hybrid hashing of the path is done instead. |
14288
00a0ab08f986
store: change names to comply with project coding standards
Adrian Buehlmann <adrian@cadifra.com>
parents:
14194
diff
changeset
|
320 This encoding uses up to _dirprefixlen characters of all directory |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
321 levels of the lowerencoded path, but not more levels than can fit into |
14288
00a0ab08f986
store: change names to comply with project coding standards
Adrian Buehlmann <adrian@cadifra.com>
parents:
14194
diff
changeset
|
322 _maxshortdirslen. |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
323 Then follows the filler followed by the sha digest of the full path. |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
324 The filler is the beginning of the basename of the lowerencoded path |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
325 (the basename is everything after the last path separator). The filler |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
326 is as long as possible, filling in characters from the basename until |
14288
00a0ab08f986
store: change names to comply with project coding standards
Adrian Buehlmann <adrian@cadifra.com>
parents:
14194
diff
changeset
|
327 the encoded path has _maxstorepathlen characters (or all chars of the |
00a0ab08f986
store: change names to comply with project coding standards
Adrian Buehlmann <adrian@cadifra.com>
parents:
14194
diff
changeset
|
328 basename have been taken). |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
329 The extension (e.g. '.i' or '.d') is preserved. |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
330 |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
331 The string 'data/' at the beginning is replaced with 'dh/', if the hashed |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
332 encoding was used. |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
333 ''' |
17609
cbc180cfd60b
store: reuse direncoded path in _hybridencode
Adrian Buehlmann <adrian@cadifra.com>
parents:
17608
diff
changeset
|
334 path = encodedir(path) |
cbc180cfd60b
store: reuse direncoded path in _hybridencode
Adrian Buehlmann <adrian@cadifra.com>
parents:
17608
diff
changeset
|
335 ef = _encodefname(path).split('/') |
17590
eb0884680f5c
store: eliminate one level of lambda functions on _hybridencode
Adrian Buehlmann <adrian@cadifra.com>
parents:
17589
diff
changeset
|
336 res = '/'.join(_auxencode(ef, dotencode)) |
14288
00a0ab08f986
store: change names to comply with project coding standards
Adrian Buehlmann <adrian@cadifra.com>
parents:
14194
diff
changeset
|
337 if len(res) > _maxstorepathlen: |
17610
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
338 res = _hashencode(path, dotencode) |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
339 return res |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
340 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
341 |
17624
ae103510f6aa
store: add a fallback _pathencode Python function
Adrian Buehlmann <adrian@cadifra.com>
parents:
17623
diff
changeset
|
342 def _pathencode(path): |
18435
8c019d2fd7c0
store: switch to C-based hashed path encoding
Bryan O'Sullivan <bryano@fb.com>
parents:
18430
diff
changeset
|
343 de = encodedir(path) |
17693
0c6de45e1212
store: optimize _pathencode by checking the length of the unencoded path
Adrian Buehlmann <adrian@cadifra.com>
parents:
17653
diff
changeset
|
344 if len(path) > _maxstorepathlen: |
18435
8c019d2fd7c0
store: switch to C-based hashed path encoding
Bryan O'Sullivan <bryano@fb.com>
parents:
18430
diff
changeset
|
345 return _hashencode(de, True) |
8c019d2fd7c0
store: switch to C-based hashed path encoding
Bryan O'Sullivan <bryano@fb.com>
parents:
18430
diff
changeset
|
346 ef = _encodefname(de).split('/') |
17624
ae103510f6aa
store: add a fallback _pathencode Python function
Adrian Buehlmann <adrian@cadifra.com>
parents:
17623
diff
changeset
|
347 res = '/'.join(_auxencode(ef, True)) |
ae103510f6aa
store: add a fallback _pathencode Python function
Adrian Buehlmann <adrian@cadifra.com>
parents:
17623
diff
changeset
|
348 if len(res) > _maxstorepathlen: |
18435
8c019d2fd7c0
store: switch to C-based hashed path encoding
Bryan O'Sullivan <bryano@fb.com>
parents:
18430
diff
changeset
|
349 return _hashencode(de, True) |
17624
ae103510f6aa
store: add a fallback _pathencode Python function
Adrian Buehlmann <adrian@cadifra.com>
parents:
17623
diff
changeset
|
350 return res |
ae103510f6aa
store: add a fallback _pathencode Python function
Adrian Buehlmann <adrian@cadifra.com>
parents:
17623
diff
changeset
|
351 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
352 |
17624
ae103510f6aa
store: add a fallback _pathencode Python function
Adrian Buehlmann <adrian@cadifra.com>
parents:
17623
diff
changeset
|
353 _pathencode = getattr(parsers, 'pathencode', _pathencode) |
ae103510f6aa
store: add a fallback _pathencode Python function
Adrian Buehlmann <adrian@cadifra.com>
parents:
17623
diff
changeset
|
354 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
355 |
17623
448e6ed7c557
store: move _plainhybridencode and _dothybridencode higher up in the file
Adrian Buehlmann <adrian@cadifra.com>
parents:
17621
diff
changeset
|
356 def _plainhybridencode(f): |
448e6ed7c557
store: move _plainhybridencode and _dothybridencode higher up in the file
Adrian Buehlmann <adrian@cadifra.com>
parents:
17621
diff
changeset
|
357 return _hybridencode(f, False) |
448e6ed7c557
store: move _plainhybridencode and _dothybridencode higher up in the file
Adrian Buehlmann <adrian@cadifra.com>
parents:
17621
diff
changeset
|
358 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
359 |
17726
7cb7e17c23b2
store: invoke "os.stat()" for "createmode" initialization via vfs
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17725
diff
changeset
|
360 def _calcmode(vfs): |
6898
69aeaaaf6e07
store: simplify class hierarchy
Matt Mackall <mpm@selenic.com>
parents:
6897
diff
changeset
|
361 try: |
69aeaaaf6e07
store: simplify class hierarchy
Matt Mackall <mpm@selenic.com>
parents:
6897
diff
changeset
|
362 # files in .hg/ will be created using this mode |
17726
7cb7e17c23b2
store: invoke "os.stat()" for "createmode" initialization via vfs
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17725
diff
changeset
|
363 mode = vfs.stat().st_mode |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
364 # avoid some useless chmods |
25658
e93036747902
global: mass rewrite to use modern octal syntax
Gregory Szorc <gregory.szorc@gmail.com>
parents:
25091
diff
changeset
|
365 if (0o777 & ~util.umask) == (0o777 & mode): |
6898
69aeaaaf6e07
store: simplify class hierarchy
Matt Mackall <mpm@selenic.com>
parents:
6897
diff
changeset
|
366 mode = None |
69aeaaaf6e07
store: simplify class hierarchy
Matt Mackall <mpm@selenic.com>
parents:
6897
diff
changeset
|
367 except OSError: |
69aeaaaf6e07
store: simplify class hierarchy
Matt Mackall <mpm@selenic.com>
parents:
6897
diff
changeset
|
368 mode = None |
69aeaaaf6e07
store: simplify class hierarchy
Matt Mackall <mpm@selenic.com>
parents:
6897
diff
changeset
|
369 return mode |
69aeaaaf6e07
store: simplify class hierarchy
Matt Mackall <mpm@selenic.com>
parents:
6897
diff
changeset
|
370 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
371 |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
372 _data = ( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
373 'bookmarks narrowspec data meta 00manifest.d 00manifest.i' |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
374 ' 00changelog.d 00changelog.i phaseroots obsstore' |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
375 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
376 |
6903
0642d9d7ec80
clone: get a list of files to clone from store
Matt Mackall <mpm@selenic.com>
parents:
6902
diff
changeset
|
377 |
37409
4c15bee42e9c
store: make file filtering during walk configurable
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35582
diff
changeset
|
378 def isrevlog(f, kind, st): |
4c15bee42e9c
store: make file filtering during walk configurable
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35582
diff
changeset
|
379 return kind == stat.S_IFREG and f[-2:] in ('.i', '.d') |
4c15bee42e9c
store: make file filtering during walk configurable
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35582
diff
changeset
|
380 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
381 |
8778
c5f36402daad
use new style classes
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8633
diff
changeset
|
382 class basicstore(object): |
6840
80e51429cb9a
introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents:
6839
diff
changeset
|
383 '''base class for local repository stores''' |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
384 |
17651
3b49c28658f6
store: rename "openertype" argument to "vfstype"
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17649
diff
changeset
|
385 def __init__(self, path, vfstype): |
17724
bf4b72d8dd4d
store: initialize vfs field first to use it for initialization of others
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17722
diff
changeset
|
386 vfs = vfstype(path) |
bf4b72d8dd4d
store: initialize vfs field first to use it for initialization of others
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17722
diff
changeset
|
387 self.path = vfs.base |
17726
7cb7e17c23b2
store: invoke "os.stat()" for "createmode" initialization via vfs
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17725
diff
changeset
|
388 self.createmode = _calcmode(vfs) |
17652
2c6f7231becc
store: rename "op" variables to "vfs"
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17651
diff
changeset
|
389 vfs.createmode = self.createmode |
17728
004bd533880d
store: invoke "os.path.isdir()" via vfs
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17727
diff
changeset
|
390 self.rawvfs = vfs |
31234
9b7a2ef4f27c
vfs: use 'vfs' module directly in 'mercurial.store'
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents:
31219
diff
changeset
|
391 self.vfs = vfsmod.filtervfs(vfs, encodedir) |
17653
dacb50696b75
store: initialize "vfs" fields by "vfs" constructors
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17652
diff
changeset
|
392 self.opener = self.vfs |
6840
80e51429cb9a
introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents:
6839
diff
changeset
|
393 |
80e51429cb9a
introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents:
6839
diff
changeset
|
394 def join(self, f): |
13426
643b8212813e
store: remove pointless pathjoiner parameter
Adrian Buehlmann <adrian@cadifra.com>
parents:
13391
diff
changeset
|
395 return self.path + '/' + encodedir(f) |
6840
80e51429cb9a
introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents:
6839
diff
changeset
|
396 |
37409
4c15bee42e9c
store: make file filtering during walk configurable
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35582
diff
changeset
|
397 def _walk(self, relpath, recurse, filefilter=isrevlog): |
6900
def492d1b592
store: change handling of decoding errors
Matt Mackall <mpm@selenic.com>
parents:
6899
diff
changeset
|
398 '''yields (unencoded, encoded, size)''' |
13426
643b8212813e
store: remove pointless pathjoiner parameter
Adrian Buehlmann <adrian@cadifra.com>
parents:
13391
diff
changeset
|
399 path = self.path |
643b8212813e
store: remove pointless pathjoiner parameter
Adrian Buehlmann <adrian@cadifra.com>
parents:
13391
diff
changeset
|
400 if relpath: |
643b8212813e
store: remove pointless pathjoiner parameter
Adrian Buehlmann <adrian@cadifra.com>
parents:
13391
diff
changeset
|
401 path += '/' + relpath |
643b8212813e
store: remove pointless pathjoiner parameter
Adrian Buehlmann <adrian@cadifra.com>
parents:
13391
diff
changeset
|
402 striplen = len(self.path) + 1 |
6899 | 403 l = [] |
17728
004bd533880d
store: invoke "os.path.isdir()" via vfs
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17727
diff
changeset
|
404 if self.rawvfs.isdir(path): |
6899 | 405 visit = [path] |
17747
aad3bce98f76
store: invoke "osutil.listdir()" via vfs
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17745
diff
changeset
|
406 readdir = self.rawvfs.readdir |
6899 | 407 while visit: |
408 p = visit.pop() | |
17747
aad3bce98f76
store: invoke "osutil.listdir()" via vfs
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17745
diff
changeset
|
409 for f, kind, st in readdir(p, stat=True): |
13426
643b8212813e
store: remove pointless pathjoiner parameter
Adrian Buehlmann <adrian@cadifra.com>
parents:
13391
diff
changeset
|
410 fp = p + '/' + f |
37409
4c15bee42e9c
store: make file filtering during walk configurable
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35582
diff
changeset
|
411 if filefilter(f, kind, st): |
6900
def492d1b592
store: change handling of decoding errors
Matt Mackall <mpm@selenic.com>
parents:
6899
diff
changeset
|
412 n = util.pconvert(fp[striplen:]) |
8531
810387f59696
filelog encoding: move the encoding/decoding into store
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8530
diff
changeset
|
413 l.append((decodedir(n), n, st.st_size)) |
6899 | 414 elif kind == stat.S_IFDIR and recurse: |
415 visit.append(fp) | |
17054
125ff5654b72
store: sort filenames in place
Bryan O'Sullivan <bryano@fb.com>
parents:
16404
diff
changeset
|
416 l.sort() |
125ff5654b72
store: sort filenames in place
Bryan O'Sullivan <bryano@fb.com>
parents:
16404
diff
changeset
|
417 return l |
6840
80e51429cb9a
introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents:
6839
diff
changeset
|
418 |
42905
3df3b139a43d
localrepo: push manifestlog and changelog construction code into store
Augie Fackler <augie@google.com>
parents:
42715
diff
changeset
|
419 def changelog(self, trypending): |
3df3b139a43d
localrepo: push manifestlog and changelog construction code into store
Augie Fackler <augie@google.com>
parents:
42715
diff
changeset
|
420 return changelog.changelog(self.vfs, trypending=trypending) |
3df3b139a43d
localrepo: push manifestlog and changelog construction code into store
Augie Fackler <augie@google.com>
parents:
42715
diff
changeset
|
421 |
3df3b139a43d
localrepo: push manifestlog and changelog construction code into store
Augie Fackler <augie@google.com>
parents:
42715
diff
changeset
|
422 def manifestlog(self, repo, storenarrowmatch): |
3df3b139a43d
localrepo: push manifestlog and changelog construction code into store
Augie Fackler <augie@google.com>
parents:
42715
diff
changeset
|
423 rootstore = manifest.manifestrevlog(self.vfs) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
424 return manifest.manifestlog(self.vfs, repo, rootstore, storenarrowmatch) |
42905
3df3b139a43d
localrepo: push manifestlog and changelog construction code into store
Augie Fackler <augie@google.com>
parents:
42715
diff
changeset
|
425 |
40340
2d45b549392f
store: pass matcher to store.datafiles()
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
39698
diff
changeset
|
426 def datafiles(self, matcher=None): |
28007
fb92927f9775
treemanifests: fix streaming clone
Martin von Zweigbergk <martinvonz@google.com>
parents:
28006
diff
changeset
|
427 return self._walk('data', True) + self._walk('meta', True) |
6840
80e51429cb9a
introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents:
6839
diff
changeset
|
428 |
19177
1e104aaa4c44
store: move top file walk to a separate function
Durham Goode <durham@fb.com>
parents:
18435
diff
changeset
|
429 def topfiles(self): |
1e104aaa4c44
store: move top file walk to a separate function
Durham Goode <durham@fb.com>
parents:
18435
diff
changeset
|
430 # yield manifest before changelog |
1e104aaa4c44
store: move top file walk to a separate function
Durham Goode <durham@fb.com>
parents:
18435
diff
changeset
|
431 return reversed(self._walk('', False)) |
1e104aaa4c44
store: move top file walk to a separate function
Durham Goode <durham@fb.com>
parents:
18435
diff
changeset
|
432 |
40340
2d45b549392f
store: pass matcher to store.datafiles()
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
39698
diff
changeset
|
433 def walk(self, matcher=None): |
2d45b549392f
store: pass matcher to store.datafiles()
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
39698
diff
changeset
|
434 '''yields (unencoded, encoded, size) |
2d45b549392f
store: pass matcher to store.datafiles()
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
39698
diff
changeset
|
435 |
2d45b549392f
store: pass matcher to store.datafiles()
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
39698
diff
changeset
|
436 if a matcher is passed, storage files of only those tracked paths |
2d45b549392f
store: pass matcher to store.datafiles()
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
39698
diff
changeset
|
437 are passed with matches the matcher |
2d45b549392f
store: pass matcher to store.datafiles()
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
39698
diff
changeset
|
438 ''' |
6840
80e51429cb9a
introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents:
6839
diff
changeset
|
439 # yield data files first |
40340
2d45b549392f
store: pass matcher to store.datafiles()
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
39698
diff
changeset
|
440 for x in self.datafiles(matcher): |
6840
80e51429cb9a
introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents:
6839
diff
changeset
|
441 yield x |
19177
1e104aaa4c44
store: move top file walk to a separate function
Durham Goode <durham@fb.com>
parents:
18435
diff
changeset
|
442 for x in self.topfiles(): |
6840
80e51429cb9a
introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents:
6839
diff
changeset
|
443 yield x |
80e51429cb9a
introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents:
6839
diff
changeset
|
444 |
6903
0642d9d7ec80
clone: get a list of files to clone from store
Matt Mackall <mpm@selenic.com>
parents:
6902
diff
changeset
|
445 def copylist(self): |
0642d9d7ec80
clone: get a list of files to clone from store
Matt Mackall <mpm@selenic.com>
parents:
6902
diff
changeset
|
446 return ['requires'] + _data.split() |
0642d9d7ec80
clone: get a list of files to clone from store
Matt Mackall <mpm@selenic.com>
parents:
6902
diff
changeset
|
447 |
20883
cd443c7589cc
fncache: move fncache writing to be in a transaction
Durham Goode <durham@fb.com>
parents:
20879
diff
changeset
|
448 def write(self, tr): |
13391
d00bbff8600e
fncachestore: defer updating the fncache file to a single file open
Adrian Buehlmann <adrian@cadifra.com>
parents:
13169
diff
changeset
|
449 pass |
d00bbff8600e
fncachestore: defer updating the fncache file to a single file open
Adrian Buehlmann <adrian@cadifra.com>
parents:
13169
diff
changeset
|
450 |
20884
2efdd186925d
caches: invalidate store caches when lock is taken
Durham Goode <durham@fb.com>
parents:
20883
diff
changeset
|
451 def invalidatecaches(self): |
2efdd186925d
caches: invalidate store caches when lock is taken
Durham Goode <durham@fb.com>
parents:
20883
diff
changeset
|
452 pass |
2efdd186925d
caches: invalidate store caches when lock is taken
Durham Goode <durham@fb.com>
parents:
20883
diff
changeset
|
453 |
20885
f49d60fa40a5
fncache: clean up fncache during strips
Durham Goode <durham@fb.com>
parents:
20884
diff
changeset
|
454 def markremoved(self, fn): |
f49d60fa40a5
fncache: clean up fncache during strips
Durham Goode <durham@fb.com>
parents:
20884
diff
changeset
|
455 pass |
f49d60fa40a5
fncache: clean up fncache during strips
Durham Goode <durham@fb.com>
parents:
20884
diff
changeset
|
456 |
17744 | 457 def __contains__(self, path): |
458 '''Checks if the store contains path''' | |
459 path = "/".join(("data", path)) | |
460 # file? | |
19903
ca875b271ac3
store: use "vfs.exists()" instead of "os.path.exists()"
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
19177
diff
changeset
|
461 if self.vfs.exists(path + ".i"): |
17744 | 462 return True |
463 # dir? | |
464 if not path.endswith("/"): | |
465 path = path + "/" | |
19903
ca875b271ac3
store: use "vfs.exists()" instead of "os.path.exists()"
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
19177
diff
changeset
|
466 return self.vfs.exists(path) |
17744 | 467 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
468 |
6898
69aeaaaf6e07
store: simplify class hierarchy
Matt Mackall <mpm@selenic.com>
parents:
6897
diff
changeset
|
469 class encodedstore(basicstore): |
17651
3b49c28658f6
store: rename "openertype" argument to "vfstype"
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17649
diff
changeset
|
470 def __init__(self, path, vfstype): |
17724
bf4b72d8dd4d
store: initialize vfs field first to use it for initialization of others
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17722
diff
changeset
|
471 vfs = vfstype(path + '/store') |
bf4b72d8dd4d
store: initialize vfs field first to use it for initialization of others
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17722
diff
changeset
|
472 self.path = vfs.base |
17726
7cb7e17c23b2
store: invoke "os.stat()" for "createmode" initialization via vfs
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17725
diff
changeset
|
473 self.createmode = _calcmode(vfs) |
17652
2c6f7231becc
store: rename "op" variables to "vfs"
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17651
diff
changeset
|
474 vfs.createmode = self.createmode |
17728
004bd533880d
store: invoke "os.path.isdir()" via vfs
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17727
diff
changeset
|
475 self.rawvfs = vfs |
31234
9b7a2ef4f27c
vfs: use 'vfs' module directly in 'mercurial.store'
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents:
31219
diff
changeset
|
476 self.vfs = vfsmod.filtervfs(vfs, encodefilename) |
17653
dacb50696b75
store: initialize "vfs" fields by "vfs" constructors
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17652
diff
changeset
|
477 self.opener = self.vfs |
6840
80e51429cb9a
introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents:
6839
diff
changeset
|
478 |
40340
2d45b549392f
store: pass matcher to store.datafiles()
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
39698
diff
changeset
|
479 def datafiles(self, matcher=None): |
28007
fb92927f9775
treemanifests: fix streaming clone
Martin von Zweigbergk <martinvonz@google.com>
parents:
28006
diff
changeset
|
480 for a, b, size in super(encodedstore, self).datafiles(): |
6892
dab95717058d
verify: check repo.store
Adrian Buehlmann <adrian@cadifra.com>
parents:
6890
diff
changeset
|
481 try: |
6900
def492d1b592
store: change handling of decoding errors
Matt Mackall <mpm@selenic.com>
parents:
6899
diff
changeset
|
482 a = decodefilename(a) |
6892
dab95717058d
verify: check repo.store
Adrian Buehlmann <adrian@cadifra.com>
parents:
6890
diff
changeset
|
483 except KeyError: |
6900
def492d1b592
store: change handling of decoding errors
Matt Mackall <mpm@selenic.com>
parents:
6899
diff
changeset
|
484 a = None |
40584
a694a7159125
store: pass in decoded filename to narrow matcher
Yuya Nishihara <yuya@tcha.org>
parents:
40494
diff
changeset
|
485 if a is not None and not _matchtrackedpath(a, matcher): |
a694a7159125
store: pass in decoded filename to narrow matcher
Yuya Nishihara <yuya@tcha.org>
parents:
40494
diff
changeset
|
486 continue |
6900
def492d1b592
store: change handling of decoding errors
Matt Mackall <mpm@selenic.com>
parents:
6899
diff
changeset
|
487 yield a, b, size |
6840
80e51429cb9a
introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents:
6839
diff
changeset
|
488 |
80e51429cb9a
introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents:
6839
diff
changeset
|
489 def join(self, f): |
13426
643b8212813e
store: remove pointless pathjoiner parameter
Adrian Buehlmann <adrian@cadifra.com>
parents:
13391
diff
changeset
|
490 return self.path + '/' + encodefilename(f) |
6840
80e51429cb9a
introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents:
6839
diff
changeset
|
491 |
6903
0642d9d7ec80
clone: get a list of files to clone from store
Matt Mackall <mpm@selenic.com>
parents:
6902
diff
changeset
|
492 def copylist(self): |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
493 return ['requires', '00changelog.i'] + [ |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
494 'store/' + f for f in _data.split() |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
495 ] |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
496 |
6903
0642d9d7ec80
clone: get a list of files to clone from store
Matt Mackall <mpm@selenic.com>
parents:
6902
diff
changeset
|
497 |
8530
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
498 class fncache(object): |
8531
810387f59696
filelog encoding: move the encoding/decoding into store
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8530
diff
changeset
|
499 # the filename used to be partially encoded |
810387f59696
filelog encoding: move the encoding/decoding into store
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8530
diff
changeset
|
500 # hence the encodedir/decodedir dance |
17722
3b976051034d
store: rename field name from "opener" to "vfs" in internal classes for fncache
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17721
diff
changeset
|
501 def __init__(self, vfs): |
3b976051034d
store: rename field name from "opener" to "vfs" in internal classes for fncache
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17721
diff
changeset
|
502 self.vfs = vfs |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
503 self.entries = None |
13391
d00bbff8600e
fncachestore: defer updating the fncache file to a single file open
Adrian Buehlmann <adrian@cadifra.com>
parents:
13169
diff
changeset
|
504 self._dirty = False |
40736
0728d87a8631
store: append to fncache if there are only new files to write
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40624
diff
changeset
|
505 # set of new additions to fncache |
0728d87a8631
store: append to fncache if there are only new files to write
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40624
diff
changeset
|
506 self.addls = set() |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
507 |
42715
f59f8a5e9096
fncache: make debugrebuildfncache not fail on broken fncache
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents:
42341
diff
changeset
|
508 def ensureloaded(self, warn=None): |
f59f8a5e9096
fncache: make debugrebuildfncache not fail on broken fncache
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents:
42341
diff
changeset
|
509 '''read the fncache file if not already read. |
f59f8a5e9096
fncache: make debugrebuildfncache not fail on broken fncache
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents:
42341
diff
changeset
|
510 |
f59f8a5e9096
fncache: make debugrebuildfncache not fail on broken fncache
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents:
42341
diff
changeset
|
511 If the file on disk is corrupted, raise. If warn is provided, |
f59f8a5e9096
fncache: make debugrebuildfncache not fail on broken fncache
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents:
42341
diff
changeset
|
512 warn and keep going instead.''' |
f59f8a5e9096
fncache: make debugrebuildfncache not fail on broken fncache
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents:
42341
diff
changeset
|
513 if self.entries is None: |
f59f8a5e9096
fncache: make debugrebuildfncache not fail on broken fncache
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents:
42341
diff
changeset
|
514 self._load(warn) |
f59f8a5e9096
fncache: make debugrebuildfncache not fail on broken fncache
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents:
42341
diff
changeset
|
515 |
f59f8a5e9096
fncache: make debugrebuildfncache not fail on broken fncache
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents:
42341
diff
changeset
|
516 def _load(self, warn=None): |
8530
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
517 '''fill the entries from the fncache file''' |
13391
d00bbff8600e
fncachestore: defer updating the fncache file to a single file open
Adrian Buehlmann <adrian@cadifra.com>
parents:
13169
diff
changeset
|
518 self._dirty = False |
8530
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
519 try: |
17722
3b976051034d
store: rename field name from "opener" to "vfs" in internal classes for fncache
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17721
diff
changeset
|
520 fp = self.vfs('fncache', mode='rb') |
8530
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
521 except IOError: |
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
522 # skip nonexistent file |
16404
9fca5b056c0a
store: speed up read and write of large fncache files
Bryan O'Sullivan <bryano@fb.com>
parents:
15742
diff
changeset
|
523 self.entries = set() |
8530
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
524 return |
41978
a56487081109
store: don't read the whole fncache in memory
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
41973
diff
changeset
|
525 |
a56487081109
store: don't read the whole fncache in memory
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
41973
diff
changeset
|
526 self.entries = set() |
a56487081109
store: don't read the whole fncache in memory
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
41973
diff
changeset
|
527 chunk = b'' |
a56487081109
store: don't read the whole fncache in memory
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
41973
diff
changeset
|
528 for c in iter(functools.partial(fp.read, fncache_chunksize), b''): |
a56487081109
store: don't read the whole fncache in memory
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
41973
diff
changeset
|
529 chunk += c |
a56487081109
store: don't read the whole fncache in memory
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
41973
diff
changeset
|
530 try: |
a56487081109
store: don't read the whole fncache in memory
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
41973
diff
changeset
|
531 p = chunk.rindex(b'\n') |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
532 self.entries.update(decodedir(chunk[: p + 1]).splitlines()) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
533 chunk = chunk[p + 1 :] |
41978
a56487081109
store: don't read the whole fncache in memory
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
41973
diff
changeset
|
534 except ValueError: |
a56487081109
store: don't read the whole fncache in memory
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
41973
diff
changeset
|
535 # substring '\n' not found, maybe the entry is bigger than the |
a56487081109
store: don't read the whole fncache in memory
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
41973
diff
changeset
|
536 # chunksize, so let's keep iterating |
a56487081109
store: don't read the whole fncache in memory
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
41973
diff
changeset
|
537 pass |
a56487081109
store: don't read the whole fncache in memory
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
41973
diff
changeset
|
538 |
41981
a920a9e1795a
store: error out if fncache does not ends with a newline
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
41978
diff
changeset
|
539 if chunk: |
42715
f59f8a5e9096
fncache: make debugrebuildfncache not fail on broken fncache
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents:
42341
diff
changeset
|
540 msg = _("fncache does not ends with a newline") |
f59f8a5e9096
fncache: make debugrebuildfncache not fail on broken fncache
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents:
42341
diff
changeset
|
541 if warn: |
f59f8a5e9096
fncache: make debugrebuildfncache not fail on broken fncache
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents:
42341
diff
changeset
|
542 warn(msg + '\n') |
f59f8a5e9096
fncache: make debugrebuildfncache not fail on broken fncache
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents:
42341
diff
changeset
|
543 else: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
544 raise error.Abort( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
545 msg, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
546 hint=_( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
547 "use 'hg debugrebuildfncache' to " "rebuild the fncache" |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
548 ), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
549 ) |
42715
f59f8a5e9096
fncache: make debugrebuildfncache not fail on broken fncache
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents:
42341
diff
changeset
|
550 self._checkentries(fp, warn) |
41973
d7ef84e595f8
store: move logic to check for invalid entry in fncache to own function
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
41093
diff
changeset
|
551 fp.close() |
d7ef84e595f8
store: move logic to check for invalid entry in fncache to own function
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
41093
diff
changeset
|
552 |
42715
f59f8a5e9096
fncache: make debugrebuildfncache not fail on broken fncache
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents:
42341
diff
changeset
|
553 def _checkentries(self, fp, warn): |
41973
d7ef84e595f8
store: move logic to check for invalid entry in fncache to own function
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
41093
diff
changeset
|
554 """ make sure there is no empty string in entries """ |
16404
9fca5b056c0a
store: speed up read and write of large fncache files
Bryan O'Sullivan <bryano@fb.com>
parents:
15742
diff
changeset
|
555 if '' in self.entries: |
9fca5b056c0a
store: speed up read and write of large fncache files
Bryan O'Sullivan <bryano@fb.com>
parents:
15742
diff
changeset
|
556 fp.seek(0) |
30398 | 557 for n, line in enumerate(util.iterfile(fp)): |
16404
9fca5b056c0a
store: speed up read and write of large fncache files
Bryan O'Sullivan <bryano@fb.com>
parents:
15742
diff
changeset
|
558 if not line.rstrip('\n'): |
26778
a95c975f42e3
l10n: use %d instead of %s for numbers
timeless@mozdev.org
parents:
26587
diff
changeset
|
559 t = _('invalid entry in fncache, line %d') % (n + 1) |
42715
f59f8a5e9096
fncache: make debugrebuildfncache not fail on broken fncache
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents:
42341
diff
changeset
|
560 if warn: |
f59f8a5e9096
fncache: make debugrebuildfncache not fail on broken fncache
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents:
42341
diff
changeset
|
561 warn(t + '\n') |
f59f8a5e9096
fncache: make debugrebuildfncache not fail on broken fncache
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents:
42341
diff
changeset
|
562 else: |
f59f8a5e9096
fncache: make debugrebuildfncache not fail on broken fncache
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents:
42341
diff
changeset
|
563 raise error.Abort(t) |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
564 |
20883
cd443c7589cc
fncache: move fncache writing to be in a transaction
Durham Goode <durham@fb.com>
parents:
20879
diff
changeset
|
565 def write(self, tr): |
16404
9fca5b056c0a
store: speed up read and write of large fncache files
Bryan O'Sullivan <bryano@fb.com>
parents:
15742
diff
changeset
|
566 if self._dirty: |
38696
89d93dd1a222
store: assert the fncache have been loaded if dirty
Boris Feld <boris.feld@octobus.net>
parents:
38661
diff
changeset
|
567 assert self.entries is not None |
40748
df8ed31a8ad8
store: write fncache only once if there are both adds and removes
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40736
diff
changeset
|
568 self.entries = self.entries | self.addls |
df8ed31a8ad8
store: write fncache only once if there are both adds and removes
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40736
diff
changeset
|
569 self.addls = set() |
20883
cd443c7589cc
fncache: move fncache writing to be in a transaction
Durham Goode <durham@fb.com>
parents:
20879
diff
changeset
|
570 tr.addbackup('fncache') |
20879
cd03854a2e06
fncache: remove the rewriting logic
Durham Goode <durham@fb.com>
parents:
19903
diff
changeset
|
571 fp = self.vfs('fncache', mode='wb', atomictemp=True) |
cd03854a2e06
fncache: remove the rewriting logic
Durham Goode <durham@fb.com>
parents:
19903
diff
changeset
|
572 if self.entries: |
cd03854a2e06
fncache: remove the rewriting logic
Durham Goode <durham@fb.com>
parents:
19903
diff
changeset
|
573 fp.write(encodedir('\n'.join(self.entries) + '\n')) |
cd03854a2e06
fncache: remove the rewriting logic
Durham Goode <durham@fb.com>
parents:
19903
diff
changeset
|
574 fp.close() |
cd03854a2e06
fncache: remove the rewriting logic
Durham Goode <durham@fb.com>
parents:
19903
diff
changeset
|
575 self._dirty = False |
40736
0728d87a8631
store: append to fncache if there are only new files to write
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40624
diff
changeset
|
576 if self.addls: |
0728d87a8631
store: append to fncache if there are only new files to write
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40624
diff
changeset
|
577 # if we have just new entries, let's append them to the fncache |
0728d87a8631
store: append to fncache if there are only new files to write
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40624
diff
changeset
|
578 tr.addbackup('fncache') |
0728d87a8631
store: append to fncache if there are only new files to write
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40624
diff
changeset
|
579 fp = self.vfs('fncache', mode='ab', atomictemp=True) |
0728d87a8631
store: append to fncache if there are only new files to write
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40624
diff
changeset
|
580 if self.addls: |
0728d87a8631
store: append to fncache if there are only new files to write
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40624
diff
changeset
|
581 fp.write(encodedir('\n'.join(self.addls) + '\n')) |
0728d87a8631
store: append to fncache if there are only new files to write
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40624
diff
changeset
|
582 fp.close() |
0728d87a8631
store: append to fncache if there are only new files to write
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40624
diff
changeset
|
583 self.entries = None |
0728d87a8631
store: append to fncache if there are only new files to write
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40624
diff
changeset
|
584 self.addls = set() |
8530
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
585 |
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
586 def add(self, fn): |
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
587 if self.entries is None: |
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
588 self._load() |
10577
d5bd1beff794
store: only add new entries to the fncache file
Adrian Buehlmann <adrian@cadifra.com>
parents:
10339
diff
changeset
|
589 if fn not in self.entries: |
40736
0728d87a8631
store: append to fncache if there are only new files to write
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40624
diff
changeset
|
590 self.addls.add(fn) |
8530
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
591 |
20885
f49d60fa40a5
fncache: clean up fncache during strips
Durham Goode <durham@fb.com>
parents:
20884
diff
changeset
|
592 def remove(self, fn): |
f49d60fa40a5
fncache: clean up fncache during strips
Durham Goode <durham@fb.com>
parents:
20884
diff
changeset
|
593 if self.entries is None: |
f49d60fa40a5
fncache: clean up fncache during strips
Durham Goode <durham@fb.com>
parents:
20884
diff
changeset
|
594 self._load() |
40736
0728d87a8631
store: append to fncache if there are only new files to write
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40624
diff
changeset
|
595 if fn in self.addls: |
0728d87a8631
store: append to fncache if there are only new files to write
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40624
diff
changeset
|
596 self.addls.remove(fn) |
0728d87a8631
store: append to fncache if there are only new files to write
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40624
diff
changeset
|
597 return |
20885
f49d60fa40a5
fncache: clean up fncache during strips
Durham Goode <durham@fb.com>
parents:
20884
diff
changeset
|
598 try: |
f49d60fa40a5
fncache: clean up fncache during strips
Durham Goode <durham@fb.com>
parents:
20884
diff
changeset
|
599 self.entries.remove(fn) |
f49d60fa40a5
fncache: clean up fncache during strips
Durham Goode <durham@fb.com>
parents:
20884
diff
changeset
|
600 self._dirty = True |
f49d60fa40a5
fncache: clean up fncache during strips
Durham Goode <durham@fb.com>
parents:
20884
diff
changeset
|
601 except KeyError: |
f49d60fa40a5
fncache: clean up fncache during strips
Durham Goode <durham@fb.com>
parents:
20884
diff
changeset
|
602 pass |
f49d60fa40a5
fncache: clean up fncache during strips
Durham Goode <durham@fb.com>
parents:
20884
diff
changeset
|
603 |
17782
8095306c1fb2
store: move __contains__() implementation from class fncache into fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents:
17747
diff
changeset
|
604 def __contains__(self, fn): |
40736
0728d87a8631
store: append to fncache if there are only new files to write
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40624
diff
changeset
|
605 if fn in self.addls: |
0728d87a8631
store: append to fncache if there are only new files to write
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40624
diff
changeset
|
606 return True |
8530
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
607 if self.entries is None: |
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
608 self._load() |
17782
8095306c1fb2
store: move __contains__() implementation from class fncache into fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents:
17747
diff
changeset
|
609 return fn in self.entries |
8530
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
610 |
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
611 def __iter__(self): |
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
612 if self.entries is None: |
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
613 self._load() |
40736
0728d87a8631
store: append to fncache if there are only new files to write
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40624
diff
changeset
|
614 return iter(self.entries | self.addls) |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
615 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
616 |
41093
6498f0e03526
vfs: fix proxyvfs inheritance
Boris Feld <boris.feld@octobus.net>
parents:
40748
diff
changeset
|
617 class _fncachevfs(vfsmod.proxyvfs): |
17721
cf236e3501c3
store: rename argument name from "op"(ener) to "vfs"
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17693
diff
changeset
|
618 def __init__(self, vfs, fnc, encode): |
33412
a42369e04aee
vfs: rename auditvfs to proxyvfs
Yuya Nishihara <yuya@tcha.org>
parents:
32372
diff
changeset
|
619 vfsmod.proxyvfs.__init__(self, vfs) |
14194
3a90fb3addc1
store: break up reference cycle introduced in 9cbff8a39a2a
Adrian Buehlmann <adrian@cadifra.com>
parents:
14166
diff
changeset
|
620 self.fncache = fnc |
3a90fb3addc1
store: break up reference cycle introduced in 9cbff8a39a2a
Adrian Buehlmann <adrian@cadifra.com>
parents:
14166
diff
changeset
|
621 self.encode = encode |
3a90fb3addc1
store: break up reference cycle introduced in 9cbff8a39a2a
Adrian Buehlmann <adrian@cadifra.com>
parents:
14166
diff
changeset
|
622 |
3a90fb3addc1
store: break up reference cycle introduced in 9cbff8a39a2a
Adrian Buehlmann <adrian@cadifra.com>
parents:
14166
diff
changeset
|
623 def __call__(self, path, mode='r', *args, **kw): |
38661
8ac0c9cd4c48
fncache: avoid loading the filename cache when not actually modifying it
Martijn Pieters <mj@zopatista.com>
parents:
37409
diff
changeset
|
624 encoded = self.encode(path) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
625 if mode not in ('r', 'rb') and ( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
626 path.startswith('data/') or path.startswith('meta/') |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
627 ): |
38661
8ac0c9cd4c48
fncache: avoid loading the filename cache when not actually modifying it
Martijn Pieters <mj@zopatista.com>
parents:
37409
diff
changeset
|
628 # do not trigger a fncache load when adding a file that already is |
8ac0c9cd4c48
fncache: avoid loading the filename cache when not actually modifying it
Martijn Pieters <mj@zopatista.com>
parents:
37409
diff
changeset
|
629 # known to exist. |
8ac0c9cd4c48
fncache: avoid loading the filename cache when not actually modifying it
Martijn Pieters <mj@zopatista.com>
parents:
37409
diff
changeset
|
630 notload = self.fncache.entries is None and self.vfs.exists(encoded) |
8ac0c9cd4c48
fncache: avoid loading the filename cache when not actually modifying it
Martijn Pieters <mj@zopatista.com>
parents:
37409
diff
changeset
|
631 if notload and 'a' in mode and not self.vfs.stat(encoded).st_size: |
8ac0c9cd4c48
fncache: avoid loading the filename cache when not actually modifying it
Martijn Pieters <mj@zopatista.com>
parents:
37409
diff
changeset
|
632 # when appending to an existing file, if the file has size zero, |
8ac0c9cd4c48
fncache: avoid loading the filename cache when not actually modifying it
Martijn Pieters <mj@zopatista.com>
parents:
37409
diff
changeset
|
633 # it should be considered as missing. Such zero-size files are |
8ac0c9cd4c48
fncache: avoid loading the filename cache when not actually modifying it
Martijn Pieters <mj@zopatista.com>
parents:
37409
diff
changeset
|
634 # the result of truncation when a transaction is aborted. |
8ac0c9cd4c48
fncache: avoid loading the filename cache when not actually modifying it
Martijn Pieters <mj@zopatista.com>
parents:
37409
diff
changeset
|
635 notload = False |
8ac0c9cd4c48
fncache: avoid loading the filename cache when not actually modifying it
Martijn Pieters <mj@zopatista.com>
parents:
37409
diff
changeset
|
636 if not notload: |
8ac0c9cd4c48
fncache: avoid loading the filename cache when not actually modifying it
Martijn Pieters <mj@zopatista.com>
parents:
37409
diff
changeset
|
637 self.fncache.add(path) |
8ac0c9cd4c48
fncache: avoid loading the filename cache when not actually modifying it
Martijn Pieters <mj@zopatista.com>
parents:
37409
diff
changeset
|
638 return self.vfs(encoded, mode, *args, **kw) |
14194
3a90fb3addc1
store: break up reference cycle introduced in 9cbff8a39a2a
Adrian Buehlmann <adrian@cadifra.com>
parents:
14166
diff
changeset
|
639 |
17725
ffd589d4b785
vfs: define "join()" in each classes derived from "abstractvfs"
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17724
diff
changeset
|
640 def join(self, path): |
ffd589d4b785
vfs: define "join()" in each classes derived from "abstractvfs"
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17724
diff
changeset
|
641 if path: |
ffd589d4b785
vfs: define "join()" in each classes derived from "abstractvfs"
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17724
diff
changeset
|
642 return self.vfs.join(self.encode(path)) |
ffd589d4b785
vfs: define "join()" in each classes derived from "abstractvfs"
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17724
diff
changeset
|
643 else: |
ffd589d4b785
vfs: define "join()" in each classes derived from "abstractvfs"
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17724
diff
changeset
|
644 return self.vfs.join(path) |
ffd589d4b785
vfs: define "join()" in each classes derived from "abstractvfs"
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17724
diff
changeset
|
645 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
646 |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
647 class fncachestore(basicstore): |
17651
3b49c28658f6
store: rename "openertype" argument to "vfstype"
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17649
diff
changeset
|
648 def __init__(self, path, vfstype, dotencode): |
17591
9a5c2ecd1158
store: move encode lambda logic into fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents:
17590
diff
changeset
|
649 if dotencode: |
18435
8c019d2fd7c0
store: switch to C-based hashed path encoding
Bryan O'Sullivan <bryano@fb.com>
parents:
18430
diff
changeset
|
650 encode = _pathencode |
17591
9a5c2ecd1158
store: move encode lambda logic into fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents:
17590
diff
changeset
|
651 else: |
9a5c2ecd1158
store: move encode lambda logic into fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents:
17590
diff
changeset
|
652 encode = _plainhybridencode |
12687
34d8247a4595
store: encode first period or space in filenames (issue1713)
Adrian Buehlmann <adrian@cadifra.com>
parents:
12171
diff
changeset
|
653 self.encode = encode |
17724
bf4b72d8dd4d
store: initialize vfs field first to use it for initialization of others
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17722
diff
changeset
|
654 vfs = vfstype(path + '/store') |
bf4b72d8dd4d
store: initialize vfs field first to use it for initialization of others
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17722
diff
changeset
|
655 self.path = vfs.base |
17562
b42b0729744d
store: reduce string concatenation when joining
Bryan O'Sullivan <bryano@fb.com>
parents:
17555
diff
changeset
|
656 self.pathsep = self.path + '/' |
17726
7cb7e17c23b2
store: invoke "os.stat()" for "createmode" initialization via vfs
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17725
diff
changeset
|
657 self.createmode = _calcmode(vfs) |
17652
2c6f7231becc
store: rename "op" variables to "vfs"
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17651
diff
changeset
|
658 vfs.createmode = self.createmode |
17727
6492b39a44d5
store: replace invocation of "getsize()" by "vfs.stat()"
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17726
diff
changeset
|
659 self.rawvfs = vfs |
17652
2c6f7231becc
store: rename "op" variables to "vfs"
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17651
diff
changeset
|
660 fnc = fncache(vfs) |
9133
996c1cd8f530
store: eliminate reference cycle in fncachestore
Simon Heimberg <simohe@besonet.ch>
parents:
8778
diff
changeset
|
661 self.fncache = fnc |
17653
dacb50696b75
store: initialize "vfs" fields by "vfs" constructors
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17652
diff
changeset
|
662 self.vfs = _fncachevfs(vfs, fnc, encode) |
dacb50696b75
store: initialize "vfs" fields by "vfs" constructors
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17652
diff
changeset
|
663 self.opener = self.vfs |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
664 |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
665 def join(self, f): |
17562
b42b0729744d
store: reduce string concatenation when joining
Bryan O'Sullivan <bryano@fb.com>
parents:
17555
diff
changeset
|
666 return self.pathsep + self.encode(f) |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
667 |
17731
c85dbae29684
store: restore getsize method
Matt Mackall <mpm@selenic.com>
parents:
17728
diff
changeset
|
668 def getsize(self, path): |
c85dbae29684
store: restore getsize method
Matt Mackall <mpm@selenic.com>
parents:
17728
diff
changeset
|
669 return self.rawvfs.stat(path).st_size |
c85dbae29684
store: restore getsize method
Matt Mackall <mpm@selenic.com>
parents:
17728
diff
changeset
|
670 |
40340
2d45b549392f
store: pass matcher to store.datafiles()
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
39698
diff
changeset
|
671 def datafiles(self, matcher=None): |
17373
4cbb1137941d
store: sort the results of fncachestore.datafiles()
Bryan O'Sullivan <bryano@fb.com>
parents:
17249
diff
changeset
|
672 for f in sorted(self.fncache): |
40494
9aeb9e2d28a7
store: introduce _matchtrackedpath() and use it to filter store files
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40340
diff
changeset
|
673 if not _matchtrackedpath(f, matcher): |
9aeb9e2d28a7
store: introduce _matchtrackedpath() and use it to filter store files
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40340
diff
changeset
|
674 continue |
12687
34d8247a4595
store: encode first period or space in filenames (issue1713)
Adrian Buehlmann <adrian@cadifra.com>
parents:
12171
diff
changeset
|
675 ef = self.encode(f) |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
676 try: |
17731
c85dbae29684
store: restore getsize method
Matt Mackall <mpm@selenic.com>
parents:
17728
diff
changeset
|
677 yield f, ef, self.getsize(ef) |
25660
328739ea70c3
global: mass rewrite to use modern exception syntax
Gregory Szorc <gregory.szorc@gmail.com>
parents:
25658
diff
changeset
|
678 except OSError as err: |
17374
0cec762790ed
store: only one kind of OSError means "nonexistent entry"
Bryan O'Sullivan <bryano@fb.com>
parents:
17373
diff
changeset
|
679 if err.errno != errno.ENOENT: |
0cec762790ed
store: only one kind of OSError means "nonexistent entry"
Bryan O'Sullivan <bryano@fb.com>
parents:
17373
diff
changeset
|
680 raise |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
681 |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
682 def copylist(self): |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
683 d = ( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
684 'bookmarks narrowspec data meta dh fncache phaseroots obsstore' |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
685 ' 00manifest.d 00manifest.i 00changelog.d 00changelog.i' |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
686 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
687 return ['requires', '00changelog.i'] + ['store/' + f for f in d.split()] |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
688 |
20883
cd443c7589cc
fncache: move fncache writing to be in a transaction
Durham Goode <durham@fb.com>
parents:
20879
diff
changeset
|
689 def write(self, tr): |
cd443c7589cc
fncache: move fncache writing to be in a transaction
Durham Goode <durham@fb.com>
parents:
20879
diff
changeset
|
690 self.fncache.write(tr) |
13391
d00bbff8600e
fncachestore: defer updating the fncache file to a single file open
Adrian Buehlmann <adrian@cadifra.com>
parents:
13169
diff
changeset
|
691 |
20884
2efdd186925d
caches: invalidate store caches when lock is taken
Durham Goode <durham@fb.com>
parents:
20883
diff
changeset
|
692 def invalidatecaches(self): |
2efdd186925d
caches: invalidate store caches when lock is taken
Durham Goode <durham@fb.com>
parents:
20883
diff
changeset
|
693 self.fncache.entries = None |
40736
0728d87a8631
store: append to fncache if there are only new files to write
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40624
diff
changeset
|
694 self.fncache.addls = set() |
20884
2efdd186925d
caches: invalidate store caches when lock is taken
Durham Goode <durham@fb.com>
parents:
20883
diff
changeset
|
695 |
20885
f49d60fa40a5
fncache: clean up fncache during strips
Durham Goode <durham@fb.com>
parents:
20884
diff
changeset
|
696 def markremoved(self, fn): |
f49d60fa40a5
fncache: clean up fncache during strips
Durham Goode <durham@fb.com>
parents:
20884
diff
changeset
|
697 self.fncache.remove(fn) |
f49d60fa40a5
fncache: clean up fncache during strips
Durham Goode <durham@fb.com>
parents:
20884
diff
changeset
|
698 |
17783
df55ce6854c3
store: add new _exists helper function on fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents:
17782
diff
changeset
|
699 def _exists(self, f): |
df55ce6854c3
store: add new _exists helper function on fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents:
17782
diff
changeset
|
700 ef = self.encode(f) |
df55ce6854c3
store: add new _exists helper function on fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents:
17782
diff
changeset
|
701 try: |
df55ce6854c3
store: add new _exists helper function on fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents:
17782
diff
changeset
|
702 self.getsize(ef) |
df55ce6854c3
store: add new _exists helper function on fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents:
17782
diff
changeset
|
703 return True |
25660
328739ea70c3
global: mass rewrite to use modern exception syntax
Gregory Szorc <gregory.szorc@gmail.com>
parents:
25658
diff
changeset
|
704 except OSError as err: |
17783
df55ce6854c3
store: add new _exists helper function on fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents:
17782
diff
changeset
|
705 if err.errno != errno.ENOENT: |
df55ce6854c3
store: add new _exists helper function on fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents:
17782
diff
changeset
|
706 raise |
df55ce6854c3
store: add new _exists helper function on fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents:
17782
diff
changeset
|
707 # nonexistent entry |
df55ce6854c3
store: add new _exists helper function on fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents:
17782
diff
changeset
|
708 return False |
df55ce6854c3
store: add new _exists helper function on fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents:
17782
diff
changeset
|
709 |
17745 | 710 def __contains__(self, path): |
711 '''Checks if the store contains path''' | |
712 path = "/".join(("data", path)) | |
17782
8095306c1fb2
store: move __contains__() implementation from class fncache into fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents:
17747
diff
changeset
|
713 # check for files (exact match) |
17784
73e1ab39792c
store: fncache may contain non-existent entries (fixes b9a56b816ff2)
Adrian Buehlmann <adrian@cadifra.com>
parents:
17783
diff
changeset
|
714 e = path + '.i' |
73e1ab39792c
store: fncache may contain non-existent entries (fixes b9a56b816ff2)
Adrian Buehlmann <adrian@cadifra.com>
parents:
17783
diff
changeset
|
715 if e in self.fncache and self._exists(e): |
17782
8095306c1fb2
store: move __contains__() implementation from class fncache into fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents:
17747
diff
changeset
|
716 return True |
8095306c1fb2
store: move __contains__() implementation from class fncache into fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents:
17747
diff
changeset
|
717 # now check for directories (prefix match) |
8095306c1fb2
store: move __contains__() implementation from class fncache into fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents:
17747
diff
changeset
|
718 if not path.endswith('/'): |
8095306c1fb2
store: move __contains__() implementation from class fncache into fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents:
17747
diff
changeset
|
719 path += '/' |
8095306c1fb2
store: move __contains__() implementation from class fncache into fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents:
17747
diff
changeset
|
720 for e in self.fncache: |
17784
73e1ab39792c
store: fncache may contain non-existent entries (fixes b9a56b816ff2)
Adrian Buehlmann <adrian@cadifra.com>
parents:
17783
diff
changeset
|
721 if e.startswith(path) and self._exists(e): |
17782
8095306c1fb2
store: move __contains__() implementation from class fncache into fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents:
17747
diff
changeset
|
722 return True |
8095306c1fb2
store: move __contains__() implementation from class fncache into fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents:
17747
diff
changeset
|
723 return False |