Mercurial > hg
annotate mercurial/store.py @ 51582:d8287e43540f
phases: stop using `repo.set` in `remotephasessummary`
The `repository.set` create changectx on the fly, an expensive operation. Using
`repo.revs` and a direct rev-num → node-id translation will be significantly
faster.
This is especially true as we prepare ourself to no longer do the rev-num →
node-id transalation there.
The speedup is a bit lost in the overall noisyness of the slow phase discovery algorithm, but it save a small amount of time in my benchmark.
author | Pierre-Yves David <pierre-yves.david@octobus.net> |
---|---|
date | Fri, 05 Apr 2024 12:24:47 +0200 |
parents | 49faa72b994e |
children | 493034cc3265 |
rev | line source |
---|---|
50680
e324329fc97e
store: use the revlog type from revlog's constant instead of custom one
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50672
diff
changeset
|
1 # store.py - repository store handling for Mercurial) |
6839
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
2 # |
46819
d4ba4d51f85f
contributor: change mentions of mpm to olivia
Raphaël Gomès <rgomes@octobus.net>
parents:
46780
diff
changeset
|
3 # Copyright 2008 Olivia Mackall <olivia@selenic.com> |
6839
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
4 # |
8225
46293a0c7e9f
updated license to be explicit about GPL version 2
Martin Geisler <mg@lazybytes.net>
parents:
8210
diff
changeset
|
5 # This software may be used and distributed according to the terms of the |
10263 | 6 # GNU General Public License version 2 or any later version. |
6839
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
7 |
50479
5217e36356bb
store: add logic to group revlog file together
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50478
diff
changeset
|
8 import collections |
41978
a56487081109
store: don't read the whole fncache in memory
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
41973
diff
changeset
|
9 import functools |
27480
509159675cdb
store: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26778
diff
changeset
|
10 import os |
46990
0b569c75d180
store: exclude `undo.` nodemap's file from `walk`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
11 import re |
27480
509159675cdb
store: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26778
diff
changeset
|
12 import stat |
50638
8fc10bfd9887
store: declare a `files` method on BaseStoreEntry
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50632
diff
changeset
|
13 from typing import Generator, List |
27480
509159675cdb
store: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26778
diff
changeset
|
14 |
509159675cdb
store: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26778
diff
changeset
|
15 from .i18n import _ |
50471
521fec115dad
store: use a StoreEntry object instead of tuple for store files
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50469
diff
changeset
|
16 from .thirdparty import attr |
46113
59fa3890d40a
node: import symbols explicitly
Joerg Sonnenberger <joerg@bec.de>
parents:
45942
diff
changeset
|
17 from .node import hex |
50672
3b56395404a1
stream-clone: avoid opening a revlog in case we do not need it
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50671
diff
changeset
|
18 from .revlogutils.constants import ( |
3b56395404a1
stream-clone: avoid opening a revlog in case we do not need it
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50671
diff
changeset
|
19 INDEX_HEADER, |
50680
e324329fc97e
store: use the revlog type from revlog's constant instead of custom one
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50672
diff
changeset
|
20 KIND_CHANGELOG, |
e324329fc97e
store: use the revlog type from revlog's constant instead of custom one
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50672
diff
changeset
|
21 KIND_FILELOG, |
e324329fc97e
store: use the revlog type from revlog's constant instead of custom one
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50672
diff
changeset
|
22 KIND_MANIFESTLOG, |
50672
3b56395404a1
stream-clone: avoid opening a revlog in case we do not need it
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50671
diff
changeset
|
23 ) |
27480
509159675cdb
store: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26778
diff
changeset
|
24 from . import ( |
42905
3df3b139a43d
localrepo: push manifestlog and changelog construction code into store
Augie Fackler <augie@google.com>
parents:
42715
diff
changeset
|
25 changelog, |
27480
509159675cdb
store: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26778
diff
changeset
|
26 error, |
50627
e1ee6910f6bc
store: add a `get_revlog_instance` method on revlog entries
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50514
diff
changeset
|
27 filelog, |
42905
3df3b139a43d
localrepo: push manifestlog and changelog construction code into store
Augie Fackler <augie@google.com>
parents:
42715
diff
changeset
|
28 manifest, |
32372
df448de7cf3b
parsers: switch to policy importer
Yuya Nishihara <yuya@tcha.org>
parents:
31362
diff
changeset
|
29 policy, |
30077
8f42d8c412c8
py3: make encodefun in store.py compatible with py3k
Mateusz Kwapich <mitrandir@fb.com>
parents:
30076
diff
changeset
|
30 pycompat, |
50672
3b56395404a1
stream-clone: avoid opening a revlog in case we do not need it
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50671
diff
changeset
|
31 revlog as revlogmod, |
27480
509159675cdb
store: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26778
diff
changeset
|
32 util, |
31234
9b7a2ef4f27c
vfs: use 'vfs' module directly in 'mercurial.store'
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents:
31219
diff
changeset
|
33 vfs as vfsmod, |
27480
509159675cdb
store: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26778
diff
changeset
|
34 ) |
44060
a61287a95dc3
core: migrate uses of hashlib.sha1 to hashutil.sha1
Augie Fackler <augie@google.com>
parents:
43506
diff
changeset
|
35 from .utils import hashutil |
6840
80e51429cb9a
introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents:
6839
diff
changeset
|
36 |
43506
9f70512ae2cf
cleanup: remove pointless r-prefixes on single-quoted strings
Augie Fackler <augie@google.com>
parents:
43106
diff
changeset
|
37 parsers = policy.importmod('parsers') |
41978
a56487081109
store: don't read the whole fncache in memory
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
41973
diff
changeset
|
38 # how much bytes should be read from fncache in one read |
a56487081109
store: don't read the whole fncache in memory
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
41973
diff
changeset
|
39 # It is done to prevent loading large fncache files into memory |
a56487081109
store: don't read the whole fncache in memory
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
41973
diff
changeset
|
40 fncache_chunksize = 10 ** 6 |
32372
df448de7cf3b
parsers: switch to policy importer
Yuya Nishihara <yuya@tcha.org>
parents:
31362
diff
changeset
|
41 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
42 |
50494
b4953fad744e
store: do the revlog matching on entry directly
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50493
diff
changeset
|
43 def _match_tracked_entry(entry, matcher): |
40494
9aeb9e2d28a7
store: introduce _matchtrackedpath() and use it to filter store files
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40340
diff
changeset
|
44 """parses a fncache entry and returns whether the entry is tracking a path |
9aeb9e2d28a7
store: introduce _matchtrackedpath() and use it to filter store files
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40340
diff
changeset
|
45 matched by matcher or not. |
9aeb9e2d28a7
store: introduce _matchtrackedpath() and use it to filter store files
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40340
diff
changeset
|
46 |
9aeb9e2d28a7
store: introduce _matchtrackedpath() and use it to filter store files
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40340
diff
changeset
|
47 If matcher is None, returns True""" |
9aeb9e2d28a7
store: introduce _matchtrackedpath() and use it to filter store files
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40340
diff
changeset
|
48 |
9aeb9e2d28a7
store: introduce _matchtrackedpath() and use it to filter store files
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40340
diff
changeset
|
49 if matcher is None: |
9aeb9e2d28a7
store: introduce _matchtrackedpath() and use it to filter store files
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40340
diff
changeset
|
50 return True |
50498
1b776f25302f
store: use the boolean property in `store`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50497
diff
changeset
|
51 if entry.is_filelog: |
50494
b4953fad744e
store: do the revlog matching on entry directly
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50493
diff
changeset
|
52 return matcher(entry.target_id) |
50498
1b776f25302f
store: use the boolean property in `store`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50497
diff
changeset
|
53 elif entry.is_manifestlog: |
50494
b4953fad744e
store: do the revlog matching on entry directly
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50493
diff
changeset
|
54 return matcher.visitdir(entry.target_id.rstrip(b'/')) |
b4953fad744e
store: do the revlog matching on entry directly
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50493
diff
changeset
|
55 raise error.ProgrammingError(b"cannot process entry %r" % entry) |
40624
66adfd58cb77
store: raise ProgrammingError if unable to decode a storage path
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40584
diff
changeset
|
56 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
57 |
8531
810387f59696
filelog encoding: move the encoding/decoding into store
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8530
diff
changeset
|
58 # This avoids a collision between a file named foo and a dir named |
810387f59696
filelog encoding: move the encoding/decoding into store
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8530
diff
changeset
|
59 # foo.i or foo.d |
17607
cc58dc47cb5e
store: use fast C implementation of encodedir() if it's available
Adrian Buehlmann <adrian@cadifra.com>
parents:
17605
diff
changeset
|
60 def _encodedir(path): |
45942
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
45483
diff
changeset
|
61 """ |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
33412
diff
changeset
|
62 >>> _encodedir(b'data/foo.i') |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
63 'data/foo.i' |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
33412
diff
changeset
|
64 >>> _encodedir(b'data/foo.i/bla.i') |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
65 'data/foo.i.hg/bla.i' |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
33412
diff
changeset
|
66 >>> _encodedir(b'data/foo.i.hg/bla.i') |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
67 'data/foo.i.hg.hg/bla.i' |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
33412
diff
changeset
|
68 >>> _encodedir(b'data/foo.i\\ndata/foo.i/bla.i\\ndata/foo.i.hg/bla.i\\n') |
17605
e9cc29be3305
store: add multiline doctest case for encodedir()
Adrian Buehlmann <adrian@cadifra.com>
parents:
17604
diff
changeset
|
69 'data/foo.i\\ndata/foo.i.hg/bla.i\\ndata/foo.i.hg.hg/bla.i\\n' |
45942
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
45483
diff
changeset
|
70 """ |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
71 return ( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
72 path.replace(b".hg/", b".hg.hg/") |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
73 .replace(b".i/", b".i.hg/") |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
74 .replace(b".d/", b".d.hg/") |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
75 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
76 |
8531
810387f59696
filelog encoding: move the encoding/decoding into store
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8530
diff
changeset
|
77 |
17607
cc58dc47cb5e
store: use fast C implementation of encodedir() if it's available
Adrian Buehlmann <adrian@cadifra.com>
parents:
17605
diff
changeset
|
78 encodedir = getattr(parsers, 'encodedir', _encodedir) |
cc58dc47cb5e
store: use fast C implementation of encodedir() if it's available
Adrian Buehlmann <adrian@cadifra.com>
parents:
17605
diff
changeset
|
79 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
80 |
8531
810387f59696
filelog encoding: move the encoding/decoding into store
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8530
diff
changeset
|
81 def decodedir(path): |
45942
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
45483
diff
changeset
|
82 """ |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
33412
diff
changeset
|
83 >>> decodedir(b'data/foo.i') |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
84 'data/foo.i' |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
33412
diff
changeset
|
85 >>> decodedir(b'data/foo.i.hg/bla.i') |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
86 'data/foo.i/bla.i' |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
33412
diff
changeset
|
87 >>> decodedir(b'data/foo.i.hg.hg/bla.i') |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
88 'data/foo.i.hg/bla.i' |
45942
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
45483
diff
changeset
|
89 """ |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
90 if b".hg/" not in path: |
8531
810387f59696
filelog encoding: move the encoding/decoding into store
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8530
diff
changeset
|
91 return path |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
92 return ( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
93 path.replace(b".d.hg/", b".d/") |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
94 .replace(b".i.hg/", b".i/") |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
95 .replace(b".hg.hg/", b".hg/") |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
96 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
97 |
8531
810387f59696
filelog encoding: move the encoding/decoding into store
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8530
diff
changeset
|
98 |
29071
2f58975eb2cb
store: treat range as a generator instead of a list for py3 compat
timeless <timeless@mozdev.org>
parents:
28007
diff
changeset
|
99 def _reserved(): |
45942
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
45483
diff
changeset
|
100 """characters that are problematic for filesystems |
29071
2f58975eb2cb
store: treat range as a generator instead of a list for py3 compat
timeless <timeless@mozdev.org>
parents:
28007
diff
changeset
|
101 |
2f58975eb2cb
store: treat range as a generator instead of a list for py3 compat
timeless <timeless@mozdev.org>
parents:
28007
diff
changeset
|
102 * ascii escapes (0..31) |
2f58975eb2cb
store: treat range as a generator instead of a list for py3 compat
timeless <timeless@mozdev.org>
parents:
28007
diff
changeset
|
103 * ascii hi (126..255) |
2f58975eb2cb
store: treat range as a generator instead of a list for py3 compat
timeless <timeless@mozdev.org>
parents:
28007
diff
changeset
|
104 * windows specials |
2f58975eb2cb
store: treat range as a generator instead of a list for py3 compat
timeless <timeless@mozdev.org>
parents:
28007
diff
changeset
|
105 |
2f58975eb2cb
store: treat range as a generator instead of a list for py3 compat
timeless <timeless@mozdev.org>
parents:
28007
diff
changeset
|
106 these characters will be escaped by encodefunctions |
45942
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
45483
diff
changeset
|
107 """ |
30076
400dfded8a29
py3: make the string unicode so its iterable in py3k
Mateusz Kwapich <mitrandir@fb.com>
parents:
29338
diff
changeset
|
108 winreserved = [ord(x) for x in u'\\:*?"<>|'] |
29071
2f58975eb2cb
store: treat range as a generator instead of a list for py3 compat
timeless <timeless@mozdev.org>
parents:
28007
diff
changeset
|
109 for x in range(32): |
2f58975eb2cb
store: treat range as a generator instead of a list for py3 compat
timeless <timeless@mozdev.org>
parents:
28007
diff
changeset
|
110 yield x |
2f58975eb2cb
store: treat range as a generator instead of a list for py3 compat
timeless <timeless@mozdev.org>
parents:
28007
diff
changeset
|
111 for x in range(126, 256): |
2f58975eb2cb
store: treat range as a generator instead of a list for py3 compat
timeless <timeless@mozdev.org>
parents:
28007
diff
changeset
|
112 yield x |
2f58975eb2cb
store: treat range as a generator instead of a list for py3 compat
timeless <timeless@mozdev.org>
parents:
28007
diff
changeset
|
113 for x in winreserved: |
2f58975eb2cb
store: treat range as a generator instead of a list for py3 compat
timeless <timeless@mozdev.org>
parents:
28007
diff
changeset
|
114 yield x |
2f58975eb2cb
store: treat range as a generator instead of a list for py3 compat
timeless <timeless@mozdev.org>
parents:
28007
diff
changeset
|
115 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
116 |
6839
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
117 def _buildencodefun(): |
45942
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
45483
diff
changeset
|
118 """ |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
119 >>> enc, dec = _buildencodefun() |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
120 |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
33412
diff
changeset
|
121 >>> enc(b'nothing/special.txt') |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
122 'nothing/special.txt' |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
33412
diff
changeset
|
123 >>> dec(b'nothing/special.txt') |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
124 'nothing/special.txt' |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
125 |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
33412
diff
changeset
|
126 >>> enc(b'HELLO') |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
127 '_h_e_l_l_o' |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
33412
diff
changeset
|
128 >>> dec(b'_h_e_l_l_o') |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
129 'HELLO' |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
130 |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
33412
diff
changeset
|
131 >>> enc(b'hello:world?') |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
132 'hello~3aworld~3f' |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
33412
diff
changeset
|
133 >>> dec(b'hello~3aworld~3f') |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
134 'hello:world?' |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
135 |
34136
414a3513c2bd
doctest: do not embed non-ascii characters in docstring
Yuya Nishihara <yuya@tcha.org>
parents:
34131
diff
changeset
|
136 >>> enc(b'the\\x07quick\\xADshot') |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
137 'the~07quick~adshot' |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
33412
diff
changeset
|
138 >>> dec(b'the~07quick~adshot') |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
139 'the\\x07quick\\xadshot' |
45942
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
45483
diff
changeset
|
140 """ |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
141 e = b'_' |
31253
64596338ba10
py3: factor out bytechr() function
Yuya Nishihara <yuya@tcha.org>
parents:
31234
diff
changeset
|
142 xchr = pycompat.bytechr |
64596338ba10
py3: factor out bytechr() function
Yuya Nishihara <yuya@tcha.org>
parents:
31234
diff
changeset
|
143 asciistr = list(map(xchr, range(127))) |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
144 capitals = list(range(ord(b"A"), ord(b"Z") + 1)) |
30077
8f42d8c412c8
py3: make encodefun in store.py compatible with py3k
Mateusz Kwapich <mitrandir@fb.com>
parents:
30076
diff
changeset
|
145 |
44452
9d2b2df2c2ba
cleanup: run pyupgrade on our source tree to clean up varying things
Augie Fackler <augie@google.com>
parents:
44060
diff
changeset
|
146 cmap = {x: x for x in asciistr} |
29071
2f58975eb2cb
store: treat range as a generator instead of a list for py3 compat
timeless <timeless@mozdev.org>
parents:
28007
diff
changeset
|
147 for x in _reserved(): |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
148 cmap[xchr(x)] = b"~%02x" % x |
30077
8f42d8c412c8
py3: make encodefun in store.py compatible with py3k
Mateusz Kwapich <mitrandir@fb.com>
parents:
30076
diff
changeset
|
149 for x in capitals + [ord(e)]: |
8f42d8c412c8
py3: make encodefun in store.py compatible with py3k
Mateusz Kwapich <mitrandir@fb.com>
parents:
30076
diff
changeset
|
150 cmap[xchr(x)] = e + xchr(x).lower() |
8f42d8c412c8
py3: make encodefun in store.py compatible with py3k
Mateusz Kwapich <mitrandir@fb.com>
parents:
30076
diff
changeset
|
151 |
6839
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
152 dmap = {} |
48913
f254fc73d956
global: bulk replace simple pycompat.iteritems(x) with x.items()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
48875
diff
changeset
|
153 for k, v in cmap.items(): |
6839
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
154 dmap[v] = k |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
155 |
6839
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
156 def decode(s): |
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
157 i = 0 |
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
158 while i < len(s): |
49284
d44e3c45f0e4
py3: replace `pycompat.xrange` by `range`
Manuel Jacob <me@manueljacob.de>
parents:
48946
diff
changeset
|
159 for l in range(1, 4): |
6839
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
160 try: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
161 yield dmap[s[i : i + l]] |
6839
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
162 i += l |
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
163 break |
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
164 except KeyError: |
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
165 pass |
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
166 else: |
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
167 raise KeyError |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
168 |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
169 return ( |
49284
d44e3c45f0e4
py3: replace `pycompat.xrange` by `range`
Manuel Jacob <me@manueljacob.de>
parents:
48946
diff
changeset
|
170 lambda s: b''.join([cmap[s[c : c + 1]] for c in range(len(s))]), |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
171 lambda s: b''.join(list(decode(s))), |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
172 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
173 |
17608
776240123525
store: extract functions _encodefname and _decodefname
Adrian Buehlmann <adrian@cadifra.com>
parents:
17607
diff
changeset
|
174 |
776240123525
store: extract functions _encodefname and _decodefname
Adrian Buehlmann <adrian@cadifra.com>
parents:
17607
diff
changeset
|
175 _encodefname, _decodefname = _buildencodefun() |
6839
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
176 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
177 |
17608
776240123525
store: extract functions _encodefname and _decodefname
Adrian Buehlmann <adrian@cadifra.com>
parents:
17607
diff
changeset
|
178 def encodefilename(s): |
45942
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
45483
diff
changeset
|
179 """ |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
33412
diff
changeset
|
180 >>> encodefilename(b'foo.i/bar.d/bla.hg/hi:world?/HELLO') |
17608
776240123525
store: extract functions _encodefname and _decodefname
Adrian Buehlmann <adrian@cadifra.com>
parents:
17607
diff
changeset
|
181 'foo.i.hg/bar.d.hg/bla.hg.hg/hi~3aworld~3f/_h_e_l_l_o' |
45942
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
45483
diff
changeset
|
182 """ |
17608
776240123525
store: extract functions _encodefname and _decodefname
Adrian Buehlmann <adrian@cadifra.com>
parents:
17607
diff
changeset
|
183 return _encodefname(encodedir(s)) |
776240123525
store: extract functions _encodefname and _decodefname
Adrian Buehlmann <adrian@cadifra.com>
parents:
17607
diff
changeset
|
184 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
185 |
17608
776240123525
store: extract functions _encodefname and _decodefname
Adrian Buehlmann <adrian@cadifra.com>
parents:
17607
diff
changeset
|
186 def decodefilename(s): |
45942
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
45483
diff
changeset
|
187 """ |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
33412
diff
changeset
|
188 >>> decodefilename(b'foo.i.hg/bar.d.hg/bla.hg.hg/hi~3aworld~3f/_h_e_l_l_o') |
17608
776240123525
store: extract functions _encodefname and _decodefname
Adrian Buehlmann <adrian@cadifra.com>
parents:
17607
diff
changeset
|
189 'foo.i/bar.d/bla.hg/hi:world?/HELLO' |
45942
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
45483
diff
changeset
|
190 """ |
17608
776240123525
store: extract functions _encodefname and _decodefname
Adrian Buehlmann <adrian@cadifra.com>
parents:
17607
diff
changeset
|
191 return decodedir(_decodefname(s)) |
6839
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
192 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
193 |
14288
00a0ab08f986
store: change names to comply with project coding standards
Adrian Buehlmann <adrian@cadifra.com>
parents:
14194
diff
changeset
|
194 def _buildlowerencodefun(): |
45942
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
45483
diff
changeset
|
195 """ |
14288
00a0ab08f986
store: change names to comply with project coding standards
Adrian Buehlmann <adrian@cadifra.com>
parents:
14194
diff
changeset
|
196 >>> f = _buildlowerencodefun() |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
33412
diff
changeset
|
197 >>> f(b'nothing/special.txt') |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
198 'nothing/special.txt' |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
33412
diff
changeset
|
199 >>> f(b'HELLO') |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
200 'hello' |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
33412
diff
changeset
|
201 >>> f(b'hello:world?') |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
202 'hello~3aworld~3f' |
34136
414a3513c2bd
doctest: do not embed non-ascii characters in docstring
Yuya Nishihara <yuya@tcha.org>
parents:
34131
diff
changeset
|
203 >>> f(b'the\\x07quick\\xADshot') |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
204 'the~07quick~adshot' |
45942
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
45483
diff
changeset
|
205 """ |
34214
7e3f078b6f31
py3: use bytechr() in store._buildlowerencodefun()
Yuya Nishihara <yuya@tcha.org>
parents:
34213
diff
changeset
|
206 xchr = pycompat.bytechr |
49284
d44e3c45f0e4
py3: replace `pycompat.xrange` by `range`
Manuel Jacob <me@manueljacob.de>
parents:
48946
diff
changeset
|
207 cmap = {xchr(x): xchr(x) for x in range(127)} |
29071
2f58975eb2cb
store: treat range as a generator instead of a list for py3 compat
timeless <timeless@mozdev.org>
parents:
28007
diff
changeset
|
208 for x in _reserved(): |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
209 cmap[xchr(x)] = b"~%02x" % x |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
210 for x in range(ord(b"A"), ord(b"Z") + 1): |
34214
7e3f078b6f31
py3: use bytechr() in store._buildlowerencodefun()
Yuya Nishihara <yuya@tcha.org>
parents:
34213
diff
changeset
|
211 cmap[xchr(x)] = xchr(x).lower() |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
212 |
34213
96808804b68f
store: give name to lowerencode function
Yuya Nishihara <yuya@tcha.org>
parents:
34136
diff
changeset
|
213 def lowerencode(s): |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
214 return b"".join([cmap[c] for c in pycompat.iterbytestr(s)]) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
215 |
34213
96808804b68f
store: give name to lowerencode function
Yuya Nishihara <yuya@tcha.org>
parents:
34136
diff
changeset
|
216 return lowerencode |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
217 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
218 |
18430
0459c6555f69
store: implement lowerencode in C
Bryan O'Sullivan <bryano@fb.com>
parents:
18054
diff
changeset
|
219 lowerencode = getattr(parsers, 'lowerencode', None) or _buildlowerencodefun() |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
220 |
17570
f53a7b256ca6
store: optimze _auxencode() a bit by grouping the reserved names by length
Adrian Buehlmann <adrian@cadifra.com>
parents:
17569
diff
changeset
|
221 # Windows reserved names: con, prn, aux, nul, com1..com9, lpt1..lpt9 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
222 _winres3 = (b'aux', b'con', b'prn', b'nul') # length 3 |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
223 _winres4 = (b'com', b'lpt') # length 4 (with trailing 1..9) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
224 |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
225 |
12687
34d8247a4595
store: encode first period or space in filenames (issue1713)
Adrian Buehlmann <adrian@cadifra.com>
parents:
12171
diff
changeset
|
226 def _auxencode(path, dotencode): |
45942
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
45483
diff
changeset
|
227 """ |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
228 Encodes filenames containing names reserved by Windows or which end in |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
229 period or space. Does not touch other single reserved characters c. |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
230 Specifically, c in '\\:*?"<>|' or ord(c) <= 31 are *not* encoded here. |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
231 Additionally encodes space or period at the beginning, if dotencode is |
17569
e9af2134825c
store: explain "aux.foo" versus "foo.aux" in doc of _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents:
17568
diff
changeset
|
232 True. Parameter path is assumed to be all lowercase. |
e9af2134825c
store: explain "aux.foo" versus "foo.aux" in doc of _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents:
17568
diff
changeset
|
233 A segment only needs encoding if a reserved name appears as a |
e9af2134825c
store: explain "aux.foo" versus "foo.aux" in doc of _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents:
17568
diff
changeset
|
234 basename (e.g. "aux", "aux.foo"). A directory or file named "foo.aux" |
e9af2134825c
store: explain "aux.foo" versus "foo.aux" in doc of _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents:
17568
diff
changeset
|
235 doesn't need encoding. |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
236 |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
33412
diff
changeset
|
237 >>> s = b'.foo/aux.txt/txt.aux/con/prn/nul/foo.' |
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
33412
diff
changeset
|
238 >>> _auxencode(s.split(b'/'), True) |
17574
81a033bb29bc
store: let _auxencode() return the list of path segments
Adrian Buehlmann <adrian@cadifra.com>
parents:
17573
diff
changeset
|
239 ['~2efoo', 'au~78.txt', 'txt.aux', 'co~6e', 'pr~6e', 'nu~6c', 'foo~2e'] |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
33412
diff
changeset
|
240 >>> s = b'.com1com2/lpt9.lpt4.lpt1/conprn/com0/lpt0/foo.' |
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
33412
diff
changeset
|
241 >>> _auxencode(s.split(b'/'), False) |
17574
81a033bb29bc
store: let _auxencode() return the list of path segments
Adrian Buehlmann <adrian@cadifra.com>
parents:
17573
diff
changeset
|
242 ['.com1com2', 'lp~749.lpt4.lpt1', 'conprn', 'com0', 'lpt0', 'foo~2e'] |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
33412
diff
changeset
|
243 >>> _auxencode([b'foo. '], True) |
17574
81a033bb29bc
store: let _auxencode() return the list of path segments
Adrian Buehlmann <adrian@cadifra.com>
parents:
17573
diff
changeset
|
244 ['foo.~20'] |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
33412
diff
changeset
|
245 >>> _auxencode([b' .foo'], True) |
17574
81a033bb29bc
store: let _auxencode() return the list of path segments
Adrian Buehlmann <adrian@cadifra.com>
parents:
17573
diff
changeset
|
246 ['~20.foo'] |
45942
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
45483
diff
changeset
|
247 """ |
17589
b11024849db6
store: parameter path of _auxencode is now a list of strings
Adrian Buehlmann <adrian@cadifra.com>
parents:
17588
diff
changeset
|
248 for i, n in enumerate(path): |
17572
b644287e79a8
store: unindent most of the contents of the for loop in _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents:
17571
diff
changeset
|
249 if not n: |
b644287e79a8
store: unindent most of the contents of the for loop in _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents:
17571
diff
changeset
|
250 continue |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
251 if dotencode and n[0] in b'. ': |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
252 n = b"~%02x" % ord(n[0:1]) + n[1:] |
17589
b11024849db6
store: parameter path of _auxencode is now a list of strings
Adrian Buehlmann <adrian@cadifra.com>
parents:
17588
diff
changeset
|
253 path[i] = n |
17572
b644287e79a8
store: unindent most of the contents of the for loop in _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents:
17571
diff
changeset
|
254 else: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
255 l = n.find(b'.') |
17572
b644287e79a8
store: unindent most of the contents of the for loop in _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents:
17571
diff
changeset
|
256 if l == -1: |
b644287e79a8
store: unindent most of the contents of the for loop in _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents:
17571
diff
changeset
|
257 l = len(n) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
258 if (l == 3 and n[:3] in _winres3) or ( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
259 l == 4 |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
260 and n[3:4] <= b'9' |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
261 and n[3:4] >= b'1' |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
262 and n[:3] in _winres4 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
263 ): |
17572
b644287e79a8
store: unindent most of the contents of the for loop in _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents:
17571
diff
changeset
|
264 # encode third letter ('aux' -> 'au~78') |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
265 ec = b"~%02x" % ord(n[2:3]) |
17572
b644287e79a8
store: unindent most of the contents of the for loop in _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents:
17571
diff
changeset
|
266 n = n[0:2] + ec + n[3:] |
17589
b11024849db6
store: parameter path of _auxencode is now a list of strings
Adrian Buehlmann <adrian@cadifra.com>
parents:
17588
diff
changeset
|
267 path[i] = n |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
268 if n[-1] in b'. ': |
17572
b644287e79a8
store: unindent most of the contents of the for loop in _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents:
17571
diff
changeset
|
269 # encode last period or space ('foo...' -> 'foo..~2e') |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
270 path[i] = n[:-1] + b"~%02x" % ord(n[-1:]) |
17589
b11024849db6
store: parameter path of _auxencode is now a list of strings
Adrian Buehlmann <adrian@cadifra.com>
parents:
17588
diff
changeset
|
271 return path |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
272 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
273 |
14288
00a0ab08f986
store: change names to comply with project coding standards
Adrian Buehlmann <adrian@cadifra.com>
parents:
14194
diff
changeset
|
274 _maxstorepathlen = 120 |
00a0ab08f986
store: change names to comply with project coding standards
Adrian Buehlmann <adrian@cadifra.com>
parents:
14194
diff
changeset
|
275 _dirprefixlen = 8 |
00a0ab08f986
store: change names to comply with project coding standards
Adrian Buehlmann <adrian@cadifra.com>
parents:
14194
diff
changeset
|
276 _maxshortdirslen = 8 * (_dirprefixlen + 1) - 4 |
17610
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
277 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
278 |
17610
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
279 def _hashencode(path, dotencode): |
46113
59fa3890d40a
node: import symbols explicitly
Joerg Sonnenberger <joerg@bec.de>
parents:
45942
diff
changeset
|
280 digest = hex(hashutil.sha1(path).digest()) |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
281 le = lowerencode(path[5:]).split(b'/') # skips prefix 'data/' or 'meta/' |
17610
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
282 parts = _auxencode(le, dotencode) |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
283 basename = parts[-1] |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
284 _root, ext = os.path.splitext(basename) |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
285 sdirs = [] |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
286 sdirslen = 0 |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
287 for p in parts[:-1]: |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
288 d = p[:_dirprefixlen] |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
289 if d[-1] in b'. ': |
17610
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
290 # Windows can't access dirs ending in period or space |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
291 d = d[:-1] + b'_' |
17610
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
292 if sdirslen == 0: |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
293 t = len(d) |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
294 else: |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
295 t = sdirslen + 1 + len(d) |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
296 if t > _maxshortdirslen: |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
297 break |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
298 sdirs.append(d) |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
299 sdirslen = t |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
300 dirs = b'/'.join(sdirs) |
17610
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
301 if len(dirs) > 0: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
302 dirs += b'/' |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
303 res = b'dh/' + dirs + digest + ext |
17610
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
304 spaceleft = _maxstorepathlen - len(res) |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
305 if spaceleft > 0: |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
306 filler = basename[:spaceleft] |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
307 res = b'dh/' + dirs + filler + digest + ext |
17610
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
308 return res |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
309 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
310 |
17590
eb0884680f5c
store: eliminate one level of lambda functions on _hybridencode
Adrian Buehlmann <adrian@cadifra.com>
parents:
17589
diff
changeset
|
311 def _hybridencode(path, dotencode): |
45942
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
45483
diff
changeset
|
312 """encodes path with a length limit |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
313 |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
314 Encodes all paths that begin with 'data/', according to the following. |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
315 |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
316 Default encoding (reversible): |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
317 |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
318 Encodes all uppercase letters 'X' as '_x'. All reserved or illegal |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
319 characters are encoded as '~xx', where xx is the two digit hex code |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
320 of the character (see encodefilename). |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
321 Relevant path components consisting of Windows reserved filenames are |
17738
b8424c92ba2b
spelling: fix minor spell checker issues
Mads Kiilerich <mads@kiilerich.com>
parents:
17731
diff
changeset
|
322 masked by encoding the third character ('aux' -> 'au~78', see _auxencode). |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
323 |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
324 Hashed encoding (not reversible): |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
325 |
14288
00a0ab08f986
store: change names to comply with project coding standards
Adrian Buehlmann <adrian@cadifra.com>
parents:
14194
diff
changeset
|
326 If the default-encoded path is longer than _maxstorepathlen, a |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
327 non-reversible hybrid hashing of the path is done instead. |
14288
00a0ab08f986
store: change names to comply with project coding standards
Adrian Buehlmann <adrian@cadifra.com>
parents:
14194
diff
changeset
|
328 This encoding uses up to _dirprefixlen characters of all directory |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
329 levels of the lowerencoded path, but not more levels than can fit into |
14288
00a0ab08f986
store: change names to comply with project coding standards
Adrian Buehlmann <adrian@cadifra.com>
parents:
14194
diff
changeset
|
330 _maxshortdirslen. |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
331 Then follows the filler followed by the sha digest of the full path. |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
332 The filler is the beginning of the basename of the lowerencoded path |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
333 (the basename is everything after the last path separator). The filler |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
334 is as long as possible, filling in characters from the basename until |
14288
00a0ab08f986
store: change names to comply with project coding standards
Adrian Buehlmann <adrian@cadifra.com>
parents:
14194
diff
changeset
|
335 the encoded path has _maxstorepathlen characters (or all chars of the |
00a0ab08f986
store: change names to comply with project coding standards
Adrian Buehlmann <adrian@cadifra.com>
parents:
14194
diff
changeset
|
336 basename have been taken). |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
337 The extension (e.g. '.i' or '.d') is preserved. |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
338 |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
339 The string 'data/' at the beginning is replaced with 'dh/', if the hashed |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
340 encoding was used. |
45942
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
45483
diff
changeset
|
341 """ |
17609
cbc180cfd60b
store: reuse direncoded path in _hybridencode
Adrian Buehlmann <adrian@cadifra.com>
parents:
17608
diff
changeset
|
342 path = encodedir(path) |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
343 ef = _encodefname(path).split(b'/') |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
344 res = b'/'.join(_auxencode(ef, dotencode)) |
14288
00a0ab08f986
store: change names to comply with project coding standards
Adrian Buehlmann <adrian@cadifra.com>
parents:
14194
diff
changeset
|
345 if len(res) > _maxstorepathlen: |
17610
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
346 res = _hashencode(path, dotencode) |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
347 return res |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
348 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
349 |
17624
ae103510f6aa
store: add a fallback _pathencode Python function
Adrian Buehlmann <adrian@cadifra.com>
parents:
17623
diff
changeset
|
350 def _pathencode(path): |
18435
8c019d2fd7c0
store: switch to C-based hashed path encoding
Bryan O'Sullivan <bryano@fb.com>
parents:
18430
diff
changeset
|
351 de = encodedir(path) |
17693
0c6de45e1212
store: optimize _pathencode by checking the length of the unencoded path
Adrian Buehlmann <adrian@cadifra.com>
parents:
17653
diff
changeset
|
352 if len(path) > _maxstorepathlen: |
18435
8c019d2fd7c0
store: switch to C-based hashed path encoding
Bryan O'Sullivan <bryano@fb.com>
parents:
18430
diff
changeset
|
353 return _hashencode(de, True) |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
354 ef = _encodefname(de).split(b'/') |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
355 res = b'/'.join(_auxencode(ef, True)) |
17624
ae103510f6aa
store: add a fallback _pathencode Python function
Adrian Buehlmann <adrian@cadifra.com>
parents:
17623
diff
changeset
|
356 if len(res) > _maxstorepathlen: |
18435
8c019d2fd7c0
store: switch to C-based hashed path encoding
Bryan O'Sullivan <bryano@fb.com>
parents:
18430
diff
changeset
|
357 return _hashencode(de, True) |
17624
ae103510f6aa
store: add a fallback _pathencode Python function
Adrian Buehlmann <adrian@cadifra.com>
parents:
17623
diff
changeset
|
358 return res |
ae103510f6aa
store: add a fallback _pathencode Python function
Adrian Buehlmann <adrian@cadifra.com>
parents:
17623
diff
changeset
|
359 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
360 |
17624
ae103510f6aa
store: add a fallback _pathencode Python function
Adrian Buehlmann <adrian@cadifra.com>
parents:
17623
diff
changeset
|
361 _pathencode = getattr(parsers, 'pathencode', _pathencode) |
ae103510f6aa
store: add a fallback _pathencode Python function
Adrian Buehlmann <adrian@cadifra.com>
parents:
17623
diff
changeset
|
362 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
363 |
17623
448e6ed7c557
store: move _plainhybridencode and _dothybridencode higher up in the file
Adrian Buehlmann <adrian@cadifra.com>
parents:
17621
diff
changeset
|
364 def _plainhybridencode(f): |
448e6ed7c557
store: move _plainhybridencode and _dothybridencode higher up in the file
Adrian Buehlmann <adrian@cadifra.com>
parents:
17621
diff
changeset
|
365 return _hybridencode(f, False) |
448e6ed7c557
store: move _plainhybridencode and _dothybridencode higher up in the file
Adrian Buehlmann <adrian@cadifra.com>
parents:
17621
diff
changeset
|
366 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
367 |
17726
7cb7e17c23b2
store: invoke "os.stat()" for "createmode" initialization via vfs
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17725
diff
changeset
|
368 def _calcmode(vfs): |
6898
69aeaaaf6e07
store: simplify class hierarchy
Matt Mackall <mpm@selenic.com>
parents:
6897
diff
changeset
|
369 try: |
69aeaaaf6e07
store: simplify class hierarchy
Matt Mackall <mpm@selenic.com>
parents:
6897
diff
changeset
|
370 # files in .hg/ will be created using this mode |
17726
7cb7e17c23b2
store: invoke "os.stat()" for "createmode" initialization via vfs
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17725
diff
changeset
|
371 mode = vfs.stat().st_mode |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
372 # avoid some useless chmods |
25658
e93036747902
global: mass rewrite to use modern octal syntax
Gregory Szorc <gregory.szorc@gmail.com>
parents:
25091
diff
changeset
|
373 if (0o777 & ~util.umask) == (0o777 & mode): |
6898
69aeaaaf6e07
store: simplify class hierarchy
Matt Mackall <mpm@selenic.com>
parents:
6897
diff
changeset
|
374 mode = None |
69aeaaaf6e07
store: simplify class hierarchy
Matt Mackall <mpm@selenic.com>
parents:
6897
diff
changeset
|
375 except OSError: |
69aeaaaf6e07
store: simplify class hierarchy
Matt Mackall <mpm@selenic.com>
parents:
6897
diff
changeset
|
376 mode = None |
69aeaaaf6e07
store: simplify class hierarchy
Matt Mackall <mpm@selenic.com>
parents:
6897
diff
changeset
|
377 return mode |
69aeaaaf6e07
store: simplify class hierarchy
Matt Mackall <mpm@selenic.com>
parents:
6897
diff
changeset
|
378 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
379 |
45351
909dafff6a78
store: refactor space delimited list to proper data structure
Pulkit Goyal <7895pulkit@gmail.com>
parents:
44452
diff
changeset
|
380 _data = [ |
909dafff6a78
store: refactor space delimited list to proper data structure
Pulkit Goyal <7895pulkit@gmail.com>
parents:
44452
diff
changeset
|
381 b'bookmarks', |
909dafff6a78
store: refactor space delimited list to proper data structure
Pulkit Goyal <7895pulkit@gmail.com>
parents:
44452
diff
changeset
|
382 b'narrowspec', |
909dafff6a78
store: refactor space delimited list to proper data structure
Pulkit Goyal <7895pulkit@gmail.com>
parents:
44452
diff
changeset
|
383 b'data', |
909dafff6a78
store: refactor space delimited list to proper data structure
Pulkit Goyal <7895pulkit@gmail.com>
parents:
44452
diff
changeset
|
384 b'meta', |
909dafff6a78
store: refactor space delimited list to proper data structure
Pulkit Goyal <7895pulkit@gmail.com>
parents:
44452
diff
changeset
|
385 b'00manifest.d', |
909dafff6a78
store: refactor space delimited list to proper data structure
Pulkit Goyal <7895pulkit@gmail.com>
parents:
44452
diff
changeset
|
386 b'00manifest.i', |
909dafff6a78
store: refactor space delimited list to proper data structure
Pulkit Goyal <7895pulkit@gmail.com>
parents:
44452
diff
changeset
|
387 b'00changelog.d', |
909dafff6a78
store: refactor space delimited list to proper data structure
Pulkit Goyal <7895pulkit@gmail.com>
parents:
44452
diff
changeset
|
388 b'00changelog.i', |
909dafff6a78
store: refactor space delimited list to proper data structure
Pulkit Goyal <7895pulkit@gmail.com>
parents:
44452
diff
changeset
|
389 b'phaseroots', |
909dafff6a78
store: refactor space delimited list to proper data structure
Pulkit Goyal <7895pulkit@gmail.com>
parents:
44452
diff
changeset
|
390 b'obsstore', |
45483
d252f51ab032
share: introduce config option to store requires in .hg/store
Pulkit Goyal <7895pulkit@gmail.com>
parents:
45351
diff
changeset
|
391 b'requires', |
45351
909dafff6a78
store: refactor space delimited list to proper data structure
Pulkit Goyal <7895pulkit@gmail.com>
parents:
44452
diff
changeset
|
392 ] |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
393 |
50671
e06d1a779eb6
store: stop relying on a `revlog_type` property
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50670
diff
changeset
|
394 REVLOG_FILES_EXT = ( |
e06d1a779eb6
store: stop relying on a `revlog_type` property
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50670
diff
changeset
|
395 b'.i', |
47324
0a3fa41fa719
revlogv2: use a unique filename for data
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47234
diff
changeset
|
396 b'.idx', |
0a3fa41fa719
revlogv2: use a unique filename for data
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47234
diff
changeset
|
397 b'.d', |
0a3fa41fa719
revlogv2: use a unique filename for data
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47234
diff
changeset
|
398 b'.dat', |
0a3fa41fa719
revlogv2: use a unique filename for data
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47234
diff
changeset
|
399 b'.n', |
0a3fa41fa719
revlogv2: use a unique filename for data
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47234
diff
changeset
|
400 b'.nd', |
47389
e6292eb33384
revlog: store sidedata in their own file
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47324
diff
changeset
|
401 b'.sda', |
47324
0a3fa41fa719
revlogv2: use a unique filename for data
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47234
diff
changeset
|
402 ) |
50479
5217e36356bb
store: add logic to group revlog file together
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50478
diff
changeset
|
403 # file extension that also use a `-SOMELONGIDHASH.ext` form |
5217e36356bb
store: add logic to group revlog file together
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50478
diff
changeset
|
404 REVLOG_FILES_LONG_EXT = ( |
5217e36356bb
store: add logic to group revlog file together
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50478
diff
changeset
|
405 b'.nd', |
5217e36356bb
store: add logic to group revlog file together
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50478
diff
changeset
|
406 b'.idx', |
5217e36356bb
store: add logic to group revlog file together
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50478
diff
changeset
|
407 b'.dat', |
5217e36356bb
store: add logic to group revlog file together
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50478
diff
changeset
|
408 b'.sda', |
5217e36356bb
store: add logic to group revlog file together
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50478
diff
changeset
|
409 ) |
46989
aed6ceaad6d7
streamclone: treat volatile file as "fullfile"
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46895
diff
changeset
|
410 # files that are "volatile" and might change between listing and streaming |
aed6ceaad6d7
streamclone: treat volatile file as "fullfile"
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46895
diff
changeset
|
411 # |
aed6ceaad6d7
streamclone: treat volatile file as "fullfile"
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46895
diff
changeset
|
412 # note: the ".nd" file are nodemap data and won't "change" but they might be |
aed6ceaad6d7
streamclone: treat volatile file as "fullfile"
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46895
diff
changeset
|
413 # deleted. |
aed6ceaad6d7
streamclone: treat volatile file as "fullfile"
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46895
diff
changeset
|
414 REVLOG_FILES_VOLATILE_EXT = (b'.n', b'.nd') |
46895
6085b7f1536d
store: also return some information about the type of file `walk` found
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46854
diff
changeset
|
415 |
46990
0b569c75d180
store: exclude `undo.` nodemap's file from `walk`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
416 # some exception to the above matching |
47659
f030c7d22032
walk: no longer ignore revlogs of files starting with `undo.` (issue6542)
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47443
diff
changeset
|
417 # |
f030c7d22032
walk: no longer ignore revlogs of files starting with `undo.` (issue6542)
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47443
diff
changeset
|
418 # XXX This is currently not in use because of issue6542 |
50319
4be9ecc982e1
py3: fix for Python 3.12 emitting SyntaxWarning on invalid escape sequences
Mads Kiilerich <mads@kiilerich.com>
parents:
50315
diff
changeset
|
419 EXCLUDED = re.compile(br'.*undo\.[^/]+\.(nd?|i)$') |
46990
0b569c75d180
store: exclude `undo.` nodemap's file from `walk`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46989
diff
changeset
|
420 |
46895
6085b7f1536d
store: also return some information about the type of file `walk` found
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46854
diff
changeset
|
421 |
6085b7f1536d
store: also return some information about the type of file `walk` found
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46854
diff
changeset
|
422 def is_revlog(f, kind, st): |
6085b7f1536d
store: also return some information about the type of file `walk` found
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46854
diff
changeset
|
423 if kind != stat.S_IFREG: |
50671
e06d1a779eb6
store: stop relying on a `revlog_type` property
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50670
diff
changeset
|
424 return False |
e06d1a779eb6
store: stop relying on a `revlog_type` property
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50670
diff
changeset
|
425 if f.endswith(REVLOG_FILES_EXT): |
e06d1a779eb6
store: stop relying on a `revlog_type` property
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50670
diff
changeset
|
426 return True |
e06d1a779eb6
store: stop relying on a `revlog_type` property
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50670
diff
changeset
|
427 return False |
46895
6085b7f1536d
store: also return some information about the type of file `walk` found
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46854
diff
changeset
|
428 |
6085b7f1536d
store: also return some information about the type of file `walk` found
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46854
diff
changeset
|
429 |
50671
e06d1a779eb6
store: stop relying on a `revlog_type` property
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50670
diff
changeset
|
430 def is_revlog_file(f): |
e06d1a779eb6
store: stop relying on a `revlog_type` property
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50670
diff
changeset
|
431 if f.endswith(REVLOG_FILES_EXT): |
e06d1a779eb6
store: stop relying on a `revlog_type` property
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50670
diff
changeset
|
432 return True |
e06d1a779eb6
store: stop relying on a `revlog_type` property
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50670
diff
changeset
|
433 return False |
46383
374d7fff7cb5
store: use `endswith` to detect revlog extension
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46322
diff
changeset
|
434 |
6903
0642d9d7ec80
clone: get a list of files to clone from store
Matt Mackall <mpm@selenic.com>
parents:
6902
diff
changeset
|
435 |
50638
8fc10bfd9887
store: declare a `files` method on BaseStoreEntry
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50632
diff
changeset
|
436 @attr.s(slots=True) |
8fc10bfd9887
store: declare a `files` method on BaseStoreEntry
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50632
diff
changeset
|
437 class StoreFile: |
8fc10bfd9887
store: declare a `files` method on BaseStoreEntry
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50632
diff
changeset
|
438 """a file matching a store entry""" |
8fc10bfd9887
store: declare a `files` method on BaseStoreEntry
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50632
diff
changeset
|
439 |
8fc10bfd9887
store: declare a `files` method on BaseStoreEntry
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50632
diff
changeset
|
440 unencoded_path = attr.ib() |
8fc10bfd9887
store: declare a `files` method on BaseStoreEntry
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50632
diff
changeset
|
441 _file_size = attr.ib(default=None) |
8fc10bfd9887
store: declare a `files` method on BaseStoreEntry
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50632
diff
changeset
|
442 is_volatile = attr.ib(default=False) |
8fc10bfd9887
store: declare a `files` method on BaseStoreEntry
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50632
diff
changeset
|
443 |
8fc10bfd9887
store: declare a `files` method on BaseStoreEntry
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50632
diff
changeset
|
444 def file_size(self, vfs): |
8fc10bfd9887
store: declare a `files` method on BaseStoreEntry
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50632
diff
changeset
|
445 if self._file_size is None: |
50639
5e60abf811f3
stream-clone: make it the responsability of the store entry to stream content
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50638
diff
changeset
|
446 if vfs is None: |
5e60abf811f3
stream-clone: make it the responsability of the store entry to stream content
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50638
diff
changeset
|
447 msg = b"calling vfs-less file_size without prior call: %s" |
5e60abf811f3
stream-clone: make it the responsability of the store entry to stream content
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50638
diff
changeset
|
448 msg %= self.unencoded_path |
5e60abf811f3
stream-clone: make it the responsability of the store entry to stream content
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50638
diff
changeset
|
449 raise error.ProgrammingError(msg) |
50638
8fc10bfd9887
store: declare a `files` method on BaseStoreEntry
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50632
diff
changeset
|
450 try: |
8fc10bfd9887
store: declare a `files` method on BaseStoreEntry
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50632
diff
changeset
|
451 self._file_size = vfs.stat(self.unencoded_path).st_size |
8fc10bfd9887
store: declare a `files` method on BaseStoreEntry
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50632
diff
changeset
|
452 except FileNotFoundError: |
8fc10bfd9887
store: declare a `files` method on BaseStoreEntry
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50632
diff
changeset
|
453 self._file_size = 0 |
8fc10bfd9887
store: declare a `files` method on BaseStoreEntry
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50632
diff
changeset
|
454 return self._file_size |
8fc10bfd9887
store: declare a `files` method on BaseStoreEntry
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50632
diff
changeset
|
455 |
51547
8e2a68e10f5c
stream: in v3, skip the "size" fast path if the entries as some unknown size
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50929
diff
changeset
|
456 @property |
8e2a68e10f5c
stream: in v3, skip the "size" fast path if the entries as some unknown size
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50929
diff
changeset
|
457 def has_size(self): |
8e2a68e10f5c
stream: in v3, skip the "size" fast path if the entries as some unknown size
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50929
diff
changeset
|
458 return self._file_size is not None |
8e2a68e10f5c
stream: in v3, skip the "size" fast path if the entries as some unknown size
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50929
diff
changeset
|
459 |
50639
5e60abf811f3
stream-clone: make it the responsability of the store entry to stream content
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50638
diff
changeset
|
460 def get_stream(self, vfs, copies): |
5e60abf811f3
stream-clone: make it the responsability of the store entry to stream content
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50638
diff
changeset
|
461 """return data "stream" information for this file |
5e60abf811f3
stream-clone: make it the responsability of the store entry to stream content
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50638
diff
changeset
|
462 |
5e60abf811f3
stream-clone: make it the responsability of the store entry to stream content
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50638
diff
changeset
|
463 (unencoded_file_path, content_iterator, content_size) |
5e60abf811f3
stream-clone: make it the responsability of the store entry to stream content
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50638
diff
changeset
|
464 """ |
5e60abf811f3
stream-clone: make it the responsability of the store entry to stream content
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50638
diff
changeset
|
465 size = self.file_size(None) |
5e60abf811f3
stream-clone: make it the responsability of the store entry to stream content
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50638
diff
changeset
|
466 |
5e60abf811f3
stream-clone: make it the responsability of the store entry to stream content
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50638
diff
changeset
|
467 def get_stream(): |
5e60abf811f3
stream-clone: make it the responsability of the store entry to stream content
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50638
diff
changeset
|
468 actual_path = copies[vfs.join(self.unencoded_path)] |
5e60abf811f3
stream-clone: make it the responsability of the store entry to stream content
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50638
diff
changeset
|
469 with open(actual_path, 'rb') as fp: |
5e60abf811f3
stream-clone: make it the responsability of the store entry to stream content
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50638
diff
changeset
|
470 yield None # ready to stream |
5e60abf811f3
stream-clone: make it the responsability of the store entry to stream content
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50638
diff
changeset
|
471 if size <= 65536: |
5e60abf811f3
stream-clone: make it the responsability of the store entry to stream content
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50638
diff
changeset
|
472 yield fp.read(size) |
5e60abf811f3
stream-clone: make it the responsability of the store entry to stream content
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50638
diff
changeset
|
473 else: |
5e60abf811f3
stream-clone: make it the responsability of the store entry to stream content
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50638
diff
changeset
|
474 yield from util.filechunkiter(fp, limit=size) |
5e60abf811f3
stream-clone: make it the responsability of the store entry to stream content
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50638
diff
changeset
|
475 |
5e60abf811f3
stream-clone: make it the responsability of the store entry to stream content
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50638
diff
changeset
|
476 s = get_stream() |
5e60abf811f3
stream-clone: make it the responsability of the store entry to stream content
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50638
diff
changeset
|
477 next(s) |
5e60abf811f3
stream-clone: make it the responsability of the store entry to stream content
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50638
diff
changeset
|
478 return (self.unencoded_path, s, size) |
5e60abf811f3
stream-clone: make it the responsability of the store entry to stream content
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50638
diff
changeset
|
479 |
50638
8fc10bfd9887
store: declare a `files` method on BaseStoreEntry
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50632
diff
changeset
|
480 |
50474
c37450a5f1dc
store: have custom init for entries class
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50473
diff
changeset
|
481 @attr.s(slots=True, init=False) |
50473
5a2fb64d38b2
store: use specialized class for store entries
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50472
diff
changeset
|
482 class BaseStoreEntry: |
50471
521fec115dad
store: use a StoreEntry object instead of tuple for store files
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50469
diff
changeset
|
483 """An entry in the store |
521fec115dad
store: use a StoreEntry object instead of tuple for store files
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50469
diff
changeset
|
484 |
521fec115dad
store: use a StoreEntry object instead of tuple for store files
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50469
diff
changeset
|
485 This is returned by `store.walk` and represent some data in the store.""" |
521fec115dad
store: use a StoreEntry object instead of tuple for store files
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50469
diff
changeset
|
486 |
51552
49faa72b994e
streamclone: stop listing files for entries that have no volatile files
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
51547
diff
changeset
|
487 maybe_volatile = True |
49faa72b994e
streamclone: stop listing files for entries that have no volatile files
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
51547
diff
changeset
|
488 |
50638
8fc10bfd9887
store: declare a `files` method on BaseStoreEntry
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50632
diff
changeset
|
489 def files(self) -> List[StoreFile]: |
8fc10bfd9887
store: declare a `files` method on BaseStoreEntry
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50632
diff
changeset
|
490 raise NotImplementedError |
8fc10bfd9887
store: declare a `files` method on BaseStoreEntry
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50632
diff
changeset
|
491 |
50640
9caa860dcbec
stream-clone: implement decidated `get_streams` method for revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50639
diff
changeset
|
492 def get_streams( |
9caa860dcbec
stream-clone: implement decidated `get_streams` method for revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50639
diff
changeset
|
493 self, |
9caa860dcbec
stream-clone: implement decidated `get_streams` method for revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50639
diff
changeset
|
494 repo=None, |
9caa860dcbec
stream-clone: implement decidated `get_streams` method for revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50639
diff
changeset
|
495 vfs=None, |
9caa860dcbec
stream-clone: implement decidated `get_streams` method for revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50639
diff
changeset
|
496 copies=None, |
9caa860dcbec
stream-clone: implement decidated `get_streams` method for revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50639
diff
changeset
|
497 max_changeset=None, |
50706
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50680
diff
changeset
|
498 preserve_file_count=False, |
50640
9caa860dcbec
stream-clone: implement decidated `get_streams` method for revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50639
diff
changeset
|
499 ): |
50639
5e60abf811f3
stream-clone: make it the responsability of the store entry to stream content
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50638
diff
changeset
|
500 """return a list of data stream associated to files for this entry |
5e60abf811f3
stream-clone: make it the responsability of the store entry to stream content
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50638
diff
changeset
|
501 |
5e60abf811f3
stream-clone: make it the responsability of the store entry to stream content
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50638
diff
changeset
|
502 return [(unencoded_file_path, content_iterator, content_size), …] |
5e60abf811f3
stream-clone: make it the responsability of the store entry to stream content
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50638
diff
changeset
|
503 """ |
5e60abf811f3
stream-clone: make it the responsability of the store entry to stream content
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50638
diff
changeset
|
504 assert vfs is not None |
5e60abf811f3
stream-clone: make it the responsability of the store entry to stream content
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50638
diff
changeset
|
505 return [f.get_stream(vfs, copies) for f in self.files()] |
5e60abf811f3
stream-clone: make it the responsability of the store entry to stream content
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50638
diff
changeset
|
506 |
50496
e50d1fe7ebb4
store: issue a single entry for each revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50495
diff
changeset
|
507 |
e50d1fe7ebb4
store: issue a single entry for each revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50495
diff
changeset
|
508 @attr.s(slots=True, init=False) |
e50d1fe7ebb4
store: issue a single entry for each revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50495
diff
changeset
|
509 class SimpleStoreEntry(BaseStoreEntry): |
e50d1fe7ebb4
store: issue a single entry for each revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50495
diff
changeset
|
510 """A generic entry in the store""" |
e50d1fe7ebb4
store: issue a single entry for each revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50495
diff
changeset
|
511 |
e50d1fe7ebb4
store: issue a single entry for each revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50495
diff
changeset
|
512 is_revlog = False |
e50d1fe7ebb4
store: issue a single entry for each revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50495
diff
changeset
|
513 |
51552
49faa72b994e
streamclone: stop listing files for entries that have no volatile files
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
51547
diff
changeset
|
514 maybe_volatile = attr.ib() |
50495
ed8cda1c18e1
store: rename `unencoded_path` to `entry_path` for StoreEntry
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50494
diff
changeset
|
515 _entry_path = attr.ib() |
50476
2b2284cf949b
store: only access is_volatile information through the file object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50475
diff
changeset
|
516 _is_volatile = attr.ib(default=False) |
50475
7d4d2a160cf5
store: only access file_size information through the file object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50474
diff
changeset
|
517 _file_size = attr.ib(default=None) |
50631
b59e0a4f692f
store: cache the `files()` return for store entries
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50629
diff
changeset
|
518 _files = attr.ib(default=None) |
50471
521fec115dad
store: use a StoreEntry object instead of tuple for store files
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50469
diff
changeset
|
519 |
50474
c37450a5f1dc
store: have custom init for entries class
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50473
diff
changeset
|
520 def __init__( |
c37450a5f1dc
store: have custom init for entries class
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50473
diff
changeset
|
521 self, |
50495
ed8cda1c18e1
store: rename `unencoded_path` to `entry_path` for StoreEntry
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50494
diff
changeset
|
522 entry_path, |
50474
c37450a5f1dc
store: have custom init for entries class
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50473
diff
changeset
|
523 is_volatile=False, |
c37450a5f1dc
store: have custom init for entries class
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50473
diff
changeset
|
524 file_size=None, |
c37450a5f1dc
store: have custom init for entries class
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50473
diff
changeset
|
525 ): |
50496
e50d1fe7ebb4
store: issue a single entry for each revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50495
diff
changeset
|
526 super().__init__() |
50495
ed8cda1c18e1
store: rename `unencoded_path` to `entry_path` for StoreEntry
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50494
diff
changeset
|
527 self._entry_path = entry_path |
50476
2b2284cf949b
store: only access is_volatile information through the file object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50475
diff
changeset
|
528 self._is_volatile = is_volatile |
50475
7d4d2a160cf5
store: only access file_size information through the file object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50474
diff
changeset
|
529 self._file_size = file_size |
50631
b59e0a4f692f
store: cache the `files()` return for store entries
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50629
diff
changeset
|
530 self._files = None |
51552
49faa72b994e
streamclone: stop listing files for entries that have no volatile files
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
51547
diff
changeset
|
531 self.maybe_volatile = is_volatile |
50474
c37450a5f1dc
store: have custom init for entries class
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50473
diff
changeset
|
532 |
50638
8fc10bfd9887
store: declare a `files` method on BaseStoreEntry
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50632
diff
changeset
|
533 def files(self) -> List[StoreFile]: |
50631
b59e0a4f692f
store: cache the `files()` return for store entries
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50629
diff
changeset
|
534 if self._files is None: |
b59e0a4f692f
store: cache the `files()` return for store entries
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50629
diff
changeset
|
535 self._files = [ |
b59e0a4f692f
store: cache the `files()` return for store entries
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50629
diff
changeset
|
536 StoreFile( |
b59e0a4f692f
store: cache the `files()` return for store entries
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50629
diff
changeset
|
537 unencoded_path=self._entry_path, |
b59e0a4f692f
store: cache the `files()` return for store entries
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50629
diff
changeset
|
538 file_size=self._file_size, |
b59e0a4f692f
store: cache the `files()` return for store entries
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50629
diff
changeset
|
539 is_volatile=self._is_volatile, |
b59e0a4f692f
store: cache the `files()` return for store entries
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50629
diff
changeset
|
540 ) |
b59e0a4f692f
store: cache the `files()` return for store entries
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50629
diff
changeset
|
541 ] |
b59e0a4f692f
store: cache the `files()` return for store entries
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50629
diff
changeset
|
542 return self._files |
50472
9fdc28e21b68
store: introduce a EntryFile object to actually access file info
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50471
diff
changeset
|
543 |
9fdc28e21b68
store: introduce a EntryFile object to actually access file info
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50471
diff
changeset
|
544 |
50474
c37450a5f1dc
store: have custom init for entries class
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50473
diff
changeset
|
545 @attr.s(slots=True, init=False) |
50473
5a2fb64d38b2
store: use specialized class for store entries
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50472
diff
changeset
|
546 class RevlogStoreEntry(BaseStoreEntry): |
5a2fb64d38b2
store: use specialized class for store entries
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50472
diff
changeset
|
547 """A revlog entry in the store""" |
5a2fb64d38b2
store: use specialized class for store entries
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50472
diff
changeset
|
548 |
5a2fb64d38b2
store: use specialized class for store entries
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50472
diff
changeset
|
549 is_revlog = True |
50496
e50d1fe7ebb4
store: issue a single entry for each revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50495
diff
changeset
|
550 |
50473
5a2fb64d38b2
store: use specialized class for store entries
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50472
diff
changeset
|
551 revlog_type = attr.ib(default=None) |
50483
60e613f6a229
store: add a `target_id` attribute on RevlogStoreEntry
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50482
diff
changeset
|
552 target_id = attr.ib(default=None) |
51552
49faa72b994e
streamclone: stop listing files for entries that have no volatile files
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
51547
diff
changeset
|
553 maybe_volatile = attr.ib(default=True) |
50496
e50d1fe7ebb4
store: issue a single entry for each revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50495
diff
changeset
|
554 _path_prefix = attr.ib(default=None) |
e50d1fe7ebb4
store: issue a single entry for each revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50495
diff
changeset
|
555 _details = attr.ib(default=None) |
50631
b59e0a4f692f
store: cache the `files()` return for store entries
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50629
diff
changeset
|
556 _files = attr.ib(default=None) |
50473
5a2fb64d38b2
store: use specialized class for store entries
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50472
diff
changeset
|
557 |
50474
c37450a5f1dc
store: have custom init for entries class
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50473
diff
changeset
|
558 def __init__( |
c37450a5f1dc
store: have custom init for entries class
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50473
diff
changeset
|
559 self, |
c37450a5f1dc
store: have custom init for entries class
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50473
diff
changeset
|
560 revlog_type, |
50496
e50d1fe7ebb4
store: issue a single entry for each revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50495
diff
changeset
|
561 path_prefix, |
50483
60e613f6a229
store: add a `target_id` attribute on RevlogStoreEntry
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50482
diff
changeset
|
562 target_id, |
50496
e50d1fe7ebb4
store: issue a single entry for each revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50495
diff
changeset
|
563 details, |
50474
c37450a5f1dc
store: have custom init for entries class
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50473
diff
changeset
|
564 ): |
50496
e50d1fe7ebb4
store: issue a single entry for each revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50495
diff
changeset
|
565 super().__init__() |
50474
c37450a5f1dc
store: have custom init for entries class
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50473
diff
changeset
|
566 self.revlog_type = revlog_type |
50483
60e613f6a229
store: add a `target_id` attribute on RevlogStoreEntry
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50482
diff
changeset
|
567 self.target_id = target_id |
50496
e50d1fe7ebb4
store: issue a single entry for each revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50495
diff
changeset
|
568 self._path_prefix = path_prefix |
e50d1fe7ebb4
store: issue a single entry for each revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50495
diff
changeset
|
569 assert b'.i' in details, (path_prefix, details) |
51552
49faa72b994e
streamclone: stop listing files for entries that have no volatile files
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
51547
diff
changeset
|
570 for ext in details: |
49faa72b994e
streamclone: stop listing files for entries that have no volatile files
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
51547
diff
changeset
|
571 if ext.endswith(REVLOG_FILES_VOLATILE_EXT): |
49faa72b994e
streamclone: stop listing files for entries that have no volatile files
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
51547
diff
changeset
|
572 self.maybe_volatile = True |
49faa72b994e
streamclone: stop listing files for entries that have no volatile files
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
51547
diff
changeset
|
573 break |
49faa72b994e
streamclone: stop listing files for entries that have no volatile files
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
51547
diff
changeset
|
574 else: |
49faa72b994e
streamclone: stop listing files for entries that have no volatile files
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
51547
diff
changeset
|
575 self.maybe_volatile = False |
50496
e50d1fe7ebb4
store: issue a single entry for each revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50495
diff
changeset
|
576 self._details = details |
50631
b59e0a4f692f
store: cache the `files()` return for store entries
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50629
diff
changeset
|
577 self._files = None |
50474
c37450a5f1dc
store: have custom init for entries class
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50473
diff
changeset
|
578 |
50497
66c556968222
store: introduce boolean property for revlog type
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50496
diff
changeset
|
579 @property |
66c556968222
store: introduce boolean property for revlog type
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50496
diff
changeset
|
580 def is_changelog(self): |
50680
e324329fc97e
store: use the revlog type from revlog's constant instead of custom one
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50672
diff
changeset
|
581 return self.revlog_type == KIND_CHANGELOG |
50497
66c556968222
store: introduce boolean property for revlog type
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50496
diff
changeset
|
582 |
66c556968222
store: introduce boolean property for revlog type
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50496
diff
changeset
|
583 @property |
66c556968222
store: introduce boolean property for revlog type
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50496
diff
changeset
|
584 def is_manifestlog(self): |
50680
e324329fc97e
store: use the revlog type from revlog's constant instead of custom one
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50672
diff
changeset
|
585 return self.revlog_type == KIND_MANIFESTLOG |
50497
66c556968222
store: introduce boolean property for revlog type
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50496
diff
changeset
|
586 |
66c556968222
store: introduce boolean property for revlog type
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50496
diff
changeset
|
587 @property |
66c556968222
store: introduce boolean property for revlog type
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50496
diff
changeset
|
588 def is_filelog(self): |
50680
e324329fc97e
store: use the revlog type from revlog's constant instead of custom one
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50672
diff
changeset
|
589 return self.revlog_type == KIND_FILELOG |
50497
66c556968222
store: introduce boolean property for revlog type
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50496
diff
changeset
|
590 |
50492
3473d18c029a
store: introduce a main_file_path method for revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50483
diff
changeset
|
591 def main_file_path(self): |
3473d18c029a
store: introduce a main_file_path method for revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50483
diff
changeset
|
592 """unencoded path of the main revlog file""" |
50496
e50d1fe7ebb4
store: issue a single entry for each revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50495
diff
changeset
|
593 return self._path_prefix + b'.i' |
e50d1fe7ebb4
store: issue a single entry for each revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50495
diff
changeset
|
594 |
50638
8fc10bfd9887
store: declare a `files` method on BaseStoreEntry
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50632
diff
changeset
|
595 def files(self) -> List[StoreFile]: |
50631
b59e0a4f692f
store: cache the `files()` return for store entries
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50629
diff
changeset
|
596 if self._files is None: |
b59e0a4f692f
store: cache the `files()` return for store entries
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50629
diff
changeset
|
597 self._files = [] |
b59e0a4f692f
store: cache the `files()` return for store entries
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50629
diff
changeset
|
598 for ext in sorted(self._details, key=_ext_key): |
b59e0a4f692f
store: cache the `files()` return for store entries
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50629
diff
changeset
|
599 path = self._path_prefix + ext |
50670
11562d72cb7b
store: directly pass the filesize in the `details` of revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50669
diff
changeset
|
600 file_size = self._details[ext] |
50668
6b522a9e7451
store: have the revlog determine which files are volatile itself
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50641
diff
changeset
|
601 # files that are "volatile" and might change between |
6b522a9e7451
store: have the revlog determine which files are volatile itself
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50641
diff
changeset
|
602 # listing and streaming |
6b522a9e7451
store: have the revlog determine which files are volatile itself
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50641
diff
changeset
|
603 # |
6b522a9e7451
store: have the revlog determine which files are volatile itself
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50641
diff
changeset
|
604 # note: the ".nd" file are nodemap data and won't "change" |
6b522a9e7451
store: have the revlog determine which files are volatile itself
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50641
diff
changeset
|
605 # but they might be deleted. |
6b522a9e7451
store: have the revlog determine which files are volatile itself
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50641
diff
changeset
|
606 volatile = ext.endswith(REVLOG_FILES_VOLATILE_EXT) |
50669
cdb471c8ebcf
store: explicitly pass file_size when creating StoreFile
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50668
diff
changeset
|
607 f = StoreFile(path, file_size, volatile) |
50668
6b522a9e7451
store: have the revlog determine which files are volatile itself
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50641
diff
changeset
|
608 self._files.append(f) |
50631
b59e0a4f692f
store: cache the `files()` return for store entries
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50629
diff
changeset
|
609 return self._files |
50492
3473d18c029a
store: introduce a main_file_path method for revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50483
diff
changeset
|
610 |
50640
9caa860dcbec
stream-clone: implement decidated `get_streams` method for revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50639
diff
changeset
|
611 def get_streams( |
9caa860dcbec
stream-clone: implement decidated `get_streams` method for revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50639
diff
changeset
|
612 self, |
9caa860dcbec
stream-clone: implement decidated `get_streams` method for revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50639
diff
changeset
|
613 repo=None, |
9caa860dcbec
stream-clone: implement decidated `get_streams` method for revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50639
diff
changeset
|
614 vfs=None, |
9caa860dcbec
stream-clone: implement decidated `get_streams` method for revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50639
diff
changeset
|
615 copies=None, |
9caa860dcbec
stream-clone: implement decidated `get_streams` method for revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50639
diff
changeset
|
616 max_changeset=None, |
50706
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50680
diff
changeset
|
617 preserve_file_count=False, |
50640
9caa860dcbec
stream-clone: implement decidated `get_streams` method for revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50639
diff
changeset
|
618 ): |
51547
8e2a68e10f5c
stream: in v3, skip the "size" fast path if the entries as some unknown size
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50929
diff
changeset
|
619 pre_sized = all(f.has_size for f in self.files()) |
8e2a68e10f5c
stream: in v3, skip the "size" fast path if the entries as some unknown size
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50929
diff
changeset
|
620 if pre_sized and ( |
50672
3b56395404a1
stream-clone: avoid opening a revlog in case we do not need it
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50671
diff
changeset
|
621 repo is None |
3b56395404a1
stream-clone: avoid opening a revlog in case we do not need it
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50671
diff
changeset
|
622 or max_changeset is None |
50640
9caa860dcbec
stream-clone: implement decidated `get_streams` method for revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50639
diff
changeset
|
623 # This use revlog-v2, ignore for now |
50672
3b56395404a1
stream-clone: avoid opening a revlog in case we do not need it
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50671
diff
changeset
|
624 or any(k.endswith(b'.idx') for k in self._details.keys()) |
3b56395404a1
stream-clone: avoid opening a revlog in case we do not need it
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50671
diff
changeset
|
625 # This is not inline, no race expected |
3b56395404a1
stream-clone: avoid opening a revlog in case we do not need it
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50671
diff
changeset
|
626 or b'.d' in self._details |
3b56395404a1
stream-clone: avoid opening a revlog in case we do not need it
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50671
diff
changeset
|
627 ): |
50640
9caa860dcbec
stream-clone: implement decidated `get_streams` method for revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50639
diff
changeset
|
628 return super().get_streams( |
9caa860dcbec
stream-clone: implement decidated `get_streams` method for revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50639
diff
changeset
|
629 repo=repo, |
9caa860dcbec
stream-clone: implement decidated `get_streams` method for revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50639
diff
changeset
|
630 vfs=vfs, |
9caa860dcbec
stream-clone: implement decidated `get_streams` method for revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50639
diff
changeset
|
631 copies=copies, |
9caa860dcbec
stream-clone: implement decidated `get_streams` method for revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50639
diff
changeset
|
632 max_changeset=max_changeset, |
50706
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50680
diff
changeset
|
633 preserve_file_count=preserve_file_count, |
50640
9caa860dcbec
stream-clone: implement decidated `get_streams` method for revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50639
diff
changeset
|
634 ) |
50706
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50680
diff
changeset
|
635 elif not preserve_file_count: |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50680
diff
changeset
|
636 stream = [ |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50680
diff
changeset
|
637 f.get_stream(vfs, copies) |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50680
diff
changeset
|
638 for f in self.files() |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50680
diff
changeset
|
639 if not f.unencoded_path.endswith((b'.i', b'.d')) |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50680
diff
changeset
|
640 ] |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50680
diff
changeset
|
641 rl = self.get_revlog_instance(repo).get_revlog() |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50680
diff
changeset
|
642 rl_stream = rl.get_streams(max_changeset) |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50680
diff
changeset
|
643 stream.extend(rl_stream) |
0452af304808
stream-clone: add a v3 version of the protocol
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50680
diff
changeset
|
644 return stream |
50672
3b56395404a1
stream-clone: avoid opening a revlog in case we do not need it
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50671
diff
changeset
|
645 |
50640
9caa860dcbec
stream-clone: implement decidated `get_streams` method for revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50639
diff
changeset
|
646 name_to_size = {} |
9caa860dcbec
stream-clone: implement decidated `get_streams` method for revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50639
diff
changeset
|
647 for f in self.files(): |
9caa860dcbec
stream-clone: implement decidated `get_streams` method for revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50639
diff
changeset
|
648 name_to_size[f.unencoded_path] = f.file_size(None) |
50672
3b56395404a1
stream-clone: avoid opening a revlog in case we do not need it
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50671
diff
changeset
|
649 |
50640
9caa860dcbec
stream-clone: implement decidated `get_streams` method for revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50639
diff
changeset
|
650 stream = [ |
9caa860dcbec
stream-clone: implement decidated `get_streams` method for revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50639
diff
changeset
|
651 f.get_stream(vfs, copies) |
9caa860dcbec
stream-clone: implement decidated `get_streams` method for revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50639
diff
changeset
|
652 for f in self.files() |
50672
3b56395404a1
stream-clone: avoid opening a revlog in case we do not need it
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50671
diff
changeset
|
653 if not f.unencoded_path.endswith(b'.i') |
50640
9caa860dcbec
stream-clone: implement decidated `get_streams` method for revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50639
diff
changeset
|
654 ] |
9caa860dcbec
stream-clone: implement decidated `get_streams` method for revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50639
diff
changeset
|
655 |
50672
3b56395404a1
stream-clone: avoid opening a revlog in case we do not need it
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50671
diff
changeset
|
656 index_path = self._path_prefix + b'.i' |
50641
5460424092e2
stream-clone: smoothly detect and handle a case were a revlog is split
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50640
diff
changeset
|
657 |
50672
3b56395404a1
stream-clone: avoid opening a revlog in case we do not need it
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50671
diff
changeset
|
658 index_file = None |
3b56395404a1
stream-clone: avoid opening a revlog in case we do not need it
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50671
diff
changeset
|
659 try: |
3b56395404a1
stream-clone: avoid opening a revlog in case we do not need it
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50671
diff
changeset
|
660 index_file = vfs(index_path) |
3b56395404a1
stream-clone: avoid opening a revlog in case we do not need it
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50671
diff
changeset
|
661 header = index_file.read(INDEX_HEADER.size) |
3b56395404a1
stream-clone: avoid opening a revlog in case we do not need it
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50671
diff
changeset
|
662 if revlogmod.revlog.is_inline_index(header): |
3b56395404a1
stream-clone: avoid opening a revlog in case we do not need it
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50671
diff
changeset
|
663 size = name_to_size[index_path] |
50641
5460424092e2
stream-clone: smoothly detect and handle a case were a revlog is split
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50640
diff
changeset
|
664 |
50672
3b56395404a1
stream-clone: avoid opening a revlog in case we do not need it
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50671
diff
changeset
|
665 # no split underneath, just return the stream |
3b56395404a1
stream-clone: avoid opening a revlog in case we do not need it
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50671
diff
changeset
|
666 def get_stream(): |
3b56395404a1
stream-clone: avoid opening a revlog in case we do not need it
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50671
diff
changeset
|
667 fp = index_file |
3b56395404a1
stream-clone: avoid opening a revlog in case we do not need it
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50671
diff
changeset
|
668 try: |
3b56395404a1
stream-clone: avoid opening a revlog in case we do not need it
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50671
diff
changeset
|
669 fp.seek(0) |
3b56395404a1
stream-clone: avoid opening a revlog in case we do not need it
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50671
diff
changeset
|
670 yield None |
3b56395404a1
stream-clone: avoid opening a revlog in case we do not need it
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50671
diff
changeset
|
671 if size <= 65536: |
3b56395404a1
stream-clone: avoid opening a revlog in case we do not need it
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50671
diff
changeset
|
672 yield fp.read(size) |
3b56395404a1
stream-clone: avoid opening a revlog in case we do not need it
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50671
diff
changeset
|
673 else: |
3b56395404a1
stream-clone: avoid opening a revlog in case we do not need it
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50671
diff
changeset
|
674 yield from util.filechunkiter(fp, limit=size) |
3b56395404a1
stream-clone: avoid opening a revlog in case we do not need it
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50671
diff
changeset
|
675 finally: |
3b56395404a1
stream-clone: avoid opening a revlog in case we do not need it
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50671
diff
changeset
|
676 fp.close() |
3b56395404a1
stream-clone: avoid opening a revlog in case we do not need it
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50671
diff
changeset
|
677 |
3b56395404a1
stream-clone: avoid opening a revlog in case we do not need it
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50671
diff
changeset
|
678 s = get_stream() |
3b56395404a1
stream-clone: avoid opening a revlog in case we do not need it
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50671
diff
changeset
|
679 next(s) |
3b56395404a1
stream-clone: avoid opening a revlog in case we do not need it
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50671
diff
changeset
|
680 index_file = None |
3b56395404a1
stream-clone: avoid opening a revlog in case we do not need it
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50671
diff
changeset
|
681 stream.append((index_path, s, size)) |
3b56395404a1
stream-clone: avoid opening a revlog in case we do not need it
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50671
diff
changeset
|
682 else: |
3b56395404a1
stream-clone: avoid opening a revlog in case we do not need it
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50671
diff
changeset
|
683 rl = self.get_revlog_instance(repo).get_revlog() |
3b56395404a1
stream-clone: avoid opening a revlog in case we do not need it
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50671
diff
changeset
|
684 rl_stream = rl.get_streams(max_changeset, force_inline=True) |
3b56395404a1
stream-clone: avoid opening a revlog in case we do not need it
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50671
diff
changeset
|
685 for name, s, size in rl_stream: |
3b56395404a1
stream-clone: avoid opening a revlog in case we do not need it
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50671
diff
changeset
|
686 if name_to_size.get(name, 0) != size: |
3b56395404a1
stream-clone: avoid opening a revlog in case we do not need it
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50671
diff
changeset
|
687 msg = _(b"expected %d bytes but %d provided for %s") |
3b56395404a1
stream-clone: avoid opening a revlog in case we do not need it
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50671
diff
changeset
|
688 msg %= name_to_size.get(name, 0), size, name |
3b56395404a1
stream-clone: avoid opening a revlog in case we do not need it
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50671
diff
changeset
|
689 raise error.Abort(msg) |
3b56395404a1
stream-clone: avoid opening a revlog in case we do not need it
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50671
diff
changeset
|
690 stream.extend(rl_stream) |
3b56395404a1
stream-clone: avoid opening a revlog in case we do not need it
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50671
diff
changeset
|
691 finally: |
3b56395404a1
stream-clone: avoid opening a revlog in case we do not need it
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50671
diff
changeset
|
692 if index_file is not None: |
3b56395404a1
stream-clone: avoid opening a revlog in case we do not need it
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50671
diff
changeset
|
693 index_file.close() |
3b56395404a1
stream-clone: avoid opening a revlog in case we do not need it
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50671
diff
changeset
|
694 |
50640
9caa860dcbec
stream-clone: implement decidated `get_streams` method for revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50639
diff
changeset
|
695 files = self.files() |
9caa860dcbec
stream-clone: implement decidated `get_streams` method for revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50639
diff
changeset
|
696 assert len(stream) == len(files), ( |
9caa860dcbec
stream-clone: implement decidated `get_streams` method for revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50639
diff
changeset
|
697 stream, |
9caa860dcbec
stream-clone: implement decidated `get_streams` method for revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50639
diff
changeset
|
698 files, |
9caa860dcbec
stream-clone: implement decidated `get_streams` method for revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50639
diff
changeset
|
699 self._path_prefix, |
9caa860dcbec
stream-clone: implement decidated `get_streams` method for revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50639
diff
changeset
|
700 self.target_id, |
9caa860dcbec
stream-clone: implement decidated `get_streams` method for revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50639
diff
changeset
|
701 ) |
9caa860dcbec
stream-clone: implement decidated `get_streams` method for revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50639
diff
changeset
|
702 return stream |
9caa860dcbec
stream-clone: implement decidated `get_streams` method for revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50639
diff
changeset
|
703 |
50627
e1ee6910f6bc
store: add a `get_revlog_instance` method on revlog entries
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50514
diff
changeset
|
704 def get_revlog_instance(self, repo): |
e1ee6910f6bc
store: add a `get_revlog_instance` method on revlog entries
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50514
diff
changeset
|
705 """Obtain a revlog instance from this store entry |
e1ee6910f6bc
store: add a `get_revlog_instance` method on revlog entries
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50514
diff
changeset
|
706 |
e1ee6910f6bc
store: add a `get_revlog_instance` method on revlog entries
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50514
diff
changeset
|
707 An instance of the appropriate class is returned. |
e1ee6910f6bc
store: add a `get_revlog_instance` method on revlog entries
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50514
diff
changeset
|
708 """ |
e1ee6910f6bc
store: add a `get_revlog_instance` method on revlog entries
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50514
diff
changeset
|
709 if self.is_changelog: |
e1ee6910f6bc
store: add a `get_revlog_instance` method on revlog entries
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50514
diff
changeset
|
710 return changelog.changelog(repo.svfs) |
e1ee6910f6bc
store: add a `get_revlog_instance` method on revlog entries
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50514
diff
changeset
|
711 elif self.is_manifestlog: |
50628
3ea3767c23a4
store: do not drop the final `/` when creating manifestlog instance
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50627
diff
changeset
|
712 mandir = self.target_id |
50627
e1ee6910f6bc
store: add a `get_revlog_instance` method on revlog entries
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50514
diff
changeset
|
713 return manifest.manifestrevlog( |
e1ee6910f6bc
store: add a `get_revlog_instance` method on revlog entries
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50514
diff
changeset
|
714 repo.nodeconstants, repo.svfs, tree=mandir |
e1ee6910f6bc
store: add a `get_revlog_instance` method on revlog entries
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50514
diff
changeset
|
715 ) |
e1ee6910f6bc
store: add a `get_revlog_instance` method on revlog entries
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50514
diff
changeset
|
716 else: |
e1ee6910f6bc
store: add a `get_revlog_instance` method on revlog entries
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50514
diff
changeset
|
717 return filelog.filelog(repo.svfs, self.target_id) |
e1ee6910f6bc
store: add a `get_revlog_instance` method on revlog entries
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50514
diff
changeset
|
718 |
50473
5a2fb64d38b2
store: use specialized class for store entries
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50472
diff
changeset
|
719 |
50479
5217e36356bb
store: add logic to group revlog file together
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50478
diff
changeset
|
720 def _gather_revlog(files_data): |
5217e36356bb
store: add logic to group revlog file together
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50478
diff
changeset
|
721 """group files per revlog prefix |
5217e36356bb
store: add logic to group revlog file together
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50478
diff
changeset
|
722 |
5217e36356bb
store: add logic to group revlog file together
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50478
diff
changeset
|
723 The returns a two level nested dict. The top level key is the revlog prefix |
5217e36356bb
store: add logic to group revlog file together
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50478
diff
changeset
|
724 without extension, the second level is all the file "suffix" that were |
5217e36356bb
store: add logic to group revlog file together
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50478
diff
changeset
|
725 seen for this revlog and arbitrary file data as value. |
5217e36356bb
store: add logic to group revlog file together
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50478
diff
changeset
|
726 """ |
5217e36356bb
store: add logic to group revlog file together
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50478
diff
changeset
|
727 revlogs = collections.defaultdict(dict) |
5217e36356bb
store: add logic to group revlog file together
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50478
diff
changeset
|
728 for u, value in files_data: |
5217e36356bb
store: add logic to group revlog file together
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50478
diff
changeset
|
729 name, ext = _split_revlog_ext(u) |
5217e36356bb
store: add logic to group revlog file together
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50478
diff
changeset
|
730 revlogs[name][ext] = value |
5217e36356bb
store: add logic to group revlog file together
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50478
diff
changeset
|
731 return sorted(revlogs.items()) |
5217e36356bb
store: add logic to group revlog file together
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50478
diff
changeset
|
732 |
5217e36356bb
store: add logic to group revlog file together
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50478
diff
changeset
|
733 |
5217e36356bb
store: add logic to group revlog file together
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50478
diff
changeset
|
734 def _split_revlog_ext(filename): |
5217e36356bb
store: add logic to group revlog file together
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50478
diff
changeset
|
735 """split the revlog file prefix from the variable extension""" |
5217e36356bb
store: add logic to group revlog file together
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50478
diff
changeset
|
736 if filename.endswith(REVLOG_FILES_LONG_EXT): |
5217e36356bb
store: add logic to group revlog file together
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50478
diff
changeset
|
737 char = b'-' |
5217e36356bb
store: add logic to group revlog file together
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50478
diff
changeset
|
738 else: |
5217e36356bb
store: add logic to group revlog file together
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50478
diff
changeset
|
739 char = b'.' |
5217e36356bb
store: add logic to group revlog file together
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50478
diff
changeset
|
740 idx = filename.rfind(char) |
5217e36356bb
store: add logic to group revlog file together
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50478
diff
changeset
|
741 return filename[:idx], filename[idx:] |
5217e36356bb
store: add logic to group revlog file together
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50478
diff
changeset
|
742 |
5217e36356bb
store: add logic to group revlog file together
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50478
diff
changeset
|
743 |
50481
b08243dbc2e3
store: also gather files per revlog in `topfiles`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50480
diff
changeset
|
744 def _ext_key(ext): |
b08243dbc2e3
store: also gather files per revlog in `topfiles`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50480
diff
changeset
|
745 """a key to order revlog suffix |
b08243dbc2e3
store: also gather files per revlog in `topfiles`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50480
diff
changeset
|
746 |
b08243dbc2e3
store: also gather files per revlog in `topfiles`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50480
diff
changeset
|
747 important to issue .i after other entry.""" |
b08243dbc2e3
store: also gather files per revlog in `topfiles`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50480
diff
changeset
|
748 # the only important part of this order is to keep the `.i` last. |
b08243dbc2e3
store: also gather files per revlog in `topfiles`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50480
diff
changeset
|
749 if ext.endswith(b'.n'): |
b08243dbc2e3
store: also gather files per revlog in `topfiles`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50480
diff
changeset
|
750 return (0, ext) |
b08243dbc2e3
store: also gather files per revlog in `topfiles`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50480
diff
changeset
|
751 elif ext.endswith(b'.nd'): |
b08243dbc2e3
store: also gather files per revlog in `topfiles`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50480
diff
changeset
|
752 return (10, ext) |
b08243dbc2e3
store: also gather files per revlog in `topfiles`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50480
diff
changeset
|
753 elif ext.endswith(b'.d'): |
b08243dbc2e3
store: also gather files per revlog in `topfiles`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50480
diff
changeset
|
754 return (20, ext) |
b08243dbc2e3
store: also gather files per revlog in `topfiles`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50480
diff
changeset
|
755 elif ext.endswith(b'.i'): |
b08243dbc2e3
store: also gather files per revlog in `topfiles`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50480
diff
changeset
|
756 return (50, ext) |
b08243dbc2e3
store: also gather files per revlog in `topfiles`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50480
diff
changeset
|
757 else: |
b08243dbc2e3
store: also gather files per revlog in `topfiles`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50480
diff
changeset
|
758 return (40, ext) |
b08243dbc2e3
store: also gather files per revlog in `topfiles`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50480
diff
changeset
|
759 |
b08243dbc2e3
store: also gather files per revlog in `topfiles`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50480
diff
changeset
|
760 |
48946
642e31cb55f0
py3: use class X: instead of class X(object):
Gregory Szorc <gregory.szorc@gmail.com>
parents:
48941
diff
changeset
|
761 class basicstore: |
6840
80e51429cb9a
introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents:
6839
diff
changeset
|
762 '''base class for local repository stores''' |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
763 |
17651
3b49c28658f6
store: rename "openertype" argument to "vfstype"
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17649
diff
changeset
|
764 def __init__(self, path, vfstype): |
17724
bf4b72d8dd4d
store: initialize vfs field first to use it for initialization of others
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17722
diff
changeset
|
765 vfs = vfstype(path) |
bf4b72d8dd4d
store: initialize vfs field first to use it for initialization of others
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17722
diff
changeset
|
766 self.path = vfs.base |
17726
7cb7e17c23b2
store: invoke "os.stat()" for "createmode" initialization via vfs
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17725
diff
changeset
|
767 self.createmode = _calcmode(vfs) |
17652
2c6f7231becc
store: rename "op" variables to "vfs"
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17651
diff
changeset
|
768 vfs.createmode = self.createmode |
17728
004bd533880d
store: invoke "os.path.isdir()" via vfs
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17727
diff
changeset
|
769 self.rawvfs = vfs |
31234
9b7a2ef4f27c
vfs: use 'vfs' module directly in 'mercurial.store'
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents:
31219
diff
changeset
|
770 self.vfs = vfsmod.filtervfs(vfs, encodedir) |
17653
dacb50696b75
store: initialize "vfs" fields by "vfs" constructors
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17652
diff
changeset
|
771 self.opener = self.vfs |
6840
80e51429cb9a
introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents:
6839
diff
changeset
|
772 |
80e51429cb9a
introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents:
6839
diff
changeset
|
773 def join(self, f): |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
774 return self.path + b'/' + encodedir(f) |
6840
80e51429cb9a
introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents:
6839
diff
changeset
|
775 |
50493
816e8bc6e066
store: split the wrapping of encodedstore between _wrap and datafiles
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50492
diff
changeset
|
776 def _walk(self, relpath, recurse, undecodable=None): |
47877
2174f54aab18
store: return just one filename in walk functions
Valentin Gatien-Baron <valentin.gatienbaron@gmail.com>
parents:
47785
diff
changeset
|
777 '''yields (revlog_type, unencoded, size)''' |
13426
643b8212813e
store: remove pointless pathjoiner parameter
Adrian Buehlmann <adrian@cadifra.com>
parents:
13391
diff
changeset
|
778 path = self.path |
643b8212813e
store: remove pointless pathjoiner parameter
Adrian Buehlmann <adrian@cadifra.com>
parents:
13391
diff
changeset
|
779 if relpath: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
780 path += b'/' + relpath |
13426
643b8212813e
store: remove pointless pathjoiner parameter
Adrian Buehlmann <adrian@cadifra.com>
parents:
13391
diff
changeset
|
781 striplen = len(self.path) + 1 |
6899 | 782 l = [] |
17728
004bd533880d
store: invoke "os.path.isdir()" via vfs
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17727
diff
changeset
|
783 if self.rawvfs.isdir(path): |
6899 | 784 visit = [path] |
17747
aad3bce98f76
store: invoke "osutil.listdir()" via vfs
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17745
diff
changeset
|
785 readdir = self.rawvfs.readdir |
6899 | 786 while visit: |
787 p = visit.pop() | |
17747
aad3bce98f76
store: invoke "osutil.listdir()" via vfs
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17745
diff
changeset
|
788 for f, kind, st in readdir(p, stat=True): |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
789 fp = p + b'/' + f |
50671
e06d1a779eb6
store: stop relying on a `revlog_type` property
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50670
diff
changeset
|
790 if is_revlog(f, kind, st): |
6900
def492d1b592
store: change handling of decoding errors
Matt Mackall <mpm@selenic.com>
parents:
6899
diff
changeset
|
791 n = util.pconvert(fp[striplen:]) |
50671
e06d1a779eb6
store: stop relying on a `revlog_type` property
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50670
diff
changeset
|
792 l.append((decodedir(n), st.st_size)) |
6899 | 793 elif kind == stat.S_IFDIR and recurse: |
794 visit.append(fp) | |
50478
1c0244a8cdaf
store: change `_walk` return to `(filename, (type, size))`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50477
diff
changeset
|
795 |
17054
125ff5654b72
store: sort filenames in place
Bryan O'Sullivan <bryano@fb.com>
parents:
16404
diff
changeset
|
796 l.sort() |
125ff5654b72
store: sort filenames in place
Bryan O'Sullivan <bryano@fb.com>
parents:
16404
diff
changeset
|
797 return l |
6840
80e51429cb9a
introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents:
6839
diff
changeset
|
798 |
46607
e9901d01d135
revlog: add a mechanism to verify expected file position before appending
Kyle Lippincott <spectral@google.com>
parents:
46383
diff
changeset
|
799 def changelog(self, trypending, concurrencychecker=None): |
e9901d01d135
revlog: add a mechanism to verify expected file position before appending
Kyle Lippincott <spectral@google.com>
parents:
46383
diff
changeset
|
800 return changelog.changelog( |
e9901d01d135
revlog: add a mechanism to verify expected file position before appending
Kyle Lippincott <spectral@google.com>
parents:
46383
diff
changeset
|
801 self.vfs, |
e9901d01d135
revlog: add a mechanism to verify expected file position before appending
Kyle Lippincott <spectral@google.com>
parents:
46383
diff
changeset
|
802 trypending=trypending, |
e9901d01d135
revlog: add a mechanism to verify expected file position before appending
Kyle Lippincott <spectral@google.com>
parents:
46383
diff
changeset
|
803 concurrencychecker=concurrencychecker, |
e9901d01d135
revlog: add a mechanism to verify expected file position before appending
Kyle Lippincott <spectral@google.com>
parents:
46383
diff
changeset
|
804 ) |
42905
3df3b139a43d
localrepo: push manifestlog and changelog construction code into store
Augie Fackler <augie@google.com>
parents:
42715
diff
changeset
|
805 |
3df3b139a43d
localrepo: push manifestlog and changelog construction code into store
Augie Fackler <augie@google.com>
parents:
42715
diff
changeset
|
806 def manifestlog(self, repo, storenarrowmatch): |
46780
6266d19556ad
node: introduce nodeconstants class
Joerg Sonnenberger <joerg@bec.de>
parents:
46607
diff
changeset
|
807 rootstore = manifest.manifestrevlog(repo.nodeconstants, self.vfs) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
808 return manifest.manifestlog(self.vfs, repo, rootstore, storenarrowmatch) |
42905
3df3b139a43d
localrepo: push manifestlog and changelog construction code into store
Augie Fackler <augie@google.com>
parents:
42715
diff
changeset
|
809 |
50504
862e3a13da44
store: rename `datafiles` to `data_entries`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50498
diff
changeset
|
810 def data_entries( |
50471
521fec115dad
store: use a StoreEntry object instead of tuple for store files
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50469
diff
changeset
|
811 self, matcher=None, undecodable=None |
50473
5a2fb64d38b2
store: use specialized class for store entries
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50472
diff
changeset
|
812 ) -> Generator[BaseStoreEntry, None, None]: |
47877
2174f54aab18
store: return just one filename in walk functions
Valentin Gatien-Baron <valentin.gatienbaron@gmail.com>
parents:
47785
diff
changeset
|
813 """Like walk, but excluding the changelog and root manifest. |
2174f54aab18
store: return just one filename in walk functions
Valentin Gatien-Baron <valentin.gatienbaron@gmail.com>
parents:
47785
diff
changeset
|
814 |
2174f54aab18
store: return just one filename in walk functions
Valentin Gatien-Baron <valentin.gatienbaron@gmail.com>
parents:
47785
diff
changeset
|
815 When [undecodable] is None, revlogs names that can't be |
2174f54aab18
store: return just one filename in walk functions
Valentin Gatien-Baron <valentin.gatienbaron@gmail.com>
parents:
47785
diff
changeset
|
816 decoded cause an exception. When it is provided, it should |
2174f54aab18
store: return just one filename in walk functions
Valentin Gatien-Baron <valentin.gatienbaron@gmail.com>
parents:
47785
diff
changeset
|
817 be a list and the filenames that can't be decoded are added |
2174f54aab18
store: return just one filename in walk functions
Valentin Gatien-Baron <valentin.gatienbaron@gmail.com>
parents:
47785
diff
changeset
|
818 to it instead. This is very rarely needed.""" |
50482
1fc25227b068
store: actually tag tree manifest revlogs as manifest revlogs
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50481
diff
changeset
|
819 dirs = [ |
50680
e324329fc97e
store: use the revlog type from revlog's constant instead of custom one
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50672
diff
changeset
|
820 (b'data', KIND_FILELOG, False), |
e324329fc97e
store: use the revlog type from revlog's constant instead of custom one
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50672
diff
changeset
|
821 (b'meta', KIND_MANIFESTLOG, True), |
50482
1fc25227b068
store: actually tag tree manifest revlogs as manifest revlogs
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50481
diff
changeset
|
822 ] |
50629
309cbd8400ae
store: properly compute the targer_id of manifestlog in no-fncache walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50628
diff
changeset
|
823 for base_dir, rl_type, strip_filename in dirs: |
50493
816e8bc6e066
store: split the wrapping of encodedstore between _wrap and datafiles
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50492
diff
changeset
|
824 files = self._walk(base_dir, True, undecodable=undecodable) |
50482
1fc25227b068
store: actually tag tree manifest revlogs as manifest revlogs
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50481
diff
changeset
|
825 for revlog, details in _gather_revlog(files): |
50496
e50d1fe7ebb4
store: issue a single entry for each revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50495
diff
changeset
|
826 revlog_target_id = revlog.split(b'/', 1)[1] |
50629
309cbd8400ae
store: properly compute the targer_id of manifestlog in no-fncache walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50628
diff
changeset
|
827 if strip_filename and b'/' in revlog: |
309cbd8400ae
store: properly compute the targer_id of manifestlog in no-fncache walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50628
diff
changeset
|
828 revlog_target_id = revlog_target_id.rsplit(b'/', 1)[0] |
309cbd8400ae
store: properly compute the targer_id of manifestlog in no-fncache walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50628
diff
changeset
|
829 revlog_target_id += b'/' |
50496
e50d1fe7ebb4
store: issue a single entry for each revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50495
diff
changeset
|
830 yield RevlogStoreEntry( |
e50d1fe7ebb4
store: issue a single entry for each revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50495
diff
changeset
|
831 path_prefix=revlog, |
e50d1fe7ebb4
store: issue a single entry for each revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50495
diff
changeset
|
832 revlog_type=rl_type, |
e50d1fe7ebb4
store: issue a single entry for each revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50495
diff
changeset
|
833 target_id=revlog_target_id, |
50671
e06d1a779eb6
store: stop relying on a `revlog_type` property
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50670
diff
changeset
|
834 details=details, |
50496
e50d1fe7ebb4
store: issue a single entry for each revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50495
diff
changeset
|
835 ) |
6840
80e51429cb9a
introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents:
6839
diff
changeset
|
836 |
50514
0925eaf09c8b
store: make `walk` return an entry for obsolescence if requested so
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50513
diff
changeset
|
837 def top_entries( |
0925eaf09c8b
store: make `walk` return an entry for obsolescence if requested so
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50513
diff
changeset
|
838 self, phase=False, obsolescence=False |
0925eaf09c8b
store: make `walk` return an entry for obsolescence if requested so
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50513
diff
changeset
|
839 ) -> Generator[BaseStoreEntry, None, None]: |
50513
5a62d56e3955
store: yield phases before changelog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50512
diff
changeset
|
840 if phase and self.vfs.exists(b'phaseroots'): |
5a62d56e3955
store: yield phases before changelog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50512
diff
changeset
|
841 yield SimpleStoreEntry( |
5a62d56e3955
store: yield phases before changelog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50512
diff
changeset
|
842 entry_path=b'phaseroots', |
5a62d56e3955
store: yield phases before changelog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50512
diff
changeset
|
843 is_volatile=True, |
5a62d56e3955
store: yield phases before changelog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50512
diff
changeset
|
844 ) |
5a62d56e3955
store: yield phases before changelog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50512
diff
changeset
|
845 |
50514
0925eaf09c8b
store: make `walk` return an entry for obsolescence if requested so
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50513
diff
changeset
|
846 if obsolescence and self.vfs.exists(b'obsstore'): |
0925eaf09c8b
store: make `walk` return an entry for obsolescence if requested so
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50513
diff
changeset
|
847 # XXX if we had the file size it could be non-volatile |
0925eaf09c8b
store: make `walk` return an entry for obsolescence if requested so
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50513
diff
changeset
|
848 yield SimpleStoreEntry( |
0925eaf09c8b
store: make `walk` return an entry for obsolescence if requested so
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50513
diff
changeset
|
849 entry_path=b'obsstore', |
0925eaf09c8b
store: make `walk` return an entry for obsolescence if requested so
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50513
diff
changeset
|
850 is_volatile=True, |
0925eaf09c8b
store: make `walk` return an entry for obsolescence if requested so
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50513
diff
changeset
|
851 ) |
0925eaf09c8b
store: make `walk` return an entry for obsolescence if requested so
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50513
diff
changeset
|
852 |
50481
b08243dbc2e3
store: also gather files per revlog in `topfiles`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50480
diff
changeset
|
853 files = reversed(self._walk(b'', False)) |
b08243dbc2e3
store: also gather files per revlog in `topfiles`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50480
diff
changeset
|
854 |
b08243dbc2e3
store: also gather files per revlog in `topfiles`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50480
diff
changeset
|
855 changelogs = collections.defaultdict(dict) |
b08243dbc2e3
store: also gather files per revlog in `topfiles`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50480
diff
changeset
|
856 manifestlogs = collections.defaultdict(dict) |
b08243dbc2e3
store: also gather files per revlog in `topfiles`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50480
diff
changeset
|
857 |
50671
e06d1a779eb6
store: stop relying on a `revlog_type` property
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50670
diff
changeset
|
858 for u, s in files: |
46895
6085b7f1536d
store: also return some information about the type of file `walk` found
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46854
diff
changeset
|
859 if u.startswith(b'00changelog'): |
50481
b08243dbc2e3
store: also gather files per revlog in `topfiles`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50480
diff
changeset
|
860 name, ext = _split_revlog_ext(u) |
50671
e06d1a779eb6
store: stop relying on a `revlog_type` property
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50670
diff
changeset
|
861 changelogs[name][ext] = s |
46895
6085b7f1536d
store: also return some information about the type of file `walk` found
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46854
diff
changeset
|
862 elif u.startswith(b'00manifest'): |
50481
b08243dbc2e3
store: also gather files per revlog in `topfiles`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50480
diff
changeset
|
863 name, ext = _split_revlog_ext(u) |
50671
e06d1a779eb6
store: stop relying on a `revlog_type` property
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50670
diff
changeset
|
864 manifestlogs[name][ext] = s |
46895
6085b7f1536d
store: also return some information about the type of file `walk` found
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46854
diff
changeset
|
865 else: |
50473
5a2fb64d38b2
store: use specialized class for store entries
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50472
diff
changeset
|
866 yield SimpleStoreEntry( |
50495
ed8cda1c18e1
store: rename `unencoded_path` to `entry_path` for StoreEntry
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50494
diff
changeset
|
867 entry_path=u, |
50671
e06d1a779eb6
store: stop relying on a `revlog_type` property
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50670
diff
changeset
|
868 is_volatile=False, |
50473
5a2fb64d38b2
store: use specialized class for store entries
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50472
diff
changeset
|
869 file_size=s, |
5a2fb64d38b2
store: use specialized class for store entries
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50472
diff
changeset
|
870 ) |
50481
b08243dbc2e3
store: also gather files per revlog in `topfiles`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50480
diff
changeset
|
871 # yield manifest before changelog |
b08243dbc2e3
store: also gather files per revlog in `topfiles`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50480
diff
changeset
|
872 top_rl = [ |
50680
e324329fc97e
store: use the revlog type from revlog's constant instead of custom one
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50672
diff
changeset
|
873 (manifestlogs, KIND_MANIFESTLOG), |
e324329fc97e
store: use the revlog type from revlog's constant instead of custom one
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50672
diff
changeset
|
874 (changelogs, KIND_CHANGELOG), |
50481
b08243dbc2e3
store: also gather files per revlog in `topfiles`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50480
diff
changeset
|
875 ] |
b08243dbc2e3
store: also gather files per revlog in `topfiles`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50480
diff
changeset
|
876 assert len(manifestlogs) <= 1 |
b08243dbc2e3
store: also gather files per revlog in `topfiles`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50480
diff
changeset
|
877 assert len(changelogs) <= 1 |
b08243dbc2e3
store: also gather files per revlog in `topfiles`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50480
diff
changeset
|
878 for data, revlog_type in top_rl: |
b08243dbc2e3
store: also gather files per revlog in `topfiles`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50480
diff
changeset
|
879 for revlog, details in sorted(data.items()): |
50496
e50d1fe7ebb4
store: issue a single entry for each revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50495
diff
changeset
|
880 yield RevlogStoreEntry( |
e50d1fe7ebb4
store: issue a single entry for each revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50495
diff
changeset
|
881 path_prefix=revlog, |
e50d1fe7ebb4
store: issue a single entry for each revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50495
diff
changeset
|
882 revlog_type=revlog_type, |
e50d1fe7ebb4
store: issue a single entry for each revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50495
diff
changeset
|
883 target_id=b'', |
50671
e06d1a779eb6
store: stop relying on a `revlog_type` property
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50670
diff
changeset
|
884 details=details, |
50496
e50d1fe7ebb4
store: issue a single entry for each revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50495
diff
changeset
|
885 ) |
19177
1e104aaa4c44
store: move top file walk to a separate function
Durham Goode <durham@fb.com>
parents:
18435
diff
changeset
|
886 |
50512
a32d739b0ffb
store: make `walk` return an entry for phase if requested so
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50505
diff
changeset
|
887 def walk( |
50514
0925eaf09c8b
store: make `walk` return an entry for obsolescence if requested so
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50513
diff
changeset
|
888 self, matcher=None, phase=False, obsolescence=False |
50512
a32d739b0ffb
store: make `walk` return an entry for phase if requested so
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50505
diff
changeset
|
889 ) -> Generator[BaseStoreEntry, None, None]: |
50451
584ff1f97201
comments: fix spelling
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50319
diff
changeset
|
890 """return files related to data storage (ie: revlogs) |
46853
eed3e2b79b48
store: document the `walk` method
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
46819
diff
changeset
|
891 |
50512
a32d739b0ffb
store: make `walk` return an entry for phase if requested so
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50505
diff
changeset
|
892 yields instance from BaseStoreEntry subclasses |
40340
2d45b549392f
store: pass matcher to store.datafiles()
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
39698
diff
changeset
|
893 |
2d45b549392f
store: pass matcher to store.datafiles()
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
39698
diff
changeset
|
894 if a matcher is passed, storage files of only those tracked paths |
2d45b549392f
store: pass matcher to store.datafiles()
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
39698
diff
changeset
|
895 are passed with matches the matcher |
45942
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
45483
diff
changeset
|
896 """ |
6840
80e51429cb9a
introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents:
6839
diff
changeset
|
897 # yield data files first |
50504
862e3a13da44
store: rename `datafiles` to `data_entries`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50498
diff
changeset
|
898 for x in self.data_entries(matcher): |
6840
80e51429cb9a
introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents:
6839
diff
changeset
|
899 yield x |
50514
0925eaf09c8b
store: make `walk` return an entry for obsolescence if requested so
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50513
diff
changeset
|
900 for x in self.top_entries(phase=phase, obsolescence=obsolescence): |
6840
80e51429cb9a
introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents:
6839
diff
changeset
|
901 yield x |
80e51429cb9a
introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents:
6839
diff
changeset
|
902 |
6903
0642d9d7ec80
clone: get a list of files to clone from store
Matt Mackall <mpm@selenic.com>
parents:
6902
diff
changeset
|
903 def copylist(self): |
45483
d252f51ab032
share: introduce config option to store requires in .hg/store
Pulkit Goyal <7895pulkit@gmail.com>
parents:
45351
diff
changeset
|
904 return _data |
6903
0642d9d7ec80
clone: get a list of files to clone from store
Matt Mackall <mpm@selenic.com>
parents:
6902
diff
changeset
|
905 |
20883
cd443c7589cc
fncache: move fncache writing to be in a transaction
Durham Goode <durham@fb.com>
parents:
20879
diff
changeset
|
906 def write(self, tr): |
13391
d00bbff8600e
fncachestore: defer updating the fncache file to a single file open
Adrian Buehlmann <adrian@cadifra.com>
parents:
13169
diff
changeset
|
907 pass |
d00bbff8600e
fncachestore: defer updating the fncache file to a single file open
Adrian Buehlmann <adrian@cadifra.com>
parents:
13169
diff
changeset
|
908 |
20884
2efdd186925d
caches: invalidate store caches when lock is taken
Durham Goode <durham@fb.com>
parents:
20883
diff
changeset
|
909 def invalidatecaches(self): |
2efdd186925d
caches: invalidate store caches when lock is taken
Durham Goode <durham@fb.com>
parents:
20883
diff
changeset
|
910 pass |
2efdd186925d
caches: invalidate store caches when lock is taken
Durham Goode <durham@fb.com>
parents:
20883
diff
changeset
|
911 |
20885
f49d60fa40a5
fncache: clean up fncache during strips
Durham Goode <durham@fb.com>
parents:
20884
diff
changeset
|
912 def markremoved(self, fn): |
f49d60fa40a5
fncache: clean up fncache during strips
Durham Goode <durham@fb.com>
parents:
20884
diff
changeset
|
913 pass |
f49d60fa40a5
fncache: clean up fncache during strips
Durham Goode <durham@fb.com>
parents:
20884
diff
changeset
|
914 |
17744 | 915 def __contains__(self, path): |
916 '''Checks if the store contains path''' | |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
917 path = b"/".join((b"data", path)) |
17744 | 918 # file? |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
919 if self.vfs.exists(path + b".i"): |
17744 | 920 return True |
921 # dir? | |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
922 if not path.endswith(b"/"): |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
923 path = path + b"/" |
19903
ca875b271ac3
store: use "vfs.exists()" instead of "os.path.exists()"
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
19177
diff
changeset
|
924 return self.vfs.exists(path) |
17744 | 925 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
926 |
6898
69aeaaaf6e07
store: simplify class hierarchy
Matt Mackall <mpm@selenic.com>
parents:
6897
diff
changeset
|
927 class encodedstore(basicstore): |
17651
3b49c28658f6
store: rename "openertype" argument to "vfstype"
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17649
diff
changeset
|
928 def __init__(self, path, vfstype): |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
929 vfs = vfstype(path + b'/store') |
17724
bf4b72d8dd4d
store: initialize vfs field first to use it for initialization of others
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17722
diff
changeset
|
930 self.path = vfs.base |
17726
7cb7e17c23b2
store: invoke "os.stat()" for "createmode" initialization via vfs
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17725
diff
changeset
|
931 self.createmode = _calcmode(vfs) |
17652
2c6f7231becc
store: rename "op" variables to "vfs"
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17651
diff
changeset
|
932 vfs.createmode = self.createmode |
17728
004bd533880d
store: invoke "os.path.isdir()" via vfs
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17727
diff
changeset
|
933 self.rawvfs = vfs |
31234
9b7a2ef4f27c
vfs: use 'vfs' module directly in 'mercurial.store'
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents:
31219
diff
changeset
|
934 self.vfs = vfsmod.filtervfs(vfs, encodefilename) |
17653
dacb50696b75
store: initialize "vfs" fields by "vfs" constructors
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17652
diff
changeset
|
935 self.opener = self.vfs |
6840
80e51429cb9a
introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents:
6839
diff
changeset
|
936 |
50493
816e8bc6e066
store: split the wrapping of encodedstore between _wrap and datafiles
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50492
diff
changeset
|
937 def _walk(self, relpath, recurse, undecodable=None): |
816e8bc6e066
store: split the wrapping of encodedstore between _wrap and datafiles
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50492
diff
changeset
|
938 old = super()._walk(relpath, recurse) |
816e8bc6e066
store: split the wrapping of encodedstore between _wrap and datafiles
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50492
diff
changeset
|
939 new = [] |
816e8bc6e066
store: split the wrapping of encodedstore between _wrap and datafiles
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50492
diff
changeset
|
940 for f1, value in old: |
6892
dab95717058d
verify: check repo.store
Adrian Buehlmann <adrian@cadifra.com>
parents:
6890
diff
changeset
|
941 try: |
47877
2174f54aab18
store: return just one filename in walk functions
Valentin Gatien-Baron <valentin.gatienbaron@gmail.com>
parents:
47785
diff
changeset
|
942 f2 = decodefilename(f1) |
6892
dab95717058d
verify: check repo.store
Adrian Buehlmann <adrian@cadifra.com>
parents:
6890
diff
changeset
|
943 except KeyError: |
47877
2174f54aab18
store: return just one filename in walk functions
Valentin Gatien-Baron <valentin.gatienbaron@gmail.com>
parents:
47785
diff
changeset
|
944 if undecodable is None: |
2174f54aab18
store: return just one filename in walk functions
Valentin Gatien-Baron <valentin.gatienbaron@gmail.com>
parents:
47785
diff
changeset
|
945 msg = _(b'undecodable revlog name %s') % f1 |
2174f54aab18
store: return just one filename in walk functions
Valentin Gatien-Baron <valentin.gatienbaron@gmail.com>
parents:
47785
diff
changeset
|
946 raise error.StorageError(msg) |
2174f54aab18
store: return just one filename in walk functions
Valentin Gatien-Baron <valentin.gatienbaron@gmail.com>
parents:
47785
diff
changeset
|
947 else: |
2174f54aab18
store: return just one filename in walk functions
Valentin Gatien-Baron <valentin.gatienbaron@gmail.com>
parents:
47785
diff
changeset
|
948 undecodable.append(f1) |
2174f54aab18
store: return just one filename in walk functions
Valentin Gatien-Baron <valentin.gatienbaron@gmail.com>
parents:
47785
diff
changeset
|
949 continue |
50493
816e8bc6e066
store: split the wrapping of encodedstore between _wrap and datafiles
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50492
diff
changeset
|
950 new.append((f2, value)) |
816e8bc6e066
store: split the wrapping of encodedstore between _wrap and datafiles
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50492
diff
changeset
|
951 return new |
816e8bc6e066
store: split the wrapping of encodedstore between _wrap and datafiles
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50492
diff
changeset
|
952 |
50504
862e3a13da44
store: rename `datafiles` to `data_entries`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50498
diff
changeset
|
953 def data_entries( |
50493
816e8bc6e066
store: split the wrapping of encodedstore between _wrap and datafiles
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50492
diff
changeset
|
954 self, matcher=None, undecodable=None |
816e8bc6e066
store: split the wrapping of encodedstore between _wrap and datafiles
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50492
diff
changeset
|
955 ) -> Generator[BaseStoreEntry, None, None]: |
50504
862e3a13da44
store: rename `datafiles` to `data_entries`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50498
diff
changeset
|
956 entries = super(encodedstore, self).data_entries( |
862e3a13da44
store: rename `datafiles` to `data_entries`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50498
diff
changeset
|
957 undecodable=undecodable |
862e3a13da44
store: rename `datafiles` to `data_entries`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50498
diff
changeset
|
958 ) |
50493
816e8bc6e066
store: split the wrapping of encodedstore between _wrap and datafiles
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50492
diff
changeset
|
959 for entry in entries: |
50494
b4953fad744e
store: do the revlog matching on entry directly
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50493
diff
changeset
|
960 if _match_tracked_entry(entry, matcher): |
50493
816e8bc6e066
store: split the wrapping of encodedstore between _wrap and datafiles
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50492
diff
changeset
|
961 yield entry |
6840
80e51429cb9a
introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents:
6839
diff
changeset
|
962 |
80e51429cb9a
introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents:
6839
diff
changeset
|
963 def join(self, f): |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
964 return self.path + b'/' + encodefilename(f) |
6840
80e51429cb9a
introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents:
6839
diff
changeset
|
965 |
6903
0642d9d7ec80
clone: get a list of files to clone from store
Matt Mackall <mpm@selenic.com>
parents:
6902
diff
changeset
|
966 def copylist(self): |
45351
909dafff6a78
store: refactor space delimited list to proper data structure
Pulkit Goyal <7895pulkit@gmail.com>
parents:
44452
diff
changeset
|
967 return [b'requires', b'00changelog.i'] + [b'store/' + f for f in _data] |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
968 |
6903
0642d9d7ec80
clone: get a list of files to clone from store
Matt Mackall <mpm@selenic.com>
parents:
6902
diff
changeset
|
969 |
48946
642e31cb55f0
py3: use class X: instead of class X(object):
Gregory Szorc <gregory.szorc@gmail.com>
parents:
48941
diff
changeset
|
970 class fncache: |
8531
810387f59696
filelog encoding: move the encoding/decoding into store
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8530
diff
changeset
|
971 # the filename used to be partially encoded |
810387f59696
filelog encoding: move the encoding/decoding into store
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8530
diff
changeset
|
972 # hence the encodedir/decodedir dance |
17722
3b976051034d
store: rename field name from "opener" to "vfs" in internal classes for fncache
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17721
diff
changeset
|
973 def __init__(self, vfs): |
3b976051034d
store: rename field name from "opener" to "vfs" in internal classes for fncache
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17721
diff
changeset
|
974 self.vfs = vfs |
50315
cf6e1d535602
fncache: make it possible to ignore some file
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
49306
diff
changeset
|
975 self._ignores = set() |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
976 self.entries = None |
13391
d00bbff8600e
fncachestore: defer updating the fncache file to a single file open
Adrian Buehlmann <adrian@cadifra.com>
parents:
13169
diff
changeset
|
977 self._dirty = False |
40736
0728d87a8631
store: append to fncache if there are only new files to write
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40624
diff
changeset
|
978 # set of new additions to fncache |
0728d87a8631
store: append to fncache if there are only new files to write
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40624
diff
changeset
|
979 self.addls = set() |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
980 |
42715
f59f8a5e9096
fncache: make debugrebuildfncache not fail on broken fncache
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents:
42341
diff
changeset
|
981 def ensureloaded(self, warn=None): |
45942
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
45483
diff
changeset
|
982 """read the fncache file if not already read. |
42715
f59f8a5e9096
fncache: make debugrebuildfncache not fail on broken fncache
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents:
42341
diff
changeset
|
983 |
f59f8a5e9096
fncache: make debugrebuildfncache not fail on broken fncache
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents:
42341
diff
changeset
|
984 If the file on disk is corrupted, raise. If warn is provided, |
45942
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
45483
diff
changeset
|
985 warn and keep going instead.""" |
42715
f59f8a5e9096
fncache: make debugrebuildfncache not fail on broken fncache
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents:
42341
diff
changeset
|
986 if self.entries is None: |
f59f8a5e9096
fncache: make debugrebuildfncache not fail on broken fncache
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents:
42341
diff
changeset
|
987 self._load(warn) |
f59f8a5e9096
fncache: make debugrebuildfncache not fail on broken fncache
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents:
42341
diff
changeset
|
988 |
f59f8a5e9096
fncache: make debugrebuildfncache not fail on broken fncache
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents:
42341
diff
changeset
|
989 def _load(self, warn=None): |
8530
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
990 '''fill the entries from the fncache file''' |
13391
d00bbff8600e
fncachestore: defer updating the fncache file to a single file open
Adrian Buehlmann <adrian@cadifra.com>
parents:
13169
diff
changeset
|
991 self._dirty = False |
8530
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
992 try: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
993 fp = self.vfs(b'fncache', mode=b'rb') |
8530
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
994 except IOError: |
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
995 # skip nonexistent file |
16404
9fca5b056c0a
store: speed up read and write of large fncache files
Bryan O'Sullivan <bryano@fb.com>
parents:
15742
diff
changeset
|
996 self.entries = set() |
8530
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
997 return |
41978
a56487081109
store: don't read the whole fncache in memory
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
41973
diff
changeset
|
998 |
a56487081109
store: don't read the whole fncache in memory
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
41973
diff
changeset
|
999 self.entries = set() |
a56487081109
store: don't read the whole fncache in memory
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
41973
diff
changeset
|
1000 chunk = b'' |
a56487081109
store: don't read the whole fncache in memory
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
41973
diff
changeset
|
1001 for c in iter(functools.partial(fp.read, fncache_chunksize), b''): |
a56487081109
store: don't read the whole fncache in memory
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
41973
diff
changeset
|
1002 chunk += c |
a56487081109
store: don't read the whole fncache in memory
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
41973
diff
changeset
|
1003 try: |
a56487081109
store: don't read the whole fncache in memory
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
41973
diff
changeset
|
1004 p = chunk.rindex(b'\n') |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
1005 self.entries.update(decodedir(chunk[: p + 1]).splitlines()) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
1006 chunk = chunk[p + 1 :] |
41978
a56487081109
store: don't read the whole fncache in memory
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
41973
diff
changeset
|
1007 except ValueError: |
a56487081109
store: don't read the whole fncache in memory
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
41973
diff
changeset
|
1008 # substring '\n' not found, maybe the entry is bigger than the |
a56487081109
store: don't read the whole fncache in memory
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
41973
diff
changeset
|
1009 # chunksize, so let's keep iterating |
a56487081109
store: don't read the whole fncache in memory
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
41973
diff
changeset
|
1010 pass |
a56487081109
store: don't read the whole fncache in memory
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
41973
diff
changeset
|
1011 |
41981
a920a9e1795a
store: error out if fncache does not ends with a newline
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
41978
diff
changeset
|
1012 if chunk: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
1013 msg = _(b"fncache does not ends with a newline") |
42715
f59f8a5e9096
fncache: make debugrebuildfncache not fail on broken fncache
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents:
42341
diff
changeset
|
1014 if warn: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
1015 warn(msg + b'\n') |
42715
f59f8a5e9096
fncache: make debugrebuildfncache not fail on broken fncache
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents:
42341
diff
changeset
|
1016 else: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
1017 raise error.Abort( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
1018 msg, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
1019 hint=_( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
1020 b"use 'hg debugrebuildfncache' to " |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
1021 b"rebuild the fncache" |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
1022 ), |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
1023 ) |
42715
f59f8a5e9096
fncache: make debugrebuildfncache not fail on broken fncache
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents:
42341
diff
changeset
|
1024 self._checkentries(fp, warn) |
41973
d7ef84e595f8
store: move logic to check for invalid entry in fncache to own function
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
41093
diff
changeset
|
1025 fp.close() |
d7ef84e595f8
store: move logic to check for invalid entry in fncache to own function
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
41093
diff
changeset
|
1026 |
42715
f59f8a5e9096
fncache: make debugrebuildfncache not fail on broken fncache
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents:
42341
diff
changeset
|
1027 def _checkentries(self, fp, warn): |
47062
f38bf44e077f
black: make codebase compatible with black v21.4b2 and v20.8b1
Kyle Lippincott <spectral@google.com>
parents:
46990
diff
changeset
|
1028 """make sure there is no empty string in entries""" |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
1029 if b'' in self.entries: |
16404
9fca5b056c0a
store: speed up read and write of large fncache files
Bryan O'Sullivan <bryano@fb.com>
parents:
15742
diff
changeset
|
1030 fp.seek(0) |
48941
fd5b8e696b75
py3: stop using util.iterfile()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
48913
diff
changeset
|
1031 for n, line in enumerate(fp): |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
1032 if not line.rstrip(b'\n'): |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
1033 t = _(b'invalid entry in fncache, line %d') % (n + 1) |
42715
f59f8a5e9096
fncache: make debugrebuildfncache not fail on broken fncache
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents:
42341
diff
changeset
|
1034 if warn: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
1035 warn(t + b'\n') |
42715
f59f8a5e9096
fncache: make debugrebuildfncache not fail on broken fncache
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents:
42341
diff
changeset
|
1036 else: |
f59f8a5e9096
fncache: make debugrebuildfncache not fail on broken fncache
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents:
42341
diff
changeset
|
1037 raise error.Abort(t) |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
1038 |
20883
cd443c7589cc
fncache: move fncache writing to be in a transaction
Durham Goode <durham@fb.com>
parents:
20879
diff
changeset
|
1039 def write(self, tr): |
16404
9fca5b056c0a
store: speed up read and write of large fncache files
Bryan O'Sullivan <bryano@fb.com>
parents:
15742
diff
changeset
|
1040 if self._dirty: |
38696
89d93dd1a222
store: assert the fncache have been loaded if dirty
Boris Feld <boris.feld@octobus.net>
parents:
38661
diff
changeset
|
1041 assert self.entries is not None |
40748
df8ed31a8ad8
store: write fncache only once if there are both adds and removes
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40736
diff
changeset
|
1042 self.entries = self.entries | self.addls |
df8ed31a8ad8
store: write fncache only once if there are both adds and removes
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40736
diff
changeset
|
1043 self.addls = set() |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
1044 tr.addbackup(b'fncache') |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
1045 fp = self.vfs(b'fncache', mode=b'wb', atomictemp=True) |
20879
cd03854a2e06
fncache: remove the rewriting logic
Durham Goode <durham@fb.com>
parents:
19903
diff
changeset
|
1046 if self.entries: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
1047 fp.write(encodedir(b'\n'.join(self.entries) + b'\n')) |
20879
cd03854a2e06
fncache: remove the rewriting logic
Durham Goode <durham@fb.com>
parents:
19903
diff
changeset
|
1048 fp.close() |
cd03854a2e06
fncache: remove the rewriting logic
Durham Goode <durham@fb.com>
parents:
19903
diff
changeset
|
1049 self._dirty = False |
40736
0728d87a8631
store: append to fncache if there are only new files to write
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40624
diff
changeset
|
1050 if self.addls: |
0728d87a8631
store: append to fncache if there are only new files to write
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40624
diff
changeset
|
1051 # if we have just new entries, let's append them to the fncache |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
1052 tr.addbackup(b'fncache') |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
1053 fp = self.vfs(b'fncache', mode=b'ab', atomictemp=True) |
40736
0728d87a8631
store: append to fncache if there are only new files to write
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40624
diff
changeset
|
1054 if self.addls: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
1055 fp.write(encodedir(b'\n'.join(self.addls) + b'\n')) |
40736
0728d87a8631
store: append to fncache if there are only new files to write
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40624
diff
changeset
|
1056 fp.close() |
0728d87a8631
store: append to fncache if there are only new files to write
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40624
diff
changeset
|
1057 self.entries = None |
0728d87a8631
store: append to fncache if there are only new files to write
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40624
diff
changeset
|
1058 self.addls = set() |
8530
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
1059 |
50315
cf6e1d535602
fncache: make it possible to ignore some file
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
49306
diff
changeset
|
1060 def addignore(self, fn): |
cf6e1d535602
fncache: make it possible to ignore some file
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
49306
diff
changeset
|
1061 self._ignores.add(fn) |
cf6e1d535602
fncache: make it possible to ignore some file
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
49306
diff
changeset
|
1062 |
8530
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
1063 def add(self, fn): |
50315
cf6e1d535602
fncache: make it possible to ignore some file
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
49306
diff
changeset
|
1064 if fn in self._ignores: |
cf6e1d535602
fncache: make it possible to ignore some file
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
49306
diff
changeset
|
1065 return |
8530
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
1066 if self.entries is None: |
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
1067 self._load() |
10577
d5bd1beff794
store: only add new entries to the fncache file
Adrian Buehlmann <adrian@cadifra.com>
parents:
10339
diff
changeset
|
1068 if fn not in self.entries: |
40736
0728d87a8631
store: append to fncache if there are only new files to write
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40624
diff
changeset
|
1069 self.addls.add(fn) |
8530
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
1070 |
20885
f49d60fa40a5
fncache: clean up fncache during strips
Durham Goode <durham@fb.com>
parents:
20884
diff
changeset
|
1071 def remove(self, fn): |
f49d60fa40a5
fncache: clean up fncache during strips
Durham Goode <durham@fb.com>
parents:
20884
diff
changeset
|
1072 if self.entries is None: |
f49d60fa40a5
fncache: clean up fncache during strips
Durham Goode <durham@fb.com>
parents:
20884
diff
changeset
|
1073 self._load() |
40736
0728d87a8631
store: append to fncache if there are only new files to write
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40624
diff
changeset
|
1074 if fn in self.addls: |
0728d87a8631
store: append to fncache if there are only new files to write
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40624
diff
changeset
|
1075 self.addls.remove(fn) |
0728d87a8631
store: append to fncache if there are only new files to write
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40624
diff
changeset
|
1076 return |
20885
f49d60fa40a5
fncache: clean up fncache during strips
Durham Goode <durham@fb.com>
parents:
20884
diff
changeset
|
1077 try: |
f49d60fa40a5
fncache: clean up fncache during strips
Durham Goode <durham@fb.com>
parents:
20884
diff
changeset
|
1078 self.entries.remove(fn) |
f49d60fa40a5
fncache: clean up fncache during strips
Durham Goode <durham@fb.com>
parents:
20884
diff
changeset
|
1079 self._dirty = True |
f49d60fa40a5
fncache: clean up fncache during strips
Durham Goode <durham@fb.com>
parents:
20884
diff
changeset
|
1080 except KeyError: |
f49d60fa40a5
fncache: clean up fncache during strips
Durham Goode <durham@fb.com>
parents:
20884
diff
changeset
|
1081 pass |
f49d60fa40a5
fncache: clean up fncache during strips
Durham Goode <durham@fb.com>
parents:
20884
diff
changeset
|
1082 |
17782
8095306c1fb2
store: move __contains__() implementation from class fncache into fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents:
17747
diff
changeset
|
1083 def __contains__(self, fn): |
40736
0728d87a8631
store: append to fncache if there are only new files to write
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40624
diff
changeset
|
1084 if fn in self.addls: |
0728d87a8631
store: append to fncache if there are only new files to write
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40624
diff
changeset
|
1085 return True |
8530
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
1086 if self.entries is None: |
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
1087 self._load() |
17782
8095306c1fb2
store: move __contains__() implementation from class fncache into fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents:
17747
diff
changeset
|
1088 return fn in self.entries |
8530
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
1089 |
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
1090 def __iter__(self): |
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
1091 if self.entries is None: |
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
1092 self._load() |
40736
0728d87a8631
store: append to fncache if there are only new files to write
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40624
diff
changeset
|
1093 return iter(self.entries | self.addls) |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
1094 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
1095 |
41093
6498f0e03526
vfs: fix proxyvfs inheritance
Boris Feld <boris.feld@octobus.net>
parents:
40748
diff
changeset
|
1096 class _fncachevfs(vfsmod.proxyvfs): |
17721
cf236e3501c3
store: rename argument name from "op"(ener) to "vfs"
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17693
diff
changeset
|
1097 def __init__(self, vfs, fnc, encode): |
33412
a42369e04aee
vfs: rename auditvfs to proxyvfs
Yuya Nishihara <yuya@tcha.org>
parents:
32372
diff
changeset
|
1098 vfsmod.proxyvfs.__init__(self, vfs) |
14194
3a90fb3addc1
store: break up reference cycle introduced in 9cbff8a39a2a
Adrian Buehlmann <adrian@cadifra.com>
parents:
14166
diff
changeset
|
1099 self.fncache = fnc |
3a90fb3addc1
store: break up reference cycle introduced in 9cbff8a39a2a
Adrian Buehlmann <adrian@cadifra.com>
parents:
14166
diff
changeset
|
1100 self.encode = encode |
3a90fb3addc1
store: break up reference cycle introduced in 9cbff8a39a2a
Adrian Buehlmann <adrian@cadifra.com>
parents:
14166
diff
changeset
|
1101 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
1102 def __call__(self, path, mode=b'r', *args, **kw): |
38661
8ac0c9cd4c48
fncache: avoid loading the filename cache when not actually modifying it
Martijn Pieters <mj@zopatista.com>
parents:
37409
diff
changeset
|
1103 encoded = self.encode(path) |
50467
0dbab42adca5
store: do not record file that are not part of a revlog in fncache
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50451
diff
changeset
|
1104 if ( |
0dbab42adca5
store: do not record file that are not part of a revlog in fncache
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50451
diff
changeset
|
1105 mode not in (b'r', b'rb') |
0dbab42adca5
store: do not record file that are not part of a revlog in fncache
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50451
diff
changeset
|
1106 and (path.startswith(b'data/') or path.startswith(b'meta/')) |
50671
e06d1a779eb6
store: stop relying on a `revlog_type` property
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50670
diff
changeset
|
1107 and is_revlog_file(path) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
1108 ): |
38661
8ac0c9cd4c48
fncache: avoid loading the filename cache when not actually modifying it
Martijn Pieters <mj@zopatista.com>
parents:
37409
diff
changeset
|
1109 # do not trigger a fncache load when adding a file that already is |
8ac0c9cd4c48
fncache: avoid loading the filename cache when not actually modifying it
Martijn Pieters <mj@zopatista.com>
parents:
37409
diff
changeset
|
1110 # known to exist. |
50812
30381812708d
fncache: fix a bug that corrupts the fncache after transaction rollback
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50706
diff
changeset
|
1111 notload = self.fncache.entries is None and ( |
30381812708d
fncache: fix a bug that corrupts the fncache after transaction rollback
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50706
diff
changeset
|
1112 # if the file has size zero, it should be considered as missing. |
30381812708d
fncache: fix a bug that corrupts the fncache after transaction rollback
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50706
diff
changeset
|
1113 # Such zero-size files are the result of truncation when a |
30381812708d
fncache: fix a bug that corrupts the fncache after transaction rollback
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50706
diff
changeset
|
1114 # transaction is aborted. |
30381812708d
fncache: fix a bug that corrupts the fncache after transaction rollback
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50706
diff
changeset
|
1115 self.vfs.exists(encoded) |
30381812708d
fncache: fix a bug that corrupts the fncache after transaction rollback
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50706
diff
changeset
|
1116 and self.vfs.stat(encoded).st_size |
30381812708d
fncache: fix a bug that corrupts the fncache after transaction rollback
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
50706
diff
changeset
|
1117 ) |
38661
8ac0c9cd4c48
fncache: avoid loading the filename cache when not actually modifying it
Martijn Pieters <mj@zopatista.com>
parents:
37409
diff
changeset
|
1118 if not notload: |
8ac0c9cd4c48
fncache: avoid loading the filename cache when not actually modifying it
Martijn Pieters <mj@zopatista.com>
parents:
37409
diff
changeset
|
1119 self.fncache.add(path) |
8ac0c9cd4c48
fncache: avoid loading the filename cache when not actually modifying it
Martijn Pieters <mj@zopatista.com>
parents:
37409
diff
changeset
|
1120 return self.vfs(encoded, mode, *args, **kw) |
14194
3a90fb3addc1
store: break up reference cycle introduced in 9cbff8a39a2a
Adrian Buehlmann <adrian@cadifra.com>
parents:
14166
diff
changeset
|
1121 |
17725
ffd589d4b785
vfs: define "join()" in each classes derived from "abstractvfs"
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17724
diff
changeset
|
1122 def join(self, path): |
ffd589d4b785
vfs: define "join()" in each classes derived from "abstractvfs"
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17724
diff
changeset
|
1123 if path: |
ffd589d4b785
vfs: define "join()" in each classes derived from "abstractvfs"
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17724
diff
changeset
|
1124 return self.vfs.join(self.encode(path)) |
ffd589d4b785
vfs: define "join()" in each classes derived from "abstractvfs"
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17724
diff
changeset
|
1125 else: |
ffd589d4b785
vfs: define "join()" in each classes derived from "abstractvfs"
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17724
diff
changeset
|
1126 return self.vfs.join(path) |
ffd589d4b785
vfs: define "join()" in each classes derived from "abstractvfs"
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17724
diff
changeset
|
1127 |
47443
9ab54aa56982
vfs: add a `register_file` method on the vfs class
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47389
diff
changeset
|
1128 def register_file(self, path): |
9ab54aa56982
vfs: add a `register_file` method on the vfs class
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47389
diff
changeset
|
1129 """generic hook point to lets fncache steer its stew""" |
9ab54aa56982
vfs: add a `register_file` method on the vfs class
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47389
diff
changeset
|
1130 if path.startswith(b'data/') or path.startswith(b'meta/'): |
9ab54aa56982
vfs: add a `register_file` method on the vfs class
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47389
diff
changeset
|
1131 self.fncache.add(path) |
9ab54aa56982
vfs: add a `register_file` method on the vfs class
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
47389
diff
changeset
|
1132 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
1133 |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
1134 class fncachestore(basicstore): |
17651
3b49c28658f6
store: rename "openertype" argument to "vfstype"
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17649
diff
changeset
|
1135 def __init__(self, path, vfstype, dotencode): |
17591
9a5c2ecd1158
store: move encode lambda logic into fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents:
17590
diff
changeset
|
1136 if dotencode: |
18435
8c019d2fd7c0
store: switch to C-based hashed path encoding
Bryan O'Sullivan <bryano@fb.com>
parents:
18430
diff
changeset
|
1137 encode = _pathencode |
17591
9a5c2ecd1158
store: move encode lambda logic into fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents:
17590
diff
changeset
|
1138 else: |
9a5c2ecd1158
store: move encode lambda logic into fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents:
17590
diff
changeset
|
1139 encode = _plainhybridencode |
12687
34d8247a4595
store: encode first period or space in filenames (issue1713)
Adrian Buehlmann <adrian@cadifra.com>
parents:
12171
diff
changeset
|
1140 self.encode = encode |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
1141 vfs = vfstype(path + b'/store') |
17724
bf4b72d8dd4d
store: initialize vfs field first to use it for initialization of others
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17722
diff
changeset
|
1142 self.path = vfs.base |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
1143 self.pathsep = self.path + b'/' |
17726
7cb7e17c23b2
store: invoke "os.stat()" for "createmode" initialization via vfs
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17725
diff
changeset
|
1144 self.createmode = _calcmode(vfs) |
17652
2c6f7231becc
store: rename "op" variables to "vfs"
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17651
diff
changeset
|
1145 vfs.createmode = self.createmode |
17727
6492b39a44d5
store: replace invocation of "getsize()" by "vfs.stat()"
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17726
diff
changeset
|
1146 self.rawvfs = vfs |
17652
2c6f7231becc
store: rename "op" variables to "vfs"
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17651
diff
changeset
|
1147 fnc = fncache(vfs) |
9133
996c1cd8f530
store: eliminate reference cycle in fncachestore
Simon Heimberg <simohe@besonet.ch>
parents:
8778
diff
changeset
|
1148 self.fncache = fnc |
17653
dacb50696b75
store: initialize "vfs" fields by "vfs" constructors
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17652
diff
changeset
|
1149 self.vfs = _fncachevfs(vfs, fnc, encode) |
dacb50696b75
store: initialize "vfs" fields by "vfs" constructors
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
17652
diff
changeset
|
1150 self.opener = self.vfs |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
1151 |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
1152 def join(self, f): |
17562
b42b0729744d
store: reduce string concatenation when joining
Bryan O'Sullivan <bryano@fb.com>
parents:
17555
diff
changeset
|
1153 return self.pathsep + self.encode(f) |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
1154 |
17731
c85dbae29684
store: restore getsize method
Matt Mackall <mpm@selenic.com>
parents:
17728
diff
changeset
|
1155 def getsize(self, path): |
c85dbae29684
store: restore getsize method
Matt Mackall <mpm@selenic.com>
parents:
17728
diff
changeset
|
1156 return self.rawvfs.stat(path).st_size |
c85dbae29684
store: restore getsize method
Matt Mackall <mpm@selenic.com>
parents:
17728
diff
changeset
|
1157 |
50504
862e3a13da44
store: rename `datafiles` to `data_entries`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50498
diff
changeset
|
1158 def data_entries( |
50471
521fec115dad
store: use a StoreEntry object instead of tuple for store files
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50469
diff
changeset
|
1159 self, matcher=None, undecodable=None |
50473
5a2fb64d38b2
store: use specialized class for store entries
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50472
diff
changeset
|
1160 ) -> Generator[BaseStoreEntry, None, None]: |
50480
d4f54aded22e
store: also group files by revlog in fncache version of datafiles
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50479
diff
changeset
|
1161 # Note: all files in fncache should be revlog related, However the |
d4f54aded22e
store: also group files by revlog in fncache version of datafiles
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50479
diff
changeset
|
1162 # fncache might contains such file added by previous version of |
d4f54aded22e
store: also group files by revlog in fncache version of datafiles
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50479
diff
changeset
|
1163 # Mercurial. |
50671
e06d1a779eb6
store: stop relying on a `revlog_type` property
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50670
diff
changeset
|
1164 files = ((f, None) for f in self.fncache if is_revlog_file(f)) |
50480
d4f54aded22e
store: also group files by revlog in fncache version of datafiles
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50479
diff
changeset
|
1165 by_revlog = _gather_revlog(files) |
d4f54aded22e
store: also group files by revlog in fncache version of datafiles
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50479
diff
changeset
|
1166 for revlog, details in by_revlog: |
50482
1fc25227b068
store: actually tag tree manifest revlogs as manifest revlogs
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50481
diff
changeset
|
1167 if revlog.startswith(b'data/'): |
50680
e324329fc97e
store: use the revlog type from revlog's constant instead of custom one
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50672
diff
changeset
|
1168 rl_type = KIND_FILELOG |
50483
60e613f6a229
store: add a `target_id` attribute on RevlogStoreEntry
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50482
diff
changeset
|
1169 revlog_target_id = revlog.split(b'/', 1)[1] |
50482
1fc25227b068
store: actually tag tree manifest revlogs as manifest revlogs
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50481
diff
changeset
|
1170 elif revlog.startswith(b'meta/'): |
50680
e324329fc97e
store: use the revlog type from revlog's constant instead of custom one
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50672
diff
changeset
|
1171 rl_type = KIND_MANIFESTLOG |
50483
60e613f6a229
store: add a `target_id` attribute on RevlogStoreEntry
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50482
diff
changeset
|
1172 # drop the initial directory and the `00manifest` file part |
60e613f6a229
store: add a `target_id` attribute on RevlogStoreEntry
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50482
diff
changeset
|
1173 tmp = revlog.split(b'/', 1)[1] |
60e613f6a229
store: add a `target_id` attribute on RevlogStoreEntry
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50482
diff
changeset
|
1174 revlog_target_id = tmp.rsplit(b'/', 1)[0] + b'/' |
50482
1fc25227b068
store: actually tag tree manifest revlogs as manifest revlogs
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50481
diff
changeset
|
1175 else: |
1fc25227b068
store: actually tag tree manifest revlogs as manifest revlogs
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50481
diff
changeset
|
1176 # unreachable |
1fc25227b068
store: actually tag tree manifest revlogs as manifest revlogs
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50481
diff
changeset
|
1177 assert False, revlog |
50496
e50d1fe7ebb4
store: issue a single entry for each revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50495
diff
changeset
|
1178 entry = RevlogStoreEntry( |
e50d1fe7ebb4
store: issue a single entry for each revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50495
diff
changeset
|
1179 path_prefix=revlog, |
e50d1fe7ebb4
store: issue a single entry for each revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50495
diff
changeset
|
1180 revlog_type=rl_type, |
e50d1fe7ebb4
store: issue a single entry for each revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50495
diff
changeset
|
1181 target_id=revlog_target_id, |
50671
e06d1a779eb6
store: stop relying on a `revlog_type` property
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50670
diff
changeset
|
1182 details=details, |
50496
e50d1fe7ebb4
store: issue a single entry for each revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50495
diff
changeset
|
1183 ) |
e50d1fe7ebb4
store: issue a single entry for each revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50495
diff
changeset
|
1184 if _match_tracked_entry(entry, matcher): |
e50d1fe7ebb4
store: issue a single entry for each revlog
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
50495
diff
changeset
|
1185 yield entry |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
1186 |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
1187 def copylist(self): |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
1188 d = ( |
45351
909dafff6a78
store: refactor space delimited list to proper data structure
Pulkit Goyal <7895pulkit@gmail.com>
parents:
44452
diff
changeset
|
1189 b'bookmarks', |
909dafff6a78
store: refactor space delimited list to proper data structure
Pulkit Goyal <7895pulkit@gmail.com>
parents:
44452
diff
changeset
|
1190 b'narrowspec', |
909dafff6a78
store: refactor space delimited list to proper data structure
Pulkit Goyal <7895pulkit@gmail.com>
parents:
44452
diff
changeset
|
1191 b'data', |
909dafff6a78
store: refactor space delimited list to proper data structure
Pulkit Goyal <7895pulkit@gmail.com>
parents:
44452
diff
changeset
|
1192 b'meta', |
909dafff6a78
store: refactor space delimited list to proper data structure
Pulkit Goyal <7895pulkit@gmail.com>
parents:
44452
diff
changeset
|
1193 b'dh', |
909dafff6a78
store: refactor space delimited list to proper data structure
Pulkit Goyal <7895pulkit@gmail.com>
parents:
44452
diff
changeset
|
1194 b'fncache', |
909dafff6a78
store: refactor space delimited list to proper data structure
Pulkit Goyal <7895pulkit@gmail.com>
parents:
44452
diff
changeset
|
1195 b'phaseroots', |
909dafff6a78
store: refactor space delimited list to proper data structure
Pulkit Goyal <7895pulkit@gmail.com>
parents:
44452
diff
changeset
|
1196 b'obsstore', |
909dafff6a78
store: refactor space delimited list to proper data structure
Pulkit Goyal <7895pulkit@gmail.com>
parents:
44452
diff
changeset
|
1197 b'00manifest.d', |
909dafff6a78
store: refactor space delimited list to proper data structure
Pulkit Goyal <7895pulkit@gmail.com>
parents:
44452
diff
changeset
|
1198 b'00manifest.i', |
909dafff6a78
store: refactor space delimited list to proper data structure
Pulkit Goyal <7895pulkit@gmail.com>
parents:
44452
diff
changeset
|
1199 b'00changelog.d', |
909dafff6a78
store: refactor space delimited list to proper data structure
Pulkit Goyal <7895pulkit@gmail.com>
parents:
44452
diff
changeset
|
1200 b'00changelog.i', |
45483
d252f51ab032
share: introduce config option to store requires in .hg/store
Pulkit Goyal <7895pulkit@gmail.com>
parents:
45351
diff
changeset
|
1201 b'requires', |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42905
diff
changeset
|
1202 ) |
45351
909dafff6a78
store: refactor space delimited list to proper data structure
Pulkit Goyal <7895pulkit@gmail.com>
parents:
44452
diff
changeset
|
1203 return [b'requires', b'00changelog.i'] + [b'store/' + f for f in d] |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
1204 |
20883
cd443c7589cc
fncache: move fncache writing to be in a transaction
Durham Goode <durham@fb.com>
parents:
20879
diff
changeset
|
1205 def write(self, tr): |
cd443c7589cc
fncache: move fncache writing to be in a transaction
Durham Goode <durham@fb.com>
parents:
20879
diff
changeset
|
1206 self.fncache.write(tr) |
13391
d00bbff8600e
fncachestore: defer updating the fncache file to a single file open
Adrian Buehlmann <adrian@cadifra.com>
parents:
13169
diff
changeset
|
1207 |
20884
2efdd186925d
caches: invalidate store caches when lock is taken
Durham Goode <durham@fb.com>
parents:
20883
diff
changeset
|
1208 def invalidatecaches(self): |
2efdd186925d
caches: invalidate store caches when lock is taken
Durham Goode <durham@fb.com>
parents:
20883
diff
changeset
|
1209 self.fncache.entries = None |
40736
0728d87a8631
store: append to fncache if there are only new files to write
Pulkit Goyal <pulkit@yandex-team.ru>
parents:
40624
diff
changeset
|
1210 self.fncache.addls = set() |
20884
2efdd186925d
caches: invalidate store caches when lock is taken
Durham Goode <durham@fb.com>
parents:
20883
diff
changeset
|
1211 |
20885
f49d60fa40a5
fncache: clean up fncache during strips
Durham Goode <durham@fb.com>
parents:
20884
diff
changeset
|
1212 def markremoved(self, fn): |
f49d60fa40a5
fncache: clean up fncache during strips
Durham Goode <durham@fb.com>
parents:
20884
diff
changeset
|
1213 self.fncache.remove(fn) |
f49d60fa40a5
fncache: clean up fncache during strips
Durham Goode <durham@fb.com>
parents:
20884
diff
changeset
|
1214 |
17783
df55ce6854c3
store: add new _exists helper function on fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents:
17782
diff
changeset
|
1215 def _exists(self, f): |
df55ce6854c3
store: add new _exists helper function on fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents:
17782
diff
changeset
|
1216 ef = self.encode(f) |
df55ce6854c3
store: add new _exists helper function on fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents:
17782
diff
changeset
|
1217 try: |
df55ce6854c3
store: add new _exists helper function on fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents:
17782
diff
changeset
|
1218 self.getsize(ef) |
df55ce6854c3
store: add new _exists helper function on fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents:
17782
diff
changeset
|
1219 return True |
49306
2e726c934fcd
py3: catch FileNotFoundError instead of checking errno == ENOENT
Manuel Jacob <me@manueljacob.de>
parents:
49284
diff
changeset
|
1220 except FileNotFoundError: |
17783
df55ce6854c3
store: add new _exists helper function on fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents:
17782
diff
changeset
|
1221 return False |
df55ce6854c3
store: add new _exists helper function on fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents:
17782
diff
changeset
|
1222 |
17745 | 1223 def __contains__(self, path): |
1224 '''Checks if the store contains path''' | |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
1225 path = b"/".join((b"data", path)) |
17782
8095306c1fb2
store: move __contains__() implementation from class fncache into fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents:
17747
diff
changeset
|
1226 # check for files (exact match) |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
1227 e = path + b'.i' |
17784
73e1ab39792c
store: fncache may contain non-existent entries (fixes b9a56b816ff2)
Adrian Buehlmann <adrian@cadifra.com>
parents:
17783
diff
changeset
|
1228 if e in self.fncache and self._exists(e): |
17782
8095306c1fb2
store: move __contains__() implementation from class fncache into fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents:
17747
diff
changeset
|
1229 return True |
8095306c1fb2
store: move __contains__() implementation from class fncache into fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents:
17747
diff
changeset
|
1230 # now check for directories (prefix match) |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
1231 if not path.endswith(b'/'): |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
1232 path += b'/' |
17782
8095306c1fb2
store: move __contains__() implementation from class fncache into fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents:
17747
diff
changeset
|
1233 for e in self.fncache: |
17784
73e1ab39792c
store: fncache may contain non-existent entries (fixes b9a56b816ff2)
Adrian Buehlmann <adrian@cadifra.com>
parents:
17783
diff
changeset
|
1234 if e.startswith(path) and self._exists(e): |
17782
8095306c1fb2
store: move __contains__() implementation from class fncache into fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents:
17747
diff
changeset
|
1235 return True |
8095306c1fb2
store: move __contains__() implementation from class fncache into fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents:
17747
diff
changeset
|
1236 return False |