Mercurial > hg
annotate mercurial/store.py @ 17644:9ae073f10572
histedit: fold in memory
Update the folding code to works in memory instead of applying patches on the
working directory. This is cleaner, faster and prepare the removal of the whole
patching logic.
This new collapse function will probably move into core sooner or later. A lot
of other rewriting operation may benefit from it.
author | Pierre-Yves David <pierre-yves.david@logilab.fr> |
---|---|
date | Fri, 21 Sep 2012 19:24:31 +0200 |
parents | ae103510f6aa |
children | f65c6a5f256c |
rev | line source |
---|---|
6839
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
1 # store.py - repository store handling for Mercurial |
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
2 # |
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
3 # Copyright 2008 Matt Mackall <mpm@selenic.com> |
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
4 # |
8225
46293a0c7e9f
updated license to be explicit about GPL version 2
Martin Geisler <mg@lazybytes.net>
parents:
8210
diff
changeset
|
5 # This software may be used and distributed according to the terms of the |
10263 | 6 # GNU General Public License version 2 or any later version. |
6839
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
7 |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
8 from i18n import _ |
17607
cc58dc47cb5e
store: use fast C implementation of encodedir() if it's available
Adrian Buehlmann <adrian@cadifra.com>
parents:
17605
diff
changeset
|
9 import osutil, scmutil, util, parsers |
17374
0cec762790ed
store: only one kind of OSError means "nonexistent entry"
Bryan O'Sullivan <bryano@fb.com>
parents:
17373
diff
changeset
|
10 import os, stat, errno |
6840
80e51429cb9a
introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents:
6839
diff
changeset
|
11 |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
12 _sha = util.sha1 |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
13 |
8531
810387f59696
filelog encoding: move the encoding/decoding into store
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8530
diff
changeset
|
14 # This avoids a collision between a file named foo and a dir named |
810387f59696
filelog encoding: move the encoding/decoding into store
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8530
diff
changeset
|
15 # foo.i or foo.d |
17607
cc58dc47cb5e
store: use fast C implementation of encodedir() if it's available
Adrian Buehlmann <adrian@cadifra.com>
parents:
17605
diff
changeset
|
16 def _encodedir(path): |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
17 ''' |
17607
cc58dc47cb5e
store: use fast C implementation of encodedir() if it's available
Adrian Buehlmann <adrian@cadifra.com>
parents:
17605
diff
changeset
|
18 >>> _encodedir('data/foo.i') |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
19 'data/foo.i' |
17607
cc58dc47cb5e
store: use fast C implementation of encodedir() if it's available
Adrian Buehlmann <adrian@cadifra.com>
parents:
17605
diff
changeset
|
20 >>> _encodedir('data/foo.i/bla.i') |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
21 'data/foo.i.hg/bla.i' |
17607
cc58dc47cb5e
store: use fast C implementation of encodedir() if it's available
Adrian Buehlmann <adrian@cadifra.com>
parents:
17605
diff
changeset
|
22 >>> _encodedir('data/foo.i.hg/bla.i') |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
23 'data/foo.i.hg.hg/bla.i' |
17607
cc58dc47cb5e
store: use fast C implementation of encodedir() if it's available
Adrian Buehlmann <adrian@cadifra.com>
parents:
17605
diff
changeset
|
24 >>> _encodedir('data/foo.i\\ndata/foo.i/bla.i\\ndata/foo.i.hg/bla.i\\n') |
17605
e9cc29be3305
store: add multiline doctest case for encodedir()
Adrian Buehlmann <adrian@cadifra.com>
parents:
17604
diff
changeset
|
25 'data/foo.i\\ndata/foo.i.hg/bla.i\\ndata/foo.i.hg.hg/bla.i\\n' |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
26 ''' |
8531
810387f59696
filelog encoding: move the encoding/decoding into store
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8530
diff
changeset
|
27 return (path |
810387f59696
filelog encoding: move the encoding/decoding into store
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8530
diff
changeset
|
28 .replace(".hg/", ".hg.hg/") |
810387f59696
filelog encoding: move the encoding/decoding into store
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8530
diff
changeset
|
29 .replace(".i/", ".i.hg/") |
810387f59696
filelog encoding: move the encoding/decoding into store
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8530
diff
changeset
|
30 .replace(".d/", ".d.hg/")) |
810387f59696
filelog encoding: move the encoding/decoding into store
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8530
diff
changeset
|
31 |
17607
cc58dc47cb5e
store: use fast C implementation of encodedir() if it's available
Adrian Buehlmann <adrian@cadifra.com>
parents:
17605
diff
changeset
|
32 encodedir = getattr(parsers, 'encodedir', _encodedir) |
cc58dc47cb5e
store: use fast C implementation of encodedir() if it's available
Adrian Buehlmann <adrian@cadifra.com>
parents:
17605
diff
changeset
|
33 |
8531
810387f59696
filelog encoding: move the encoding/decoding into store
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8530
diff
changeset
|
34 def decodedir(path): |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
35 ''' |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
36 >>> decodedir('data/foo.i') |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
37 'data/foo.i' |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
38 >>> decodedir('data/foo.i.hg/bla.i') |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
39 'data/foo.i/bla.i' |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
40 >>> decodedir('data/foo.i.hg.hg/bla.i') |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
41 'data/foo.i.hg/bla.i' |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
42 ''' |
17586
2f1475da1940
store: remove uneeded startswith('data/') checks in encodedir() and decodedir()
Adrian Buehlmann <adrian@cadifra.com>
parents:
17585
diff
changeset
|
43 if ".hg/" not in path: |
8531
810387f59696
filelog encoding: move the encoding/decoding into store
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8530
diff
changeset
|
44 return path |
810387f59696
filelog encoding: move the encoding/decoding into store
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8530
diff
changeset
|
45 return (path |
810387f59696
filelog encoding: move the encoding/decoding into store
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8530
diff
changeset
|
46 .replace(".d.hg/", ".d/") |
810387f59696
filelog encoding: move the encoding/decoding into store
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8530
diff
changeset
|
47 .replace(".i.hg/", ".i/") |
810387f59696
filelog encoding: move the encoding/decoding into store
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8530
diff
changeset
|
48 .replace(".hg.hg/", ".hg/")) |
810387f59696
filelog encoding: move the encoding/decoding into store
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8530
diff
changeset
|
49 |
6839
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
50 def _buildencodefun(): |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
51 ''' |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
52 >>> enc, dec = _buildencodefun() |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
53 |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
54 >>> enc('nothing/special.txt') |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
55 'nothing/special.txt' |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
56 >>> dec('nothing/special.txt') |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
57 'nothing/special.txt' |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
58 |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
59 >>> enc('HELLO') |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
60 '_h_e_l_l_o' |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
61 >>> dec('_h_e_l_l_o') |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
62 'HELLO' |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
63 |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
64 >>> enc('hello:world?') |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
65 'hello~3aworld~3f' |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
66 >>> dec('hello~3aworld~3f') |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
67 'hello:world?' |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
68 |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
69 >>> enc('the\x07quick\xADshot') |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
70 'the~07quick~adshot' |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
71 >>> dec('the~07quick~adshot') |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
72 'the\\x07quick\\xadshot' |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
73 ''' |
6839
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
74 e = '_' |
14288
00a0ab08f986
store: change names to comply with project coding standards
Adrian Buehlmann <adrian@cadifra.com>
parents:
14194
diff
changeset
|
75 winreserved = [ord(x) for x in '\\:*?"<>|'] |
10282
08a0f04b56bd
many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents:
10263
diff
changeset
|
76 cmap = dict([(chr(x), chr(x)) for x in xrange(127)]) |
14288
00a0ab08f986
store: change names to comply with project coding standards
Adrian Buehlmann <adrian@cadifra.com>
parents:
14194
diff
changeset
|
77 for x in (range(32) + range(126, 256) + winreserved): |
6839
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
78 cmap[chr(x)] = "~%02x" % x |
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
79 for x in range(ord("A"), ord("Z")+1) + [ord(e)]: |
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
80 cmap[chr(x)] = e + chr(x).lower() |
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
81 dmap = {} |
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
82 for k, v in cmap.iteritems(): |
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
83 dmap[v] = k |
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
84 def decode(s): |
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
85 i = 0 |
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
86 while i < len(s): |
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
87 for l in xrange(1, 4): |
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
88 try: |
10282
08a0f04b56bd
many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents:
10263
diff
changeset
|
89 yield dmap[s[i:i + l]] |
6839
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
90 i += l |
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
91 break |
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
92 except KeyError: |
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
93 pass |
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
94 else: |
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
95 raise KeyError |
17608
776240123525
store: extract functions _encodefname and _decodefname
Adrian Buehlmann <adrian@cadifra.com>
parents:
17607
diff
changeset
|
96 return (lambda s: ''.join([cmap[c] for c in s]), |
776240123525
store: extract functions _encodefname and _decodefname
Adrian Buehlmann <adrian@cadifra.com>
parents:
17607
diff
changeset
|
97 lambda s: ''.join(list(decode(s)))) |
776240123525
store: extract functions _encodefname and _decodefname
Adrian Buehlmann <adrian@cadifra.com>
parents:
17607
diff
changeset
|
98 |
776240123525
store: extract functions _encodefname and _decodefname
Adrian Buehlmann <adrian@cadifra.com>
parents:
17607
diff
changeset
|
99 _encodefname, _decodefname = _buildencodefun() |
6839
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
100 |
17608
776240123525
store: extract functions _encodefname and _decodefname
Adrian Buehlmann <adrian@cadifra.com>
parents:
17607
diff
changeset
|
101 def encodefilename(s): |
776240123525
store: extract functions _encodefname and _decodefname
Adrian Buehlmann <adrian@cadifra.com>
parents:
17607
diff
changeset
|
102 ''' |
776240123525
store: extract functions _encodefname and _decodefname
Adrian Buehlmann <adrian@cadifra.com>
parents:
17607
diff
changeset
|
103 >>> encodefilename('foo.i/bar.d/bla.hg/hi:world?/HELLO') |
776240123525
store: extract functions _encodefname and _decodefname
Adrian Buehlmann <adrian@cadifra.com>
parents:
17607
diff
changeset
|
104 'foo.i.hg/bar.d.hg/bla.hg.hg/hi~3aworld~3f/_h_e_l_l_o' |
776240123525
store: extract functions _encodefname and _decodefname
Adrian Buehlmann <adrian@cadifra.com>
parents:
17607
diff
changeset
|
105 ''' |
776240123525
store: extract functions _encodefname and _decodefname
Adrian Buehlmann <adrian@cadifra.com>
parents:
17607
diff
changeset
|
106 return _encodefname(encodedir(s)) |
776240123525
store: extract functions _encodefname and _decodefname
Adrian Buehlmann <adrian@cadifra.com>
parents:
17607
diff
changeset
|
107 |
776240123525
store: extract functions _encodefname and _decodefname
Adrian Buehlmann <adrian@cadifra.com>
parents:
17607
diff
changeset
|
108 def decodefilename(s): |
776240123525
store: extract functions _encodefname and _decodefname
Adrian Buehlmann <adrian@cadifra.com>
parents:
17607
diff
changeset
|
109 ''' |
776240123525
store: extract functions _encodefname and _decodefname
Adrian Buehlmann <adrian@cadifra.com>
parents:
17607
diff
changeset
|
110 >>> decodefilename('foo.i.hg/bar.d.hg/bla.hg.hg/hi~3aworld~3f/_h_e_l_l_o') |
776240123525
store: extract functions _encodefname and _decodefname
Adrian Buehlmann <adrian@cadifra.com>
parents:
17607
diff
changeset
|
111 'foo.i/bar.d/bla.hg/hi:world?/HELLO' |
776240123525
store: extract functions _encodefname and _decodefname
Adrian Buehlmann <adrian@cadifra.com>
parents:
17607
diff
changeset
|
112 ''' |
776240123525
store: extract functions _encodefname and _decodefname
Adrian Buehlmann <adrian@cadifra.com>
parents:
17607
diff
changeset
|
113 return decodedir(_decodefname(s)) |
6839
01db3e101362
move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff
changeset
|
114 |
14288
00a0ab08f986
store: change names to comply with project coding standards
Adrian Buehlmann <adrian@cadifra.com>
parents:
14194
diff
changeset
|
115 def _buildlowerencodefun(): |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
116 ''' |
14288
00a0ab08f986
store: change names to comply with project coding standards
Adrian Buehlmann <adrian@cadifra.com>
parents:
14194
diff
changeset
|
117 >>> f = _buildlowerencodefun() |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
118 >>> f('nothing/special.txt') |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
119 'nothing/special.txt' |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
120 >>> f('HELLO') |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
121 'hello' |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
122 >>> f('hello:world?') |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
123 'hello~3aworld~3f' |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
124 >>> f('the\x07quick\xADshot') |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
125 'the~07quick~adshot' |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
126 ''' |
14288
00a0ab08f986
store: change names to comply with project coding standards
Adrian Buehlmann <adrian@cadifra.com>
parents:
14194
diff
changeset
|
127 winreserved = [ord(x) for x in '\\:*?"<>|'] |
10282
08a0f04b56bd
many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents:
10263
diff
changeset
|
128 cmap = dict([(chr(x), chr(x)) for x in xrange(127)]) |
14288
00a0ab08f986
store: change names to comply with project coding standards
Adrian Buehlmann <adrian@cadifra.com>
parents:
14194
diff
changeset
|
129 for x in (range(32) + range(126, 256) + winreserved): |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
130 cmap[chr(x)] = "~%02x" % x |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
131 for x in range(ord("A"), ord("Z")+1): |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
132 cmap[chr(x)] = chr(x).lower() |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
133 return lambda s: "".join([cmap[c] for c in s]) |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
134 |
14288
00a0ab08f986
store: change names to comply with project coding standards
Adrian Buehlmann <adrian@cadifra.com>
parents:
14194
diff
changeset
|
135 lowerencode = _buildlowerencodefun() |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
136 |
17570
f53a7b256ca6
store: optimze _auxencode() a bit by grouping the reserved names by length
Adrian Buehlmann <adrian@cadifra.com>
parents:
17569
diff
changeset
|
137 # Windows reserved names: con, prn, aux, nul, com1..com9, lpt1..lpt9 |
f53a7b256ca6
store: optimze _auxencode() a bit by grouping the reserved names by length
Adrian Buehlmann <adrian@cadifra.com>
parents:
17569
diff
changeset
|
138 _winres3 = ('aux', 'con', 'prn', 'nul') # length 3 |
f53a7b256ca6
store: optimze _auxencode() a bit by grouping the reserved names by length
Adrian Buehlmann <adrian@cadifra.com>
parents:
17569
diff
changeset
|
139 _winres4 = ('com', 'lpt') # length 4 (with trailing 1..9) |
12687
34d8247a4595
store: encode first period or space in filenames (issue1713)
Adrian Buehlmann <adrian@cadifra.com>
parents:
12171
diff
changeset
|
140 def _auxencode(path, dotencode): |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
141 ''' |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
142 Encodes filenames containing names reserved by Windows or which end in |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
143 period or space. Does not touch other single reserved characters c. |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
144 Specifically, c in '\\:*?"<>|' or ord(c) <= 31 are *not* encoded here. |
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
145 Additionally encodes space or period at the beginning, if dotencode is |
17569
e9af2134825c
store: explain "aux.foo" versus "foo.aux" in doc of _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents:
17568
diff
changeset
|
146 True. Parameter path is assumed to be all lowercase. |
e9af2134825c
store: explain "aux.foo" versus "foo.aux" in doc of _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents:
17568
diff
changeset
|
147 A segment only needs encoding if a reserved name appears as a |
e9af2134825c
store: explain "aux.foo" versus "foo.aux" in doc of _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents:
17568
diff
changeset
|
148 basename (e.g. "aux", "aux.foo"). A directory or file named "foo.aux" |
e9af2134825c
store: explain "aux.foo" versus "foo.aux" in doc of _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents:
17568
diff
changeset
|
149 doesn't need encoding. |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
150 |
17589
b11024849db6
store: parameter path of _auxencode is now a list of strings
Adrian Buehlmann <adrian@cadifra.com>
parents:
17588
diff
changeset
|
151 >>> s = '.foo/aux.txt/txt.aux/con/prn/nul/foo.' |
b11024849db6
store: parameter path of _auxencode is now a list of strings
Adrian Buehlmann <adrian@cadifra.com>
parents:
17588
diff
changeset
|
152 >>> _auxencode(s.split('/'), True) |
17574
81a033bb29bc
store: let _auxencode() return the list of path segments
Adrian Buehlmann <adrian@cadifra.com>
parents:
17573
diff
changeset
|
153 ['~2efoo', 'au~78.txt', 'txt.aux', 'co~6e', 'pr~6e', 'nu~6c', 'foo~2e'] |
17589
b11024849db6
store: parameter path of _auxencode is now a list of strings
Adrian Buehlmann <adrian@cadifra.com>
parents:
17588
diff
changeset
|
154 >>> s = '.com1com2/lpt9.lpt4.lpt1/conprn/com0/lpt0/foo.' |
b11024849db6
store: parameter path of _auxencode is now a list of strings
Adrian Buehlmann <adrian@cadifra.com>
parents:
17588
diff
changeset
|
155 >>> _auxencode(s.split('/'), False) |
17574
81a033bb29bc
store: let _auxencode() return the list of path segments
Adrian Buehlmann <adrian@cadifra.com>
parents:
17573
diff
changeset
|
156 ['.com1com2', 'lp~749.lpt4.lpt1', 'conprn', 'com0', 'lpt0', 'foo~2e'] |
17589
b11024849db6
store: parameter path of _auxencode is now a list of strings
Adrian Buehlmann <adrian@cadifra.com>
parents:
17588
diff
changeset
|
157 >>> _auxencode(['foo. '], True) |
17574
81a033bb29bc
store: let _auxencode() return the list of path segments
Adrian Buehlmann <adrian@cadifra.com>
parents:
17573
diff
changeset
|
158 ['foo.~20'] |
17589
b11024849db6
store: parameter path of _auxencode is now a list of strings
Adrian Buehlmann <adrian@cadifra.com>
parents:
17588
diff
changeset
|
159 >>> _auxencode([' .foo'], True) |
17574
81a033bb29bc
store: let _auxencode() return the list of path segments
Adrian Buehlmann <adrian@cadifra.com>
parents:
17573
diff
changeset
|
160 ['~20.foo'] |
13949
ba43aa1e173c
store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents:
13426
diff
changeset
|
161 ''' |
17589
b11024849db6
store: parameter path of _auxencode is now a list of strings
Adrian Buehlmann <adrian@cadifra.com>
parents:
17588
diff
changeset
|
162 for i, n in enumerate(path): |
17572
b644287e79a8
store: unindent most of the contents of the for loop in _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents:
17571
diff
changeset
|
163 if not n: |
b644287e79a8
store: unindent most of the contents of the for loop in _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents:
17571
diff
changeset
|
164 continue |
b644287e79a8
store: unindent most of the contents of the for loop in _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents:
17571
diff
changeset
|
165 if dotencode and n[0] in '. ': |
b644287e79a8
store: unindent most of the contents of the for loop in _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents:
17571
diff
changeset
|
166 n = "~%02x" % ord(n[0]) + n[1:] |
17589
b11024849db6
store: parameter path of _auxencode is now a list of strings
Adrian Buehlmann <adrian@cadifra.com>
parents:
17588
diff
changeset
|
167 path[i] = n |
17572
b644287e79a8
store: unindent most of the contents of the for loop in _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents:
17571
diff
changeset
|
168 else: |
b644287e79a8
store: unindent most of the contents of the for loop in _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents:
17571
diff
changeset
|
169 l = n.find('.') |
b644287e79a8
store: unindent most of the contents of the for loop in _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents:
17571
diff
changeset
|
170 if l == -1: |
b644287e79a8
store: unindent most of the contents of the for loop in _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents:
17571
diff
changeset
|
171 l = len(n) |
b644287e79a8
store: unindent most of the contents of the for loop in _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents:
17571
diff
changeset
|
172 if ((l == 3 and n[:3] in _winres3) or |
b644287e79a8
store: unindent most of the contents of the for loop in _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents:
17571
diff
changeset
|
173 (l == 4 and n[3] <= '9' and n[3] >= '1' |
b644287e79a8
store: unindent most of the contents of the for loop in _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents:
17571
diff
changeset
|
174 and n[:3] in _winres4)): |
b644287e79a8
store: unindent most of the contents of the for loop in _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents:
17571
diff
changeset
|
175 # encode third letter ('aux' -> 'au~78') |
b644287e79a8
store: unindent most of the contents of the for loop in _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents:
17571
diff
changeset
|
176 ec = "~%02x" % ord(n[2]) |
b644287e79a8
store: unindent most of the contents of the for loop in _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents:
17571
diff
changeset
|
177 n = n[0:2] + ec + n[3:] |
17589
b11024849db6
store: parameter path of _auxencode is now a list of strings
Adrian Buehlmann <adrian@cadifra.com>
parents:
17588
diff
changeset
|
178 path[i] = n |
17572
b644287e79a8
store: unindent most of the contents of the for loop in _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents:
17571
diff
changeset
|
179 if n[-1] in '. ': |
b644287e79a8
store: unindent most of the contents of the for loop in _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents:
17571
diff
changeset
|
180 # encode last period or space ('foo...' -> 'foo..~2e') |
17589
b11024849db6
store: parameter path of _auxencode is now a list of strings
Adrian Buehlmann <adrian@cadifra.com>
parents:
17588
diff
changeset
|
181 path[i] = n[:-1] + "~%02x" % ord(n[-1]) |
b11024849db6
store: parameter path of _auxencode is now a list of strings
Adrian Buehlmann <adrian@cadifra.com>
parents:
17588
diff
changeset
|
182 return path |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
183 |
14288
00a0ab08f986
store: change names to comply with project coding standards
Adrian Buehlmann <adrian@cadifra.com>
parents:
14194
diff
changeset
|
184 _maxstorepathlen = 120 |
00a0ab08f986
store: change names to comply with project coding standards
Adrian Buehlmann <adrian@cadifra.com>
parents:
14194
diff
changeset
|
185 _dirprefixlen = 8 |
00a0ab08f986
store: change names to comply with project coding standards
Adrian Buehlmann <adrian@cadifra.com>
parents:
14194
diff
changeset
|
186 _maxshortdirslen = 8 * (_dirprefixlen + 1) - 4 |
17610
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
187 |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
188 def _hashencode(path, dotencode): |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
189 digest = _sha(path).hexdigest() |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
190 le = lowerencode(path).split('/')[1:] |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
191 parts = _auxencode(le, dotencode) |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
192 basename = parts[-1] |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
193 _root, ext = os.path.splitext(basename) |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
194 sdirs = [] |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
195 sdirslen = 0 |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
196 for p in parts[:-1]: |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
197 d = p[:_dirprefixlen] |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
198 if d[-1] in '. ': |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
199 # Windows can't access dirs ending in period or space |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
200 d = d[:-1] + '_' |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
201 if sdirslen == 0: |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
202 t = len(d) |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
203 else: |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
204 t = sdirslen + 1 + len(d) |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
205 if t > _maxshortdirslen: |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
206 break |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
207 sdirs.append(d) |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
208 sdirslen = t |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
209 dirs = '/'.join(sdirs) |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
210 if len(dirs) > 0: |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
211 dirs += '/' |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
212 res = 'dh/' + dirs + digest + ext |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
213 spaceleft = _maxstorepathlen - len(res) |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
214 if spaceleft > 0: |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
215 filler = basename[:spaceleft] |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
216 res = 'dh/' + dirs + filler + digest + ext |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
217 return res |
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
218 |
17590
eb0884680f5c
store: eliminate one level of lambda functions on _hybridencode
Adrian Buehlmann <adrian@cadifra.com>
parents:
17589
diff
changeset
|
219 def _hybridencode(path, dotencode): |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
220 '''encodes path with a length limit |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
221 |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
222 Encodes all paths that begin with 'data/', according to the following. |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
223 |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
224 Default encoding (reversible): |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
225 |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
226 Encodes all uppercase letters 'X' as '_x'. All reserved or illegal |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
227 characters are encoded as '~xx', where xx is the two digit hex code |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
228 of the character (see encodefilename). |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
229 Relevant path components consisting of Windows reserved filenames are |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
230 masked by encoding the third character ('aux' -> 'au~78', see auxencode). |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
231 |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
232 Hashed encoding (not reversible): |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
233 |
14288
00a0ab08f986
store: change names to comply with project coding standards
Adrian Buehlmann <adrian@cadifra.com>
parents:
14194
diff
changeset
|
234 If the default-encoded path is longer than _maxstorepathlen, a |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
235 non-reversible hybrid hashing of the path is done instead. |
14288
00a0ab08f986
store: change names to comply with project coding standards
Adrian Buehlmann <adrian@cadifra.com>
parents:
14194
diff
changeset
|
236 This encoding uses up to _dirprefixlen characters of all directory |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
237 levels of the lowerencoded path, but not more levels than can fit into |
14288
00a0ab08f986
store: change names to comply with project coding standards
Adrian Buehlmann <adrian@cadifra.com>
parents:
14194
diff
changeset
|
238 _maxshortdirslen. |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
239 Then follows the filler followed by the sha digest of the full path. |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
240 The filler is the beginning of the basename of the lowerencoded path |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
241 (the basename is everything after the last path separator). The filler |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
242 is as long as possible, filling in characters from the basename until |
14288
00a0ab08f986
store: change names to comply with project coding standards
Adrian Buehlmann <adrian@cadifra.com>
parents:
14194
diff
changeset
|
243 the encoded path has _maxstorepathlen characters (or all chars of the |
00a0ab08f986
store: change names to comply with project coding standards
Adrian Buehlmann <adrian@cadifra.com>
parents:
14194
diff
changeset
|
244 basename have been taken). |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
245 The extension (e.g. '.i' or '.d') is preserved. |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
246 |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
247 The string 'data/' at the beginning is replaced with 'dh/', if the hashed |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
248 encoding was used. |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
249 ''' |
17609
cbc180cfd60b
store: reuse direncoded path in _hybridencode
Adrian Buehlmann <adrian@cadifra.com>
parents:
17608
diff
changeset
|
250 path = encodedir(path) |
cbc180cfd60b
store: reuse direncoded path in _hybridencode
Adrian Buehlmann <adrian@cadifra.com>
parents:
17608
diff
changeset
|
251 ef = _encodefname(path).split('/') |
17590
eb0884680f5c
store: eliminate one level of lambda functions on _hybridencode
Adrian Buehlmann <adrian@cadifra.com>
parents:
17589
diff
changeset
|
252 res = '/'.join(_auxencode(ef, dotencode)) |
14288
00a0ab08f986
store: change names to comply with project coding standards
Adrian Buehlmann <adrian@cadifra.com>
parents:
14194
diff
changeset
|
253 if len(res) > _maxstorepathlen: |
17610
d0afa149e059
store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents:
17609
diff
changeset
|
254 res = _hashencode(path, dotencode) |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
255 return res |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
256 |
17624
ae103510f6aa
store: add a fallback _pathencode Python function
Adrian Buehlmann <adrian@cadifra.com>
parents:
17623
diff
changeset
|
257 def _pathencode(path): |
ae103510f6aa
store: add a fallback _pathencode Python function
Adrian Buehlmann <adrian@cadifra.com>
parents:
17623
diff
changeset
|
258 ef = _encodefname(encodedir(path)).split('/') |
ae103510f6aa
store: add a fallback _pathencode Python function
Adrian Buehlmann <adrian@cadifra.com>
parents:
17623
diff
changeset
|
259 res = '/'.join(_auxencode(ef, True)) |
ae103510f6aa
store: add a fallback _pathencode Python function
Adrian Buehlmann <adrian@cadifra.com>
parents:
17623
diff
changeset
|
260 if len(res) > _maxstorepathlen: |
ae103510f6aa
store: add a fallback _pathencode Python function
Adrian Buehlmann <adrian@cadifra.com>
parents:
17623
diff
changeset
|
261 return None |
ae103510f6aa
store: add a fallback _pathencode Python function
Adrian Buehlmann <adrian@cadifra.com>
parents:
17623
diff
changeset
|
262 return res |
ae103510f6aa
store: add a fallback _pathencode Python function
Adrian Buehlmann <adrian@cadifra.com>
parents:
17623
diff
changeset
|
263 |
ae103510f6aa
store: add a fallback _pathencode Python function
Adrian Buehlmann <adrian@cadifra.com>
parents:
17623
diff
changeset
|
264 _pathencode = getattr(parsers, 'pathencode', _pathencode) |
ae103510f6aa
store: add a fallback _pathencode Python function
Adrian Buehlmann <adrian@cadifra.com>
parents:
17623
diff
changeset
|
265 |
ae103510f6aa
store: add a fallback _pathencode Python function
Adrian Buehlmann <adrian@cadifra.com>
parents:
17623
diff
changeset
|
266 def _dothybridencode(f): |
ae103510f6aa
store: add a fallback _pathencode Python function
Adrian Buehlmann <adrian@cadifra.com>
parents:
17623
diff
changeset
|
267 ef = _pathencode(f) |
ae103510f6aa
store: add a fallback _pathencode Python function
Adrian Buehlmann <adrian@cadifra.com>
parents:
17623
diff
changeset
|
268 if ef is None: |
ae103510f6aa
store: add a fallback _pathencode Python function
Adrian Buehlmann <adrian@cadifra.com>
parents:
17623
diff
changeset
|
269 return _hashencode(encodedir(f), True) |
ae103510f6aa
store: add a fallback _pathencode Python function
Adrian Buehlmann <adrian@cadifra.com>
parents:
17623
diff
changeset
|
270 return ef |
ae103510f6aa
store: add a fallback _pathencode Python function
Adrian Buehlmann <adrian@cadifra.com>
parents:
17623
diff
changeset
|
271 |
17623
448e6ed7c557
store: move _plainhybridencode and _dothybridencode higher up in the file
Adrian Buehlmann <adrian@cadifra.com>
parents:
17621
diff
changeset
|
272 def _plainhybridencode(f): |
448e6ed7c557
store: move _plainhybridencode and _dothybridencode higher up in the file
Adrian Buehlmann <adrian@cadifra.com>
parents:
17621
diff
changeset
|
273 return _hybridencode(f, False) |
448e6ed7c557
store: move _plainhybridencode and _dothybridencode higher up in the file
Adrian Buehlmann <adrian@cadifra.com>
parents:
17621
diff
changeset
|
274 |
6898
69aeaaaf6e07
store: simplify class hierarchy
Matt Mackall <mpm@selenic.com>
parents:
6897
diff
changeset
|
275 def _calcmode(path): |
69aeaaaf6e07
store: simplify class hierarchy
Matt Mackall <mpm@selenic.com>
parents:
6897
diff
changeset
|
276 try: |
69aeaaaf6e07
store: simplify class hierarchy
Matt Mackall <mpm@selenic.com>
parents:
6897
diff
changeset
|
277 # files in .hg/ will be created using this mode |
69aeaaaf6e07
store: simplify class hierarchy
Matt Mackall <mpm@selenic.com>
parents:
6897
diff
changeset
|
278 mode = os.stat(path).st_mode |
69aeaaaf6e07
store: simplify class hierarchy
Matt Mackall <mpm@selenic.com>
parents:
6897
diff
changeset
|
279 # avoid some useless chmods |
7890
e710f0f592b2
util: split out posix, windows, and win32 modules
Matt Mackall <mpm@selenic.com>
parents:
7515
diff
changeset
|
280 if (0777 & ~util.umask) == (0777 & mode): |
6898
69aeaaaf6e07
store: simplify class hierarchy
Matt Mackall <mpm@selenic.com>
parents:
6897
diff
changeset
|
281 mode = None |
69aeaaaf6e07
store: simplify class hierarchy
Matt Mackall <mpm@selenic.com>
parents:
6897
diff
changeset
|
282 except OSError: |
69aeaaaf6e07
store: simplify class hierarchy
Matt Mackall <mpm@selenic.com>
parents:
6897
diff
changeset
|
283 mode = None |
69aeaaaf6e07
store: simplify class hierarchy
Matt Mackall <mpm@selenic.com>
parents:
6897
diff
changeset
|
284 return mode |
69aeaaaf6e07
store: simplify class hierarchy
Matt Mackall <mpm@selenic.com>
parents:
6897
diff
changeset
|
285 |
17249
7d4747c711a9
clone: copy obsolete markers during local clone
Pierre-Yves.David@ens-lyon.org
parents:
17154
diff
changeset
|
286 _data = ('data 00manifest.d 00manifest.i 00changelog.d 00changelog.i' |
7d4747c711a9
clone: copy obsolete markers during local clone
Pierre-Yves.David@ens-lyon.org
parents:
17154
diff
changeset
|
287 ' phaseroots obsstore') |
6903
0642d9d7ec80
clone: get a list of files to clone from store
Matt Mackall <mpm@selenic.com>
parents:
6902
diff
changeset
|
288 |
8778
c5f36402daad
use new style classes
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8633
diff
changeset
|
289 class basicstore(object): |
6840
80e51429cb9a
introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents:
6839
diff
changeset
|
290 '''base class for local repository stores''' |
14092
222c8ec7a274
store: rename the 'opener' argument to 'openertype'
Dan Villiom Podlaski Christiansen <danchr@gmail.com>
parents:
14090
diff
changeset
|
291 def __init__(self, path, openertype): |
6840
80e51429cb9a
introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents:
6839
diff
changeset
|
292 self.path = path |
6898
69aeaaaf6e07
store: simplify class hierarchy
Matt Mackall <mpm@selenic.com>
parents:
6897
diff
changeset
|
293 self.createmode = _calcmode(path) |
14092
222c8ec7a274
store: rename the 'opener' argument to 'openertype'
Dan Villiom Podlaski Christiansen <danchr@gmail.com>
parents:
14090
diff
changeset
|
294 op = openertype(self.path) |
8633
c31fe74a6633
store encoding: .i/.d encoding for non-store repo (broken by 810387f59696)
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8531
diff
changeset
|
295 op.createmode = self.createmode |
14090
e24b5e3c2f27
add filteropener abstraction for store openers
Dan Villiom Podlaski Christiansen <danchr@gmail.com>
parents:
13949
diff
changeset
|
296 self.opener = scmutil.filteropener(op, encodedir) |
6840
80e51429cb9a
introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents:
6839
diff
changeset
|
297 |
80e51429cb9a
introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents:
6839
diff
changeset
|
298 def join(self, f): |
13426
643b8212813e
store: remove pointless pathjoiner parameter
Adrian Buehlmann <adrian@cadifra.com>
parents:
13391
diff
changeset
|
299 return self.path + '/' + encodedir(f) |
6840
80e51429cb9a
introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents:
6839
diff
changeset
|
300 |
6899 | 301 def _walk(self, relpath, recurse): |
6900
def492d1b592
store: change handling of decoding errors
Matt Mackall <mpm@selenic.com>
parents:
6899
diff
changeset
|
302 '''yields (unencoded, encoded, size)''' |
13426
643b8212813e
store: remove pointless pathjoiner parameter
Adrian Buehlmann <adrian@cadifra.com>
parents:
13391
diff
changeset
|
303 path = self.path |
643b8212813e
store: remove pointless pathjoiner parameter
Adrian Buehlmann <adrian@cadifra.com>
parents:
13391
diff
changeset
|
304 if relpath: |
643b8212813e
store: remove pointless pathjoiner parameter
Adrian Buehlmann <adrian@cadifra.com>
parents:
13391
diff
changeset
|
305 path += '/' + relpath |
643b8212813e
store: remove pointless pathjoiner parameter
Adrian Buehlmann <adrian@cadifra.com>
parents:
13391
diff
changeset
|
306 striplen = len(self.path) + 1 |
6899 | 307 l = [] |
308 if os.path.isdir(path): | |
309 visit = [path] | |
310 while visit: | |
311 p = visit.pop() | |
312 for f, kind, st in osutil.listdir(p, stat=True): | |
13426
643b8212813e
store: remove pointless pathjoiner parameter
Adrian Buehlmann <adrian@cadifra.com>
parents:
13391
diff
changeset
|
313 fp = p + '/' + f |
6899 | 314 if kind == stat.S_IFREG and f[-2:] in ('.d', '.i'): |
6900
def492d1b592
store: change handling of decoding errors
Matt Mackall <mpm@selenic.com>
parents:
6899
diff
changeset
|
315 n = util.pconvert(fp[striplen:]) |
8531
810387f59696
filelog encoding: move the encoding/decoding into store
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8530
diff
changeset
|
316 l.append((decodedir(n), n, st.st_size)) |
6899 | 317 elif kind == stat.S_IFDIR and recurse: |
318 visit.append(fp) | |
17054
125ff5654b72
store: sort filenames in place
Bryan O'Sullivan <bryano@fb.com>
parents:
16404
diff
changeset
|
319 l.sort() |
125ff5654b72
store: sort filenames in place
Bryan O'Sullivan <bryano@fb.com>
parents:
16404
diff
changeset
|
320 return l |
6840
80e51429cb9a
introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents:
6839
diff
changeset
|
321 |
6900
def492d1b592
store: change handling of decoding errors
Matt Mackall <mpm@selenic.com>
parents:
6899
diff
changeset
|
322 def datafiles(self): |
6899 | 323 return self._walk('data', True) |
6840
80e51429cb9a
introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents:
6839
diff
changeset
|
324 |
80e51429cb9a
introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents:
6839
diff
changeset
|
325 def walk(self): |
6900
def492d1b592
store: change handling of decoding errors
Matt Mackall <mpm@selenic.com>
parents:
6899
diff
changeset
|
326 '''yields (unencoded, encoded, size)''' |
6840
80e51429cb9a
introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents:
6839
diff
changeset
|
327 # yield data files first |
6892
dab95717058d
verify: check repo.store
Adrian Buehlmann <adrian@cadifra.com>
parents:
6890
diff
changeset
|
328 for x in self.datafiles(): |
6840
80e51429cb9a
introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents:
6839
diff
changeset
|
329 yield x |
80e51429cb9a
introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents:
6839
diff
changeset
|
330 # yield manifest before changelog |
8210
344751cd8cb8
replace various uses of list.reverse()
Matt Mackall <mpm@selenic.com>
parents:
8209
diff
changeset
|
331 for x in reversed(self._walk('', False)): |
6840
80e51429cb9a
introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents:
6839
diff
changeset
|
332 yield x |
80e51429cb9a
introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents:
6839
diff
changeset
|
333 |
6903
0642d9d7ec80
clone: get a list of files to clone from store
Matt Mackall <mpm@selenic.com>
parents:
6902
diff
changeset
|
334 def copylist(self): |
0642d9d7ec80
clone: get a list of files to clone from store
Matt Mackall <mpm@selenic.com>
parents:
6902
diff
changeset
|
335 return ['requires'] + _data.split() |
0642d9d7ec80
clone: get a list of files to clone from store
Matt Mackall <mpm@selenic.com>
parents:
6902
diff
changeset
|
336 |
13391
d00bbff8600e
fncachestore: defer updating the fncache file to a single file open
Adrian Buehlmann <adrian@cadifra.com>
parents:
13169
diff
changeset
|
337 def write(self): |
d00bbff8600e
fncachestore: defer updating the fncache file to a single file open
Adrian Buehlmann <adrian@cadifra.com>
parents:
13169
diff
changeset
|
338 pass |
d00bbff8600e
fncachestore: defer updating the fncache file to a single file open
Adrian Buehlmann <adrian@cadifra.com>
parents:
13169
diff
changeset
|
339 |
6898
69aeaaaf6e07
store: simplify class hierarchy
Matt Mackall <mpm@selenic.com>
parents:
6897
diff
changeset
|
340 class encodedstore(basicstore): |
14092
222c8ec7a274
store: rename the 'opener' argument to 'openertype'
Dan Villiom Podlaski Christiansen <danchr@gmail.com>
parents:
14090
diff
changeset
|
341 def __init__(self, path, openertype): |
13426
643b8212813e
store: remove pointless pathjoiner parameter
Adrian Buehlmann <adrian@cadifra.com>
parents:
13391
diff
changeset
|
342 self.path = path + '/store' |
6898
69aeaaaf6e07
store: simplify class hierarchy
Matt Mackall <mpm@selenic.com>
parents:
6897
diff
changeset
|
343 self.createmode = _calcmode(self.path) |
14092
222c8ec7a274
store: rename the 'opener' argument to 'openertype'
Dan Villiom Podlaski Christiansen <danchr@gmail.com>
parents:
14090
diff
changeset
|
344 op = openertype(self.path) |
6840
80e51429cb9a
introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents:
6839
diff
changeset
|
345 op.createmode = self.createmode |
14090
e24b5e3c2f27
add filteropener abstraction for store openers
Dan Villiom Podlaski Christiansen <danchr@gmail.com>
parents:
13949
diff
changeset
|
346 self.opener = scmutil.filteropener(op, encodefilename) |
6840
80e51429cb9a
introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents:
6839
diff
changeset
|
347 |
6900
def492d1b592
store: change handling of decoding errors
Matt Mackall <mpm@selenic.com>
parents:
6899
diff
changeset
|
348 def datafiles(self): |
def492d1b592
store: change handling of decoding errors
Matt Mackall <mpm@selenic.com>
parents:
6899
diff
changeset
|
349 for a, b, size in self._walk('data', True): |
6892
dab95717058d
verify: check repo.store
Adrian Buehlmann <adrian@cadifra.com>
parents:
6890
diff
changeset
|
350 try: |
6900
def492d1b592
store: change handling of decoding errors
Matt Mackall <mpm@selenic.com>
parents:
6899
diff
changeset
|
351 a = decodefilename(a) |
6892
dab95717058d
verify: check repo.store
Adrian Buehlmann <adrian@cadifra.com>
parents:
6890
diff
changeset
|
352 except KeyError: |
6900
def492d1b592
store: change handling of decoding errors
Matt Mackall <mpm@selenic.com>
parents:
6899
diff
changeset
|
353 a = None |
def492d1b592
store: change handling of decoding errors
Matt Mackall <mpm@selenic.com>
parents:
6899
diff
changeset
|
354 yield a, b, size |
6840
80e51429cb9a
introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents:
6839
diff
changeset
|
355 |
80e51429cb9a
introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents:
6839
diff
changeset
|
356 def join(self, f): |
13426
643b8212813e
store: remove pointless pathjoiner parameter
Adrian Buehlmann <adrian@cadifra.com>
parents:
13391
diff
changeset
|
357 return self.path + '/' + encodefilename(f) |
6840
80e51429cb9a
introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents:
6839
diff
changeset
|
358 |
6903
0642d9d7ec80
clone: get a list of files to clone from store
Matt Mackall <mpm@selenic.com>
parents:
6902
diff
changeset
|
359 def copylist(self): |
0642d9d7ec80
clone: get a list of files to clone from store
Matt Mackall <mpm@selenic.com>
parents:
6902
diff
changeset
|
360 return (['requires', '00changelog.i'] + |
13426
643b8212813e
store: remove pointless pathjoiner parameter
Adrian Buehlmann <adrian@cadifra.com>
parents:
13391
diff
changeset
|
361 ['store/' + f for f in _data.split()]) |
6903
0642d9d7ec80
clone: get a list of files to clone from store
Matt Mackall <mpm@selenic.com>
parents:
6902
diff
changeset
|
362 |
8530
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
363 class fncache(object): |
8531
810387f59696
filelog encoding: move the encoding/decoding into store
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8530
diff
changeset
|
364 # the filename used to be partially encoded |
810387f59696
filelog encoding: move the encoding/decoding into store
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8530
diff
changeset
|
365 # hence the encodedir/decodedir dance |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
366 def __init__(self, opener): |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
367 self.opener = opener |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
368 self.entries = None |
13391
d00bbff8600e
fncachestore: defer updating the fncache file to a single file open
Adrian Buehlmann <adrian@cadifra.com>
parents:
13169
diff
changeset
|
369 self._dirty = False |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
370 |
8530
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
371 def _load(self): |
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
372 '''fill the entries from the fncache file''' |
13391
d00bbff8600e
fncachestore: defer updating the fncache file to a single file open
Adrian Buehlmann <adrian@cadifra.com>
parents:
13169
diff
changeset
|
373 self._dirty = False |
8530
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
374 try: |
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
375 fp = self.opener('fncache', mode='rb') |
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
376 except IOError: |
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
377 # skip nonexistent file |
16404
9fca5b056c0a
store: speed up read and write of large fncache files
Bryan O'Sullivan <bryano@fb.com>
parents:
15742
diff
changeset
|
378 self.entries = set() |
8530
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
379 return |
17604
bf9894889d82
store: optimize fncache._load a bit by dirdecoding the contents in one go
Adrian Buehlmann <adrian@cadifra.com>
parents:
17592
diff
changeset
|
380 self.entries = set(decodedir(fp.read()).splitlines()) |
16404
9fca5b056c0a
store: speed up read and write of large fncache files
Bryan O'Sullivan <bryano@fb.com>
parents:
15742
diff
changeset
|
381 if '' in self.entries: |
9fca5b056c0a
store: speed up read and write of large fncache files
Bryan O'Sullivan <bryano@fb.com>
parents:
15742
diff
changeset
|
382 fp.seek(0) |
9fca5b056c0a
store: speed up read and write of large fncache files
Bryan O'Sullivan <bryano@fb.com>
parents:
15742
diff
changeset
|
383 for n, line in enumerate(fp): |
9fca5b056c0a
store: speed up read and write of large fncache files
Bryan O'Sullivan <bryano@fb.com>
parents:
15742
diff
changeset
|
384 if not line.rstrip('\n'): |
9fca5b056c0a
store: speed up read and write of large fncache files
Bryan O'Sullivan <bryano@fb.com>
parents:
15742
diff
changeset
|
385 t = _('invalid entry in fncache, line %s') % (n + 1) |
9fca5b056c0a
store: speed up read and write of large fncache files
Bryan O'Sullivan <bryano@fb.com>
parents:
15742
diff
changeset
|
386 raise util.Abort(t) |
8530
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
387 fp.close() |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
388 |
16404
9fca5b056c0a
store: speed up read and write of large fncache files
Bryan O'Sullivan <bryano@fb.com>
parents:
15742
diff
changeset
|
389 def _write(self, files, atomictemp): |
9fca5b056c0a
store: speed up read and write of large fncache files
Bryan O'Sullivan <bryano@fb.com>
parents:
15742
diff
changeset
|
390 fp = self.opener('fncache', mode='wb', atomictemp=atomictemp) |
9fca5b056c0a
store: speed up read and write of large fncache files
Bryan O'Sullivan <bryano@fb.com>
parents:
15742
diff
changeset
|
391 if files: |
17592
64c6a0d4d4bd
store: optimize fncache._write by direncoding the contents in one go
Adrian Buehlmann <adrian@cadifra.com>
parents:
17591
diff
changeset
|
392 fp.write(encodedir('\n'.join(files) + '\n')) |
16404
9fca5b056c0a
store: speed up read and write of large fncache files
Bryan O'Sullivan <bryano@fb.com>
parents:
15742
diff
changeset
|
393 fp.close() |
9fca5b056c0a
store: speed up read and write of large fncache files
Bryan O'Sullivan <bryano@fb.com>
parents:
15742
diff
changeset
|
394 self._dirty = False |
9fca5b056c0a
store: speed up read and write of large fncache files
Bryan O'Sullivan <bryano@fb.com>
parents:
15742
diff
changeset
|
395 |
8530
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
396 def rewrite(self, files): |
16404
9fca5b056c0a
store: speed up read and write of large fncache files
Bryan O'Sullivan <bryano@fb.com>
parents:
15742
diff
changeset
|
397 self._write(files, False) |
8530
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
398 self.entries = set(files) |
13391
d00bbff8600e
fncachestore: defer updating the fncache file to a single file open
Adrian Buehlmann <adrian@cadifra.com>
parents:
13169
diff
changeset
|
399 |
d00bbff8600e
fncachestore: defer updating the fncache file to a single file open
Adrian Buehlmann <adrian@cadifra.com>
parents:
13169
diff
changeset
|
400 def write(self): |
16404
9fca5b056c0a
store: speed up read and write of large fncache files
Bryan O'Sullivan <bryano@fb.com>
parents:
15742
diff
changeset
|
401 if self._dirty: |
9fca5b056c0a
store: speed up read and write of large fncache files
Bryan O'Sullivan <bryano@fb.com>
parents:
15742
diff
changeset
|
402 self._write(self.entries, True) |
8530
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
403 |
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
404 def add(self, fn): |
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
405 if self.entries is None: |
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
406 self._load() |
10577
d5bd1beff794
store: only add new entries to the fncache file
Adrian Buehlmann <adrian@cadifra.com>
parents:
10339
diff
changeset
|
407 if fn not in self.entries: |
13391
d00bbff8600e
fncachestore: defer updating the fncache file to a single file open
Adrian Buehlmann <adrian@cadifra.com>
parents:
13169
diff
changeset
|
408 self._dirty = True |
10577
d5bd1beff794
store: only add new entries to the fncache file
Adrian Buehlmann <adrian@cadifra.com>
parents:
10339
diff
changeset
|
409 self.entries.add(fn) |
8530
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
410 |
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
411 def __contains__(self, fn): |
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
412 if self.entries is None: |
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
413 self._load() |
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
414 return fn in self.entries |
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
415 |
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
416 def __iter__(self): |
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
417 if self.entries is None: |
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
418 self._load() |
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
419 return iter(self.entries) |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
420 |
14194
3a90fb3addc1
store: break up reference cycle introduced in 9cbff8a39a2a
Adrian Buehlmann <adrian@cadifra.com>
parents:
14166
diff
changeset
|
421 class _fncacheopener(scmutil.abstractopener): |
3a90fb3addc1
store: break up reference cycle introduced in 9cbff8a39a2a
Adrian Buehlmann <adrian@cadifra.com>
parents:
14166
diff
changeset
|
422 def __init__(self, op, fnc, encode): |
3a90fb3addc1
store: break up reference cycle introduced in 9cbff8a39a2a
Adrian Buehlmann <adrian@cadifra.com>
parents:
14166
diff
changeset
|
423 self.opener = op |
3a90fb3addc1
store: break up reference cycle introduced in 9cbff8a39a2a
Adrian Buehlmann <adrian@cadifra.com>
parents:
14166
diff
changeset
|
424 self.fncache = fnc |
3a90fb3addc1
store: break up reference cycle introduced in 9cbff8a39a2a
Adrian Buehlmann <adrian@cadifra.com>
parents:
14166
diff
changeset
|
425 self.encode = encode |
3a90fb3addc1
store: break up reference cycle introduced in 9cbff8a39a2a
Adrian Buehlmann <adrian@cadifra.com>
parents:
14166
diff
changeset
|
426 |
17555
57eba8158736
scmutil: delegate mustaudit property to the real opener
Bryan O'Sullivan <bryano@fb.com>
parents:
17374
diff
changeset
|
427 def _getmustaudit(self): |
57eba8158736
scmutil: delegate mustaudit property to the real opener
Bryan O'Sullivan <bryano@fb.com>
parents:
17374
diff
changeset
|
428 return self.opener.mustaudit |
57eba8158736
scmutil: delegate mustaudit property to the real opener
Bryan O'Sullivan <bryano@fb.com>
parents:
17374
diff
changeset
|
429 |
57eba8158736
scmutil: delegate mustaudit property to the real opener
Bryan O'Sullivan <bryano@fb.com>
parents:
17374
diff
changeset
|
430 def _setmustaudit(self, onoff): |
57eba8158736
scmutil: delegate mustaudit property to the real opener
Bryan O'Sullivan <bryano@fb.com>
parents:
17374
diff
changeset
|
431 self.opener.mustaudit = onoff |
57eba8158736
scmutil: delegate mustaudit property to the real opener
Bryan O'Sullivan <bryano@fb.com>
parents:
17374
diff
changeset
|
432 |
57eba8158736
scmutil: delegate mustaudit property to the real opener
Bryan O'Sullivan <bryano@fb.com>
parents:
17374
diff
changeset
|
433 mustaudit = property(_getmustaudit, _setmustaudit) |
57eba8158736
scmutil: delegate mustaudit property to the real opener
Bryan O'Sullivan <bryano@fb.com>
parents:
17374
diff
changeset
|
434 |
14194
3a90fb3addc1
store: break up reference cycle introduced in 9cbff8a39a2a
Adrian Buehlmann <adrian@cadifra.com>
parents:
14166
diff
changeset
|
435 def __call__(self, path, mode='r', *args, **kw): |
3a90fb3addc1
store: break up reference cycle introduced in 9cbff8a39a2a
Adrian Buehlmann <adrian@cadifra.com>
parents:
14166
diff
changeset
|
436 if mode not in ('r', 'rb') and path.startswith('data/'): |
3a90fb3addc1
store: break up reference cycle introduced in 9cbff8a39a2a
Adrian Buehlmann <adrian@cadifra.com>
parents:
14166
diff
changeset
|
437 self.fncache.add(path) |
3a90fb3addc1
store: break up reference cycle introduced in 9cbff8a39a2a
Adrian Buehlmann <adrian@cadifra.com>
parents:
14166
diff
changeset
|
438 return self.opener(self.encode(path), mode, *args, **kw) |
3a90fb3addc1
store: break up reference cycle introduced in 9cbff8a39a2a
Adrian Buehlmann <adrian@cadifra.com>
parents:
14166
diff
changeset
|
439 |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
440 class fncachestore(basicstore): |
17591
9a5c2ecd1158
store: move encode lambda logic into fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents:
17590
diff
changeset
|
441 def __init__(self, path, openertype, dotencode): |
9a5c2ecd1158
store: move encode lambda logic into fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents:
17590
diff
changeset
|
442 if dotencode: |
9a5c2ecd1158
store: move encode lambda logic into fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents:
17590
diff
changeset
|
443 encode = _dothybridencode |
9a5c2ecd1158
store: move encode lambda logic into fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents:
17590
diff
changeset
|
444 else: |
9a5c2ecd1158
store: move encode lambda logic into fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents:
17590
diff
changeset
|
445 encode = _plainhybridencode |
12687
34d8247a4595
store: encode first period or space in filenames (issue1713)
Adrian Buehlmann <adrian@cadifra.com>
parents:
12171
diff
changeset
|
446 self.encode = encode |
13426
643b8212813e
store: remove pointless pathjoiner parameter
Adrian Buehlmann <adrian@cadifra.com>
parents:
13391
diff
changeset
|
447 self.path = path + '/store' |
17562
b42b0729744d
store: reduce string concatenation when joining
Bryan O'Sullivan <bryano@fb.com>
parents:
17555
diff
changeset
|
448 self.pathsep = self.path + '/' |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
449 self.createmode = _calcmode(self.path) |
14194
3a90fb3addc1
store: break up reference cycle introduced in 9cbff8a39a2a
Adrian Buehlmann <adrian@cadifra.com>
parents:
14166
diff
changeset
|
450 op = openertype(self.path) |
9133
996c1cd8f530
store: eliminate reference cycle in fncachestore
Simon Heimberg <simohe@besonet.ch>
parents:
8778
diff
changeset
|
451 op.createmode = self.createmode |
996c1cd8f530
store: eliminate reference cycle in fncachestore
Simon Heimberg <simohe@besonet.ch>
parents:
8778
diff
changeset
|
452 fnc = fncache(op) |
996c1cd8f530
store: eliminate reference cycle in fncachestore
Simon Heimberg <simohe@besonet.ch>
parents:
8778
diff
changeset
|
453 self.fncache = fnc |
14194
3a90fb3addc1
store: break up reference cycle introduced in 9cbff8a39a2a
Adrian Buehlmann <adrian@cadifra.com>
parents:
14166
diff
changeset
|
454 self.opener = _fncacheopener(op, fnc, encode) |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
455 |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
456 def join(self, f): |
17562
b42b0729744d
store: reduce string concatenation when joining
Bryan O'Sullivan <bryano@fb.com>
parents:
17555
diff
changeset
|
457 return self.pathsep + self.encode(f) |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
458 |
17154
d592759aabc7
store: abstract out how we retrieve a file's size
Bryan O'Sullivan <bryano@fb.com>
parents:
17054
diff
changeset
|
459 def getsize(self, path): |
17562
b42b0729744d
store: reduce string concatenation when joining
Bryan O'Sullivan <bryano@fb.com>
parents:
17555
diff
changeset
|
460 return os.stat(self.pathsep + path).st_size |
17154
d592759aabc7
store: abstract out how we retrieve a file's size
Bryan O'Sullivan <bryano@fb.com>
parents:
17054
diff
changeset
|
461 |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
462 def datafiles(self): |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
463 rewrite = False |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
464 existing = [] |
17373
4cbb1137941d
store: sort the results of fncachestore.datafiles()
Bryan O'Sullivan <bryano@fb.com>
parents:
17249
diff
changeset
|
465 for f in sorted(self.fncache): |
12687
34d8247a4595
store: encode first period or space in filenames (issue1713)
Adrian Buehlmann <adrian@cadifra.com>
parents:
12171
diff
changeset
|
466 ef = self.encode(f) |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
467 try: |
17154
d592759aabc7
store: abstract out how we retrieve a file's size
Bryan O'Sullivan <bryano@fb.com>
parents:
17054
diff
changeset
|
468 yield f, ef, self.getsize(ef) |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
469 existing.append(f) |
17374
0cec762790ed
store: only one kind of OSError means "nonexistent entry"
Bryan O'Sullivan <bryano@fb.com>
parents:
17373
diff
changeset
|
470 except OSError, err: |
0cec762790ed
store: only one kind of OSError means "nonexistent entry"
Bryan O'Sullivan <bryano@fb.com>
parents:
17373
diff
changeset
|
471 if err.errno != errno.ENOENT: |
0cec762790ed
store: only one kind of OSError means "nonexistent entry"
Bryan O'Sullivan <bryano@fb.com>
parents:
17373
diff
changeset
|
472 raise |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
473 # nonexistent entry |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
474 rewrite = True |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
475 if rewrite: |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
476 # rewrite fncache to remove nonexistent entries |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
477 # (may be caused by rollback / strip) |
8530
03196ac9a8b9
store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8480
diff
changeset
|
478 self.fncache.rewrite(existing) |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
479 |
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
480 def copylist(self): |
17249
7d4747c711a9
clone: copy obsolete markers during local clone
Pierre-Yves.David@ens-lyon.org
parents:
17154
diff
changeset
|
481 d = ('data dh fncache phaseroots obsstore' |
15742
65df60a3f96b
phases: prevent rebase to rebase immutable changeset.
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents:
15740
diff
changeset
|
482 ' 00manifest.d 00manifest.i 00changelog.d 00changelog.i') |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
483 return (['requires', '00changelog.i'] + |
13426
643b8212813e
store: remove pointless pathjoiner parameter
Adrian Buehlmann <adrian@cadifra.com>
parents:
13391
diff
changeset
|
484 ['store/' + f for f in d.split()]) |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
485 |
13391
d00bbff8600e
fncachestore: defer updating the fncache file to a single file open
Adrian Buehlmann <adrian@cadifra.com>
parents:
13169
diff
changeset
|
486 def write(self): |
d00bbff8600e
fncachestore: defer updating the fncache file to a single file open
Adrian Buehlmann <adrian@cadifra.com>
parents:
13169
diff
changeset
|
487 self.fncache.write() |
d00bbff8600e
fncachestore: defer updating the fncache file to a single file open
Adrian Buehlmann <adrian@cadifra.com>
parents:
13169
diff
changeset
|
488 |
14092
222c8ec7a274
store: rename the 'opener' argument to 'openertype'
Dan Villiom Podlaski Christiansen <danchr@gmail.com>
parents:
14090
diff
changeset
|
489 def store(requirements, path, openertype): |
6898
69aeaaaf6e07
store: simplify class hierarchy
Matt Mackall <mpm@selenic.com>
parents:
6897
diff
changeset
|
490 if 'store' in requirements: |
7229
7946503ec76e
introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents:
6989
diff
changeset
|
491 if 'fncache' in requirements: |
17591
9a5c2ecd1158
store: move encode lambda logic into fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents:
17590
diff
changeset
|
492 return fncachestore(path, openertype, 'dotencode' in requirements) |
14092
222c8ec7a274
store: rename the 'opener' argument to 'openertype'
Dan Villiom Podlaski Christiansen <danchr@gmail.com>
parents:
14090
diff
changeset
|
493 return encodedstore(path, openertype) |
222c8ec7a274
store: rename the 'opener' argument to 'openertype'
Dan Villiom Podlaski Christiansen <danchr@gmail.com>
parents:
14090
diff
changeset
|
494 return basicstore(path, openertype) |