Mercurial > hg
annotate mercurial/pure/parsers.py @ 19652:187bf2dde7c1
pack_dirstate: only invalidate mtime for files written in the last second
Previously we'd place files written in the last second in the lookup set. This
can lead to pathological cases where a file always remains in the lookup set if
it gets modified before the next time status is run.
With this patch, only the mtime of those files is invalidated. This means that
if a file's size or mode changes, we can immediately declare it as modified
without needing to compare file contents.
author | Siddharth Agarwal <sid0@fb.com> |
---|---|
date | Sat, 17 Aug 2013 20:48:49 -0700 |
parents | 194e63c1ccb9 |
children | e250b8300e6e |
rev | line source |
---|---|
7700
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
1 # parsers.py - Python implementation of parsers.c |
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
2 # |
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
3 # Copyright 2009 Matt Mackall <mpm@selenic.com> and others |
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
4 # |
8225
46293a0c7e9f
updated license to be explicit about GPL version 2
Martin Geisler <mg@lazybytes.net>
parents:
7945
diff
changeset
|
5 # This software may be used and distributed according to the terms of the |
10263 | 6 # GNU General Public License version 2 or any later version. |
7700
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
7 |
14064
e4bfb9c337f3
remove unused imports and variables
Alexander Solovyov <alexander@solovyov.net>
parents:
13435
diff
changeset
|
8 from mercurial.node import bin, nullid |
7945
94d7e14cfa42
pure/parsers: fix circular imports, import mercurial modules properly
Matt Mackall <mpm@selenic.com>
parents:
7873
diff
changeset
|
9 from mercurial import util |
18567
194e63c1ccb9
dirstate: move pure python dirstate packing to pure/parsers.py
Siddharth Agarwal <sid0@fb.com>
parents:
17425
diff
changeset
|
10 import struct, zlib, cStringIO |
7700
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
11 |
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
12 _pack = struct.pack |
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
13 _unpack = struct.unpack |
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
14 _compress = zlib.compress |
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
15 _decompress = zlib.decompress |
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
16 _sha = util.sha1 |
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
17 |
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
18 def parse_manifest(mfdict, fdict, lines): |
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
19 for l in lines.splitlines(): |
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
20 f, n = l.split('\0') |
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
21 if len(n) > 40: |
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
22 fdict[f] = n[40:] |
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
23 mfdict[f] = bin(n[:40]) |
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
24 else: |
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
25 mfdict[f] = bin(n) |
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
26 |
13261
20a54bdf2328
pure: update index parsing
Matt Mackall <mpm@selenic.com>
parents:
13253
diff
changeset
|
27 def parse_index2(data, inline): |
7945
94d7e14cfa42
pure/parsers: fix circular imports, import mercurial modules properly
Matt Mackall <mpm@selenic.com>
parents:
7873
diff
changeset
|
28 def gettype(q): |
94d7e14cfa42
pure/parsers: fix circular imports, import mercurial modules properly
Matt Mackall <mpm@selenic.com>
parents:
7873
diff
changeset
|
29 return int(q & 0xFFFF) |
94d7e14cfa42
pure/parsers: fix circular imports, import mercurial modules properly
Matt Mackall <mpm@selenic.com>
parents:
7873
diff
changeset
|
30 |
94d7e14cfa42
pure/parsers: fix circular imports, import mercurial modules properly
Matt Mackall <mpm@selenic.com>
parents:
7873
diff
changeset
|
31 def offset_type(offset, type): |
94d7e14cfa42
pure/parsers: fix circular imports, import mercurial modules properly
Matt Mackall <mpm@selenic.com>
parents:
7873
diff
changeset
|
32 return long(long(offset) << 16 | type) |
94d7e14cfa42
pure/parsers: fix circular imports, import mercurial modules properly
Matt Mackall <mpm@selenic.com>
parents:
7873
diff
changeset
|
33 |
94d7e14cfa42
pure/parsers: fix circular imports, import mercurial modules properly
Matt Mackall <mpm@selenic.com>
parents:
7873
diff
changeset
|
34 indexformatng = ">Qiiiiii20s12x" |
94d7e14cfa42
pure/parsers: fix circular imports, import mercurial modules properly
Matt Mackall <mpm@selenic.com>
parents:
7873
diff
changeset
|
35 |
7700
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
36 s = struct.calcsize(indexformatng) |
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
37 index = [] |
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
38 cache = None |
14995
8d928799dab5
parsers: remove redundant 'n' variable in parsers.parse_index2() (issue2935)
py4fun
parents:
14421
diff
changeset
|
39 off = 0 |
13253 | 40 |
7700
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
41 l = len(data) - s |
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
42 append = index.append |
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
43 if inline: |
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
44 cache = (0, data) |
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
45 while off <= l: |
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
46 e = _unpack(indexformatng, data[off:off + s]) |
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
47 append(e) |
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
48 if e[1] < 0: |
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
49 break |
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
50 off += e[1] + s |
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
51 else: |
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
52 while off <= l: |
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
53 e = _unpack(indexformatng, data[off:off + s]) |
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
54 append(e) |
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
55 off += s |
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
56 |
14421
639f26cab2f5
pure parsers: properly detect corrupt index files
Augie Fackler <durin42@gmail.com>
parents:
14064
diff
changeset
|
57 if off != len(data): |
639f26cab2f5
pure parsers: properly detect corrupt index files
Augie Fackler <durin42@gmail.com>
parents:
14064
diff
changeset
|
58 raise ValueError('corrupt index file') |
639f26cab2f5
pure parsers: properly detect corrupt index files
Augie Fackler <durin42@gmail.com>
parents:
14064
diff
changeset
|
59 |
13435
90d7ce986565
pure: fix index parsing on empty repositories
Wagner Bruna <wbruna@softwareexpress.com.br>
parents:
13261
diff
changeset
|
60 if index: |
90d7ce986565
pure: fix index parsing on empty repositories
Wagner Bruna <wbruna@softwareexpress.com.br>
parents:
13261
diff
changeset
|
61 e = list(index[0]) |
90d7ce986565
pure: fix index parsing on empty repositories
Wagner Bruna <wbruna@softwareexpress.com.br>
parents:
13261
diff
changeset
|
62 type = gettype(e[0]) |
90d7ce986565
pure: fix index parsing on empty repositories
Wagner Bruna <wbruna@softwareexpress.com.br>
parents:
13261
diff
changeset
|
63 e[0] = offset_type(0, type) |
90d7ce986565
pure: fix index parsing on empty repositories
Wagner Bruna <wbruna@softwareexpress.com.br>
parents:
13261
diff
changeset
|
64 index[0] = tuple(e) |
7700
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
65 |
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
66 # add the magic null revision at -1 |
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
67 index.append((0, 0, 0, -1, -1, -1, -1, nullid)) |
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
68 |
13261
20a54bdf2328
pure: update index parsing
Matt Mackall <mpm@selenic.com>
parents:
13253
diff
changeset
|
69 return index, cache |
7700
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
70 |
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
71 def parse_dirstate(dmap, copymap, st): |
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
72 parents = [st[:20], st[20: 40]] |
17425
e95ec38f86b0
fix wording and not-completely-trivial spelling errors and bad docstrings
Mads Kiilerich <mads@kiilerich.com>
parents:
14995
diff
changeset
|
73 # dereference fields so they will be local in loop |
7945
94d7e14cfa42
pure/parsers: fix circular imports, import mercurial modules properly
Matt Mackall <mpm@selenic.com>
parents:
7873
diff
changeset
|
74 format = ">cllll" |
94d7e14cfa42
pure/parsers: fix circular imports, import mercurial modules properly
Matt Mackall <mpm@selenic.com>
parents:
7873
diff
changeset
|
75 e_size = struct.calcsize(format) |
7700
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
76 pos1 = 40 |
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
77 l = len(st) |
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
78 |
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
79 # the inner loop |
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
80 while pos1 < l: |
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
81 pos2 = pos1 + e_size |
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
82 e = _unpack(">cllll", st[pos1:pos2]) # a literal here is faster |
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
83 pos1 = pos2 + e[4] |
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
84 f = st[pos2:pos1] |
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
85 if '\0' in f: |
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
86 f, c = f.split('\0') |
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
87 copymap[f] = c |
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
88 dmap[f] = e[:4] |
f410c552fa15
pure Python implementation of parsers.c
Martin Geisler <mg@daimi.au.dk>
parents:
diff
changeset
|
89 return parents |
18567
194e63c1ccb9
dirstate: move pure python dirstate packing to pure/parsers.py
Siddharth Agarwal <sid0@fb.com>
parents:
17425
diff
changeset
|
90 |
194e63c1ccb9
dirstate: move pure python dirstate packing to pure/parsers.py
Siddharth Agarwal <sid0@fb.com>
parents:
17425
diff
changeset
|
91 def pack_dirstate(dmap, copymap, pl, now): |
194e63c1ccb9
dirstate: move pure python dirstate packing to pure/parsers.py
Siddharth Agarwal <sid0@fb.com>
parents:
17425
diff
changeset
|
92 now = int(now) |
194e63c1ccb9
dirstate: move pure python dirstate packing to pure/parsers.py
Siddharth Agarwal <sid0@fb.com>
parents:
17425
diff
changeset
|
93 cs = cStringIO.StringIO() |
194e63c1ccb9
dirstate: move pure python dirstate packing to pure/parsers.py
Siddharth Agarwal <sid0@fb.com>
parents:
17425
diff
changeset
|
94 write = cs.write |
194e63c1ccb9
dirstate: move pure python dirstate packing to pure/parsers.py
Siddharth Agarwal <sid0@fb.com>
parents:
17425
diff
changeset
|
95 write("".join(pl)) |
194e63c1ccb9
dirstate: move pure python dirstate packing to pure/parsers.py
Siddharth Agarwal <sid0@fb.com>
parents:
17425
diff
changeset
|
96 for f, e in dmap.iteritems(): |
194e63c1ccb9
dirstate: move pure python dirstate packing to pure/parsers.py
Siddharth Agarwal <sid0@fb.com>
parents:
17425
diff
changeset
|
97 if e[0] == 'n' and e[3] == now: |
194e63c1ccb9
dirstate: move pure python dirstate packing to pure/parsers.py
Siddharth Agarwal <sid0@fb.com>
parents:
17425
diff
changeset
|
98 # The file was last modified "simultaneously" with the current |
194e63c1ccb9
dirstate: move pure python dirstate packing to pure/parsers.py
Siddharth Agarwal <sid0@fb.com>
parents:
17425
diff
changeset
|
99 # write to dirstate (i.e. within the same second for file- |
194e63c1ccb9
dirstate: move pure python dirstate packing to pure/parsers.py
Siddharth Agarwal <sid0@fb.com>
parents:
17425
diff
changeset
|
100 # systems with a granularity of 1 sec). This commonly happens |
194e63c1ccb9
dirstate: move pure python dirstate packing to pure/parsers.py
Siddharth Agarwal <sid0@fb.com>
parents:
17425
diff
changeset
|
101 # for at least a couple of files on 'update'. |
194e63c1ccb9
dirstate: move pure python dirstate packing to pure/parsers.py
Siddharth Agarwal <sid0@fb.com>
parents:
17425
diff
changeset
|
102 # The user could change the file without changing its size |
19652
187bf2dde7c1
pack_dirstate: only invalidate mtime for files written in the last second
Siddharth Agarwal <sid0@fb.com>
parents:
18567
diff
changeset
|
103 # within the same second. Invalidate the file's mtime in |
18567
194e63c1ccb9
dirstate: move pure python dirstate packing to pure/parsers.py
Siddharth Agarwal <sid0@fb.com>
parents:
17425
diff
changeset
|
104 # dirstate, forcing future 'status' calls to compare the |
19652
187bf2dde7c1
pack_dirstate: only invalidate mtime for files written in the last second
Siddharth Agarwal <sid0@fb.com>
parents:
18567
diff
changeset
|
105 # contents of the file if the size is the same. This prevents |
187bf2dde7c1
pack_dirstate: only invalidate mtime for files written in the last second
Siddharth Agarwal <sid0@fb.com>
parents:
18567
diff
changeset
|
106 # mistakenly treating such files as clean. |
187bf2dde7c1
pack_dirstate: only invalidate mtime for files written in the last second
Siddharth Agarwal <sid0@fb.com>
parents:
18567
diff
changeset
|
107 e = (e[0], e[1], e[2], -1) |
18567
194e63c1ccb9
dirstate: move pure python dirstate packing to pure/parsers.py
Siddharth Agarwal <sid0@fb.com>
parents:
17425
diff
changeset
|
108 dmap[f] = e |
194e63c1ccb9
dirstate: move pure python dirstate packing to pure/parsers.py
Siddharth Agarwal <sid0@fb.com>
parents:
17425
diff
changeset
|
109 |
194e63c1ccb9
dirstate: move pure python dirstate packing to pure/parsers.py
Siddharth Agarwal <sid0@fb.com>
parents:
17425
diff
changeset
|
110 if f in copymap: |
194e63c1ccb9
dirstate: move pure python dirstate packing to pure/parsers.py
Siddharth Agarwal <sid0@fb.com>
parents:
17425
diff
changeset
|
111 f = "%s\0%s" % (f, copymap[f]) |
194e63c1ccb9
dirstate: move pure python dirstate packing to pure/parsers.py
Siddharth Agarwal <sid0@fb.com>
parents:
17425
diff
changeset
|
112 e = _pack(">cllll", e[0], e[1], e[2], e[3], len(f)) |
194e63c1ccb9
dirstate: move pure python dirstate packing to pure/parsers.py
Siddharth Agarwal <sid0@fb.com>
parents:
17425
diff
changeset
|
113 write(e) |
194e63c1ccb9
dirstate: move pure python dirstate packing to pure/parsers.py
Siddharth Agarwal <sid0@fb.com>
parents:
17425
diff
changeset
|
114 write(f) |
194e63c1ccb9
dirstate: move pure python dirstate packing to pure/parsers.py
Siddharth Agarwal <sid0@fb.com>
parents:
17425
diff
changeset
|
115 return cs.getvalue() |