author | Boris Feld <boris.feld@octobus.net> |
Sat, 22 Sep 2018 14:40:33 +0200 | |
changeset 39927 | 6335c0de80fa |
parent 39886 | d63153611ed5 |
child 40003 | 0e8836be9541 |
permissions | -rw-r--r-- |
39882
f8eb71f9e3bd
storageutil: new module for storage primitives (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
1 |
# storageutil.py - Storage functionality agnostic of backend implementation. |
f8eb71f9e3bd
storageutil: new module for storage primitives (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
2 |
# |
f8eb71f9e3bd
storageutil: new module for storage primitives (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
3 |
# Copyright 2018 Gregory Szorc <gregory.szorc@gmail.com> |
f8eb71f9e3bd
storageutil: new module for storage primitives (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
4 |
# |
f8eb71f9e3bd
storageutil: new module for storage primitives (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
5 |
# This software may be used and distributed according to the terms of the |
f8eb71f9e3bd
storageutil: new module for storage primitives (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
6 |
# GNU General Public License version 2 or any later version. |
f8eb71f9e3bd
storageutil: new module for storage primitives (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
7 |
|
f8eb71f9e3bd
storageutil: new module for storage primitives (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
8 |
from __future__ import absolute_import |
f8eb71f9e3bd
storageutil: new module for storage primitives (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
9 |
|
f8eb71f9e3bd
storageutil: new module for storage primitives (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
10 |
import hashlib |
39883
3e896b51aa5d
storageutil: move metadata parsing and packing from revlog (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39882
diff
changeset
|
11 |
import re |
39882
f8eb71f9e3bd
storageutil: new module for storage primitives (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
12 |
|
f8eb71f9e3bd
storageutil: new module for storage primitives (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
13 |
from ..node import ( |
f8eb71f9e3bd
storageutil: new module for storage primitives (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
14 |
nullid, |
f8eb71f9e3bd
storageutil: new module for storage primitives (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
15 |
) |
39886
d63153611ed5
storageutil: extract revision number iteration
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39885
diff
changeset
|
16 |
from .. import ( |
d63153611ed5
storageutil: extract revision number iteration
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39885
diff
changeset
|
17 |
pycompat, |
d63153611ed5
storageutil: extract revision number iteration
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39885
diff
changeset
|
18 |
) |
39882
f8eb71f9e3bd
storageutil: new module for storage primitives (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
19 |
|
f8eb71f9e3bd
storageutil: new module for storage primitives (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
20 |
_nullhash = hashlib.sha1(nullid) |
f8eb71f9e3bd
storageutil: new module for storage primitives (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
21 |
|
f8eb71f9e3bd
storageutil: new module for storage primitives (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
22 |
def hashrevisionsha1(text, p1, p2): |
f8eb71f9e3bd
storageutil: new module for storage primitives (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
23 |
"""Compute the SHA-1 for revision data and its parents. |
f8eb71f9e3bd
storageutil: new module for storage primitives (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
24 |
|
f8eb71f9e3bd
storageutil: new module for storage primitives (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
25 |
This hash combines both the current file contents and its history |
f8eb71f9e3bd
storageutil: new module for storage primitives (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
26 |
in a manner that makes it easy to distinguish nodes with the same |
f8eb71f9e3bd
storageutil: new module for storage primitives (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
27 |
content in the revision graph. |
f8eb71f9e3bd
storageutil: new module for storage primitives (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
28 |
""" |
f8eb71f9e3bd
storageutil: new module for storage primitives (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
29 |
# As of now, if one of the parent node is null, p2 is null |
f8eb71f9e3bd
storageutil: new module for storage primitives (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
30 |
if p2 == nullid: |
f8eb71f9e3bd
storageutil: new module for storage primitives (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
31 |
# deep copy of a hash is faster than creating one |
f8eb71f9e3bd
storageutil: new module for storage primitives (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
32 |
s = _nullhash.copy() |
f8eb71f9e3bd
storageutil: new module for storage primitives (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
33 |
s.update(p1) |
f8eb71f9e3bd
storageutil: new module for storage primitives (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
34 |
else: |
f8eb71f9e3bd
storageutil: new module for storage primitives (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
35 |
# none of the parent nodes are nullid |
f8eb71f9e3bd
storageutil: new module for storage primitives (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
36 |
if p1 < p2: |
f8eb71f9e3bd
storageutil: new module for storage primitives (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
37 |
a = p1 |
f8eb71f9e3bd
storageutil: new module for storage primitives (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
38 |
b = p2 |
f8eb71f9e3bd
storageutil: new module for storage primitives (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
39 |
else: |
f8eb71f9e3bd
storageutil: new module for storage primitives (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
40 |
a = p2 |
f8eb71f9e3bd
storageutil: new module for storage primitives (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
41 |
b = p1 |
f8eb71f9e3bd
storageutil: new module for storage primitives (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
42 |
s = hashlib.sha1(a) |
f8eb71f9e3bd
storageutil: new module for storage primitives (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
43 |
s.update(b) |
f8eb71f9e3bd
storageutil: new module for storage primitives (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
44 |
s.update(text) |
f8eb71f9e3bd
storageutil: new module for storage primitives (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
45 |
return s.digest() |
39883
3e896b51aa5d
storageutil: move metadata parsing and packing from revlog (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39882
diff
changeset
|
46 |
|
3e896b51aa5d
storageutil: move metadata parsing and packing from revlog (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39882
diff
changeset
|
47 |
METADATA_RE = re.compile(b'\x01\n') |
3e896b51aa5d
storageutil: move metadata parsing and packing from revlog (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39882
diff
changeset
|
48 |
|
3e896b51aa5d
storageutil: move metadata parsing and packing from revlog (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39882
diff
changeset
|
49 |
def parsemeta(text): |
3e896b51aa5d
storageutil: move metadata parsing and packing from revlog (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39882
diff
changeset
|
50 |
"""Parse metadata header from revision data. |
3e896b51aa5d
storageutil: move metadata parsing and packing from revlog (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39882
diff
changeset
|
51 |
|
3e896b51aa5d
storageutil: move metadata parsing and packing from revlog (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39882
diff
changeset
|
52 |
Returns a 2-tuple of (metadata, offset), where both can be None if there |
3e896b51aa5d
storageutil: move metadata parsing and packing from revlog (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39882
diff
changeset
|
53 |
is no metadata. |
3e896b51aa5d
storageutil: move metadata parsing and packing from revlog (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39882
diff
changeset
|
54 |
""" |
3e896b51aa5d
storageutil: move metadata parsing and packing from revlog (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39882
diff
changeset
|
55 |
# text can be buffer, so we can't use .startswith or .index |
3e896b51aa5d
storageutil: move metadata parsing and packing from revlog (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39882
diff
changeset
|
56 |
if text[:2] != b'\x01\n': |
3e896b51aa5d
storageutil: move metadata parsing and packing from revlog (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39882
diff
changeset
|
57 |
return None, None |
3e896b51aa5d
storageutil: move metadata parsing and packing from revlog (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39882
diff
changeset
|
58 |
s = METADATA_RE.search(text, 2).start() |
3e896b51aa5d
storageutil: move metadata parsing and packing from revlog (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39882
diff
changeset
|
59 |
mtext = text[2:s] |
3e896b51aa5d
storageutil: move metadata parsing and packing from revlog (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39882
diff
changeset
|
60 |
meta = {} |
3e896b51aa5d
storageutil: move metadata parsing and packing from revlog (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39882
diff
changeset
|
61 |
for l in mtext.splitlines(): |
3e896b51aa5d
storageutil: move metadata parsing and packing from revlog (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39882
diff
changeset
|
62 |
k, v = l.split(b': ', 1) |
3e896b51aa5d
storageutil: move metadata parsing and packing from revlog (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39882
diff
changeset
|
63 |
meta[k] = v |
3e896b51aa5d
storageutil: move metadata parsing and packing from revlog (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39882
diff
changeset
|
64 |
return meta, s + 2 |
3e896b51aa5d
storageutil: move metadata parsing and packing from revlog (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39882
diff
changeset
|
65 |
|
3e896b51aa5d
storageutil: move metadata parsing and packing from revlog (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39882
diff
changeset
|
66 |
def packmeta(meta, text): |
3e896b51aa5d
storageutil: move metadata parsing and packing from revlog (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39882
diff
changeset
|
67 |
"""Add metadata to fulltext to produce revision text.""" |
3e896b51aa5d
storageutil: move metadata parsing and packing from revlog (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39882
diff
changeset
|
68 |
keys = sorted(meta) |
3e896b51aa5d
storageutil: move metadata parsing and packing from revlog (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39882
diff
changeset
|
69 |
metatext = b''.join(b'%s: %s\n' % (k, meta[k]) for k in keys) |
3e896b51aa5d
storageutil: move metadata parsing and packing from revlog (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39882
diff
changeset
|
70 |
return b'\x01\n%s\x01\n%s' % (metatext, text) |
39884
d269ddbf54f0
storageutil: move _censoredtext() from revlog
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39883
diff
changeset
|
71 |
|
d269ddbf54f0
storageutil: move _censoredtext() from revlog
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39883
diff
changeset
|
72 |
def iscensoredtext(text): |
d269ddbf54f0
storageutil: move _censoredtext() from revlog
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39883
diff
changeset
|
73 |
meta = parsemeta(text)[0] |
d269ddbf54f0
storageutil: move _censoredtext() from revlog
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39883
diff
changeset
|
74 |
return meta and b'censored' in meta |
39885
1b65fb4d43d6
storageutil: new function for extracting metadata-less content from text
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39884
diff
changeset
|
75 |
|
1b65fb4d43d6
storageutil: new function for extracting metadata-less content from text
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39884
diff
changeset
|
76 |
def filtermetadata(text): |
1b65fb4d43d6
storageutil: new function for extracting metadata-less content from text
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39884
diff
changeset
|
77 |
"""Extract just the revision data from source text. |
1b65fb4d43d6
storageutil: new function for extracting metadata-less content from text
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39884
diff
changeset
|
78 |
|
1b65fb4d43d6
storageutil: new function for extracting metadata-less content from text
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39884
diff
changeset
|
79 |
Returns ``text`` unless it has a metadata header, in which case we return |
1b65fb4d43d6
storageutil: new function for extracting metadata-less content from text
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39884
diff
changeset
|
80 |
a new buffer without hte metadata. |
1b65fb4d43d6
storageutil: new function for extracting metadata-less content from text
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39884
diff
changeset
|
81 |
""" |
1b65fb4d43d6
storageutil: new function for extracting metadata-less content from text
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39884
diff
changeset
|
82 |
if not text.startswith(b'\x01\n'): |
1b65fb4d43d6
storageutil: new function for extracting metadata-less content from text
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39884
diff
changeset
|
83 |
return text |
1b65fb4d43d6
storageutil: new function for extracting metadata-less content from text
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39884
diff
changeset
|
84 |
|
1b65fb4d43d6
storageutil: new function for extracting metadata-less content from text
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39884
diff
changeset
|
85 |
offset = text.index(b'\x01\n', 2) |
1b65fb4d43d6
storageutil: new function for extracting metadata-less content from text
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39884
diff
changeset
|
86 |
return text[offset + 2:] |
39886
d63153611ed5
storageutil: extract revision number iteration
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39885
diff
changeset
|
87 |
|
d63153611ed5
storageutil: extract revision number iteration
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39885
diff
changeset
|
88 |
def iterrevs(storelen, start=0, stop=None): |
d63153611ed5
storageutil: extract revision number iteration
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39885
diff
changeset
|
89 |
"""Iterate over revision numbers in a store.""" |
d63153611ed5
storageutil: extract revision number iteration
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39885
diff
changeset
|
90 |
step = 1 |
d63153611ed5
storageutil: extract revision number iteration
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39885
diff
changeset
|
91 |
|
d63153611ed5
storageutil: extract revision number iteration
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39885
diff
changeset
|
92 |
if stop is not None: |
d63153611ed5
storageutil: extract revision number iteration
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39885
diff
changeset
|
93 |
if start > stop: |
d63153611ed5
storageutil: extract revision number iteration
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39885
diff
changeset
|
94 |
step = -1 |
d63153611ed5
storageutil: extract revision number iteration
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39885
diff
changeset
|
95 |
stop += step |
d63153611ed5
storageutil: extract revision number iteration
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39885
diff
changeset
|
96 |
if stop > storelen: |
d63153611ed5
storageutil: extract revision number iteration
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39885
diff
changeset
|
97 |
stop = storelen |
d63153611ed5
storageutil: extract revision number iteration
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39885
diff
changeset
|
98 |
else: |
d63153611ed5
storageutil: extract revision number iteration
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39885
diff
changeset
|
99 |
stop = storelen |
d63153611ed5
storageutil: extract revision number iteration
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39885
diff
changeset
|
100 |
|
d63153611ed5
storageutil: extract revision number iteration
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39885
diff
changeset
|
101 |
return pycompat.xrange(start, stop, step) |