Mercurial > hg
annotate mercurial/utils/cborutil.py @ 50896:b2b8c25f9462
hgwebmod: use sysstr to check for attribute presence
We do not need bytes here.
author | Pierre-Yves David <pierre-yves.david@octobus.net> |
---|---|
date | Wed, 30 Aug 2023 13:28:09 +0200 |
parents | 642e31cb55f0 |
children | f4733654f144 |
rev | line source |
---|---|
37711
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
1 # cborutil.py - CBOR extensions |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
2 # |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
3 # Copyright 2018 Gregory Szorc <gregory.szorc@gmail.com> |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
4 # |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
5 # This software may be used and distributed according to the terms of the |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
6 # GNU General Public License version 2 or any later version. |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
7 |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
8 |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
9 import struct |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
10 |
39456
8d858fbf2759
cbor: teach the encoder to handle python `long` type for Windows
Matt Harbison <matt_harbison@yahoo.com>
parents:
39413
diff
changeset
|
11 |
37711
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
12 # Very short very of RFC 7049... |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
13 # |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
14 # Each item begins with a byte. The 3 high bits of that byte denote the |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
15 # "major type." The lower 5 bits denote the "subtype." Each major type |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
16 # has its own encoding mechanism. |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
17 # |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
18 # Most types have lengths. However, bytestring, string, array, and map |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
19 # can be indefinite length. These are denotes by a subtype with value 31. |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
20 # Sub-components of those types then come afterwards and are terminated |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
21 # by a "break" byte. |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
22 |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
23 MAJOR_TYPE_UINT = 0 |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
24 MAJOR_TYPE_NEGINT = 1 |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
25 MAJOR_TYPE_BYTESTRING = 2 |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
26 MAJOR_TYPE_STRING = 3 |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
27 MAJOR_TYPE_ARRAY = 4 |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
28 MAJOR_TYPE_MAP = 5 |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
29 MAJOR_TYPE_SEMANTIC = 6 |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
30 MAJOR_TYPE_SPECIAL = 7 |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
31 |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
32 SUBTYPE_MASK = 0b00011111 |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
33 |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
34 SUBTYPE_FALSE = 20 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
35 SUBTYPE_TRUE = 21 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
36 SUBTYPE_NULL = 22 |
37711
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
37 SUBTYPE_HALF_FLOAT = 25 |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
38 SUBTYPE_SINGLE_FLOAT = 26 |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
39 SUBTYPE_DOUBLE_FLOAT = 27 |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
40 SUBTYPE_INDEFINITE = 31 |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
41 |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
42 SEMANTIC_TAG_FINITE_SET = 258 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
43 |
37711
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
44 # Indefinite types begin with their major type ORd with information value 31. |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
45 BEGIN_INDEFINITE_BYTESTRING = struct.pack( |
43506
9f70512ae2cf
cleanup: remove pointless r-prefixes on single-quoted strings
Augie Fackler <augie@google.com>
parents:
43117
diff
changeset
|
46 '>B', MAJOR_TYPE_BYTESTRING << 5 | SUBTYPE_INDEFINITE |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
47 ) |
37711
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
48 BEGIN_INDEFINITE_ARRAY = struct.pack( |
43506
9f70512ae2cf
cleanup: remove pointless r-prefixes on single-quoted strings
Augie Fackler <augie@google.com>
parents:
43117
diff
changeset
|
49 '>B', MAJOR_TYPE_ARRAY << 5 | SUBTYPE_INDEFINITE |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
50 ) |
37711
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
51 BEGIN_INDEFINITE_MAP = struct.pack( |
43506
9f70512ae2cf
cleanup: remove pointless r-prefixes on single-quoted strings
Augie Fackler <augie@google.com>
parents:
43117
diff
changeset
|
52 '>B', MAJOR_TYPE_MAP << 5 | SUBTYPE_INDEFINITE |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
53 ) |
37711
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
54 |
43506
9f70512ae2cf
cleanup: remove pointless r-prefixes on single-quoted strings
Augie Fackler <augie@google.com>
parents:
43117
diff
changeset
|
55 ENCODED_LENGTH_1 = struct.Struct('>B') |
9f70512ae2cf
cleanup: remove pointless r-prefixes on single-quoted strings
Augie Fackler <augie@google.com>
parents:
43117
diff
changeset
|
56 ENCODED_LENGTH_2 = struct.Struct('>BB') |
9f70512ae2cf
cleanup: remove pointless r-prefixes on single-quoted strings
Augie Fackler <augie@google.com>
parents:
43117
diff
changeset
|
57 ENCODED_LENGTH_3 = struct.Struct('>BH') |
9f70512ae2cf
cleanup: remove pointless r-prefixes on single-quoted strings
Augie Fackler <augie@google.com>
parents:
43117
diff
changeset
|
58 ENCODED_LENGTH_4 = struct.Struct('>BL') |
9f70512ae2cf
cleanup: remove pointless r-prefixes on single-quoted strings
Augie Fackler <augie@google.com>
parents:
43117
diff
changeset
|
59 ENCODED_LENGTH_5 = struct.Struct('>BQ') |
37711
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
60 |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
61 # The break ends an indefinite length item. |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
62 BREAK = b'\xff' |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
63 BREAK_INT = 255 |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
64 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
65 |
37711
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
66 def encodelength(majortype, length): |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
67 """Obtain a value encoding the major type and its length.""" |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
68 if length < 24: |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
69 return ENCODED_LENGTH_1.pack(majortype << 5 | length) |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
70 elif length < 256: |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
71 return ENCODED_LENGTH_2.pack(majortype << 5 | 24, length) |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
72 elif length < 65536: |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
73 return ENCODED_LENGTH_3.pack(majortype << 5 | 25, length) |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
74 elif length < 4294967296: |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
75 return ENCODED_LENGTH_4.pack(majortype << 5 | 26, length) |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
76 else: |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
77 return ENCODED_LENGTH_5.pack(majortype << 5 | 27, length) |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
78 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
79 |
37711
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
80 def streamencodebytestring(v): |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
81 yield encodelength(MAJOR_TYPE_BYTESTRING, len(v)) |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
82 yield v |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
83 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
84 |
37711
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
85 def streamencodebytestringfromiter(it): |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
86 """Convert an iterator of chunks to an indefinite bytestring. |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
87 |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
88 Given an input that is iterable and each element in the iterator is |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
89 representable as bytes, emit an indefinite length bytestring. |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
90 """ |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
91 yield BEGIN_INDEFINITE_BYTESTRING |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
92 |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
93 for chunk in it: |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
94 yield encodelength(MAJOR_TYPE_BYTESTRING, len(chunk)) |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
95 yield chunk |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
96 |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
97 yield BREAK |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
98 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
99 |
37711
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
100 def streamencodeindefinitebytestring(source, chunksize=65536): |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
101 """Given a large source buffer, emit as an indefinite length bytestring. |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
102 |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
103 This is a generator of chunks constituting the encoded CBOR data. |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
104 """ |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
105 yield BEGIN_INDEFINITE_BYTESTRING |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
106 |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
107 i = 0 |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
108 l = len(source) |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
109 |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
110 while True: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
111 chunk = source[i : i + chunksize] |
37711
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
112 i += len(chunk) |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
113 |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
114 yield encodelength(MAJOR_TYPE_BYTESTRING, len(chunk)) |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
115 yield chunk |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
116 |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
117 if i >= l: |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
118 break |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
119 |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
120 yield BREAK |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
121 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
122 |
37711
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
123 def streamencodeint(v): |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
124 if v >= 18446744073709551616 or v < -18446744073709551616: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
125 raise ValueError(b'big integers not supported') |
37711
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
126 |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
127 if v >= 0: |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
128 yield encodelength(MAJOR_TYPE_UINT, v) |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
129 else: |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
130 yield encodelength(MAJOR_TYPE_NEGINT, abs(v) - 1) |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
131 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
132 |
37711
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
133 def streamencodearray(l): |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
134 """Encode a known size iterable to an array.""" |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
135 |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
136 yield encodelength(MAJOR_TYPE_ARRAY, len(l)) |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
137 |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
138 for i in l: |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
139 for chunk in streamencode(i): |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
140 yield chunk |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
141 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
142 |
37711
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
143 def streamencodearrayfromiter(it): |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
144 """Encode an iterator of items to an indefinite length array.""" |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
145 |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
146 yield BEGIN_INDEFINITE_ARRAY |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
147 |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
148 for i in it: |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
149 for chunk in streamencode(i): |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
150 yield chunk |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
151 |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
152 yield BREAK |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
153 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
154 |
37898
2ae6a3134362
cborutil: port to Python 3
Augie Fackler <augie@google.com>
parents:
37711
diff
changeset
|
155 def _mixedtypesortkey(v): |
2ae6a3134362
cborutil: port to Python 3
Augie Fackler <augie@google.com>
parents:
37711
diff
changeset
|
156 return type(v).__name__, v |
2ae6a3134362
cborutil: port to Python 3
Augie Fackler <augie@google.com>
parents:
37711
diff
changeset
|
157 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
158 |
37711
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
159 def streamencodeset(s): |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
160 # https://www.iana.org/assignments/cbor-tags/cbor-tags.xhtml defines |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
161 # semantic tag 258 for finite sets. |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
162 yield encodelength(MAJOR_TYPE_SEMANTIC, SEMANTIC_TAG_FINITE_SET) |
37711
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
163 |
37898
2ae6a3134362
cborutil: port to Python 3
Augie Fackler <augie@google.com>
parents:
37711
diff
changeset
|
164 for chunk in streamencodearray(sorted(s, key=_mixedtypesortkey)): |
37711
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
165 yield chunk |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
166 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
167 |
37711
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
168 def streamencodemap(d): |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
169 """Encode dictionary to a generator. |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
170 |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
171 Does not supporting indefinite length dictionaries. |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
172 """ |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
173 yield encodelength(MAJOR_TYPE_MAP, len(d)) |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
174 |
48913
f254fc73d956
global: bulk replace simple pycompat.iteritems(x) with x.items()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
48875
diff
changeset
|
175 for key, value in sorted(d.items(), key=lambda x: _mixedtypesortkey(x[0])): |
37711
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
176 for chunk in streamencode(key): |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
177 yield chunk |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
178 for chunk in streamencode(value): |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
179 yield chunk |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
180 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
181 |
37711
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
182 def streamencodemapfromiter(it): |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
183 """Given an iterable of (key, value), encode to an indefinite length map.""" |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
184 yield BEGIN_INDEFINITE_MAP |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
185 |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
186 for key, value in it: |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
187 for chunk in streamencode(key): |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
188 yield chunk |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
189 for chunk in streamencode(value): |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
190 yield chunk |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
191 |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
192 yield BREAK |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
193 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
194 |
37711
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
195 def streamencodebool(b): |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
196 # major type 7, simple value 20 and 21. |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
197 yield b'\xf5' if b else b'\xf4' |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
198 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
199 |
37711
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
200 def streamencodenone(v): |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
201 # major type 7, simple value 22. |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
202 yield b'\xf6' |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
203 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
204 |
37711
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
205 STREAM_ENCODERS = { |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
206 bytes: streamencodebytestring, |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
207 int: streamencodeint, |
48932
176f1a0d15dc
py3: use int instead of pycompat.long
Gregory Szorc <gregory.szorc@gmail.com>
parents:
48913
diff
changeset
|
208 int: streamencodeint, |
37711
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
209 list: streamencodearray, |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
210 tuple: streamencodearray, |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
211 dict: streamencodemap, |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
212 set: streamencodeset, |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
213 bool: streamencodebool, |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
214 type(None): streamencodenone, |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
215 } |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
216 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
217 |
37711
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
218 def streamencode(v): |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
219 """Encode a value in a streaming manner. |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
220 |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
221 Given an input object, encode it to CBOR recursively. |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
222 |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
223 Returns a generator of CBOR encoded bytes. There is no guarantee |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
224 that each emitted chunk fully decodes to a value or sub-value. |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
225 |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
226 Encoding is deterministic - unordered collections are sorted. |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
227 """ |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
228 fn = STREAM_ENCODERS.get(v.__class__) |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
229 |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
230 if not fn: |
42480
b6387a65851d
cborutil: fix streamencode() to handle subtypes
Yuya Nishihara <yuya@tcha.org>
parents:
40124
diff
changeset
|
231 # handle subtypes such as encoding.localstr and util.sortdict |
b6387a65851d
cborutil: fix streamencode() to handle subtypes
Yuya Nishihara <yuya@tcha.org>
parents:
40124
diff
changeset
|
232 for ty in STREAM_ENCODERS: |
b6387a65851d
cborutil: fix streamencode() to handle subtypes
Yuya Nishihara <yuya@tcha.org>
parents:
40124
diff
changeset
|
233 if not isinstance(v, ty): |
b6387a65851d
cborutil: fix streamencode() to handle subtypes
Yuya Nishihara <yuya@tcha.org>
parents:
40124
diff
changeset
|
234 continue |
b6387a65851d
cborutil: fix streamencode() to handle subtypes
Yuya Nishihara <yuya@tcha.org>
parents:
40124
diff
changeset
|
235 fn = STREAM_ENCODERS[ty] |
b6387a65851d
cborutil: fix streamencode() to handle subtypes
Yuya Nishihara <yuya@tcha.org>
parents:
40124
diff
changeset
|
236 break |
b6387a65851d
cborutil: fix streamencode() to handle subtypes
Yuya Nishihara <yuya@tcha.org>
parents:
40124
diff
changeset
|
237 |
b6387a65851d
cborutil: fix streamencode() to handle subtypes
Yuya Nishihara <yuya@tcha.org>
parents:
40124
diff
changeset
|
238 if not fn: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
239 raise ValueError(b'do not know how to encode %s' % type(v)) |
37711
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
240 |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
241 return fn(v) |
65a23cc8e75b
cborutil: implement support for streaming encoding, bytestring decoding
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
242 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
243 |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
244 class CBORDecodeError(Exception): |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
245 """Represents an error decoding CBOR.""" |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
246 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
247 |
48942
bce8f66d3045
cborutil: remove Python 2 definition of _elementtointeger()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
48932
diff
changeset
|
248 def _elementtointeger(b, i): |
bce8f66d3045
cborutil: remove Python 2 definition of _elementtointeger()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
48932
diff
changeset
|
249 return b[i] |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
250 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
251 |
43506
9f70512ae2cf
cleanup: remove pointless r-prefixes on single-quoted strings
Augie Fackler <augie@google.com>
parents:
43117
diff
changeset
|
252 STRUCT_BIG_UBYTE = struct.Struct('>B') |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
253 STRUCT_BIG_USHORT = struct.Struct(b'>H') |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
254 STRUCT_BIG_ULONG = struct.Struct(b'>L') |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
255 STRUCT_BIG_ULONGLONG = struct.Struct(b'>Q') |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
256 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
257 SPECIAL_NONE = 0 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
258 SPECIAL_START_INDEFINITE_BYTESTRING = 1 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
259 SPECIAL_START_ARRAY = 2 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
260 SPECIAL_START_MAP = 3 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
261 SPECIAL_START_SET = 4 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
262 SPECIAL_INDEFINITE_BREAK = 5 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
263 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
264 |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
265 def decodeitem(b, offset=0): |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
266 """Decode a new CBOR value from a buffer at offset. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
267 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
268 This function attempts to decode up to one complete CBOR value |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
269 from ``b`` starting at offset ``offset``. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
270 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
271 The beginning of a collection (such as an array, map, set, or |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
272 indefinite length bytestring) counts as a single value. For these |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
273 special cases, a state flag will indicate that a special value was seen. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
274 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
275 When called, the function either returns a decoded value or gives |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
276 a hint as to how many more bytes are needed to do so. By calling |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
277 the function repeatedly given a stream of bytes, the caller can |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
278 build up the original values. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
279 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
280 Returns a tuple with the following elements: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
281 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
282 * Bool indicating whether a complete value was decoded. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
283 * A decoded value if first value is True otherwise None |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
284 * Integer number of bytes. If positive, the number of bytes |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
285 read. If negative, the number of bytes we need to read to |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
286 decode this value or the next chunk in this value. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
287 * One of the ``SPECIAL_*`` constants indicating special treatment |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
288 for this value. ``SPECIAL_NONE`` means this is a fully decoded |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
289 simple value (such as an integer or bool). |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
290 """ |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
291 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
292 initial = _elementtointeger(b, offset) |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
293 offset += 1 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
294 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
295 majortype = initial >> 5 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
296 subtype = initial & SUBTYPE_MASK |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
297 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
298 if majortype == MAJOR_TYPE_UINT: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
299 complete, value, readcount = decodeuint(subtype, b, offset) |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
300 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
301 if complete: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
302 return True, value, readcount + 1, SPECIAL_NONE |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
303 else: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
304 return False, None, readcount, SPECIAL_NONE |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
305 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
306 elif majortype == MAJOR_TYPE_NEGINT: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
307 # Negative integers are the same as UINT except inverted minus 1. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
308 complete, value, readcount = decodeuint(subtype, b, offset) |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
309 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
310 if complete: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
311 return True, -value - 1, readcount + 1, SPECIAL_NONE |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
312 else: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
313 return False, None, readcount, SPECIAL_NONE |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
314 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
315 elif majortype == MAJOR_TYPE_BYTESTRING: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
316 # Beginning of bytestrings are treated as uints in order to |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
317 # decode their length, which may be indefinite. |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
318 complete, size, readcount = decodeuint( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
319 subtype, b, offset, allowindefinite=True |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
320 ) |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
321 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
322 # We don't know the size of the bytestring. It must be a definitive |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
323 # length since the indefinite subtype would be encoded in the initial |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
324 # byte. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
325 if not complete: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
326 return False, None, readcount, SPECIAL_NONE |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
327 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
328 # We know the length of the bytestring. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
329 if size is not None: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
330 # And the data is available in the buffer. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
331 if offset + readcount + size <= len(b): |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
332 value = b[offset + readcount : offset + readcount + size] |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
333 return True, value, readcount + size + 1, SPECIAL_NONE |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
334 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
335 # And we need more data in order to return the bytestring. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
336 else: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
337 wanted = len(b) - offset - readcount - size |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
338 return False, None, wanted, SPECIAL_NONE |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
339 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
340 # It is an indefinite length bytestring. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
341 else: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
342 return True, None, 1, SPECIAL_START_INDEFINITE_BYTESTRING |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
343 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
344 elif majortype == MAJOR_TYPE_STRING: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
345 raise CBORDecodeError(b'string major type not supported') |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
346 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
347 elif majortype == MAJOR_TYPE_ARRAY: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
348 # Beginning of arrays are treated as uints in order to decode their |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
349 # length. We don't allow indefinite length arrays. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
350 complete, size, readcount = decodeuint(subtype, b, offset) |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
351 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
352 if complete: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
353 return True, size, readcount + 1, SPECIAL_START_ARRAY |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
354 else: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
355 return False, None, readcount, SPECIAL_NONE |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
356 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
357 elif majortype == MAJOR_TYPE_MAP: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
358 # Beginning of maps are treated as uints in order to decode their |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
359 # number of elements. We don't allow indefinite length arrays. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
360 complete, size, readcount = decodeuint(subtype, b, offset) |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
361 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
362 if complete: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
363 return True, size, readcount + 1, SPECIAL_START_MAP |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
364 else: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
365 return False, None, readcount, SPECIAL_NONE |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
366 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
367 elif majortype == MAJOR_TYPE_SEMANTIC: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
368 # Semantic tag value is read the same as a uint. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
369 complete, tagvalue, readcount = decodeuint(subtype, b, offset) |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
370 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
371 if not complete: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
372 return False, None, readcount, SPECIAL_NONE |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
373 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
374 # This behavior here is a little wonky. The main type being "decorated" |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
375 # by this semantic tag follows. A more robust parser would probably emit |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
376 # a special flag indicating this as a semantic tag and let the caller |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
377 # deal with the types that follow. But since we don't support many |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
378 # semantic tags, it is easier to deal with the special cases here and |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
379 # hide complexity from the caller. If we add support for more semantic |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
380 # tags, we should probably move semantic tag handling into the caller. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
381 if tagvalue == SEMANTIC_TAG_FINITE_SET: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
382 if offset + readcount >= len(b): |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
383 return False, None, -1, SPECIAL_NONE |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
384 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
385 complete, size, readcount2, special = decodeitem( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
386 b, offset + readcount |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
387 ) |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
388 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
389 if not complete: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
390 return False, None, readcount2, SPECIAL_NONE |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
391 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
392 if special != SPECIAL_START_ARRAY: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
393 raise CBORDecodeError( |
43117
8ff1ecfadcd1
cleanup: join string literals that are already on one line
Martin von Zweigbergk <martinvonz@google.com>
parents:
43106
diff
changeset
|
394 b'expected array after finite set semantic tag' |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
395 ) |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
396 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
397 return True, size, readcount + readcount2 + 1, SPECIAL_START_SET |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
398 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
399 else: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
400 raise CBORDecodeError(b'semantic tag %d not allowed' % tagvalue) |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
401 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
402 elif majortype == MAJOR_TYPE_SPECIAL: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
403 # Only specific values for the information field are allowed. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
404 if subtype == SUBTYPE_FALSE: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
405 return True, False, 1, SPECIAL_NONE |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
406 elif subtype == SUBTYPE_TRUE: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
407 return True, True, 1, SPECIAL_NONE |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
408 elif subtype == SUBTYPE_NULL: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
409 return True, None, 1, SPECIAL_NONE |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
410 elif subtype == SUBTYPE_INDEFINITE: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
411 return True, None, 1, SPECIAL_INDEFINITE_BREAK |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
412 # If value is 24, subtype is in next byte. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
413 else: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
414 raise CBORDecodeError(b'special type %d not allowed' % subtype) |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
415 else: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
416 assert False |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
417 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
418 |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
419 def decodeuint(subtype, b, offset=0, allowindefinite=False): |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
420 """Decode an unsigned integer. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
421 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
422 ``subtype`` is the lower 5 bits from the initial byte CBOR item |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
423 "header." ``b`` is a buffer containing bytes. ``offset`` points to |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
424 the index of the first byte after the byte that ``subtype`` was |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
425 derived from. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
426 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
427 ``allowindefinite`` allows the special indefinite length value |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
428 indicator. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
429 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
430 Returns a 3-tuple of (successful, value, count). |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
431 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
432 The first element is a bool indicating if decoding completed. The 2nd |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
433 is the decoded integer value or None if not fully decoded or the subtype |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
434 is 31 and ``allowindefinite`` is True. The 3rd value is the count of bytes. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
435 If positive, it is the number of additional bytes decoded. If negative, |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
436 it is the number of additional bytes needed to decode this value. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
437 """ |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
438 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
439 # Small values are inline. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
440 if subtype < 24: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
441 return True, subtype, 0 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
442 # Indefinite length specifier. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
443 elif subtype == 31: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
444 if allowindefinite: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
445 return True, None, 0 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
446 else: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
447 raise CBORDecodeError(b'indefinite length uint not allowed here') |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
448 elif subtype >= 28: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
449 raise CBORDecodeError( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
450 b'unsupported subtype on integer type: %d' % subtype |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
451 ) |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
452 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
453 if subtype == 24: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
454 s = STRUCT_BIG_UBYTE |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
455 elif subtype == 25: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
456 s = STRUCT_BIG_USHORT |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
457 elif subtype == 26: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
458 s = STRUCT_BIG_ULONG |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
459 elif subtype == 27: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
460 s = STRUCT_BIG_ULONGLONG |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
461 else: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
462 raise CBORDecodeError(b'bounds condition checking violation') |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
463 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
464 if len(b) - offset >= s.size: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
465 return True, s.unpack_from(b, offset)[0], s.size |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
466 else: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
467 return False, None, len(b) - offset - s.size |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
468 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
469 |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
470 class bytestringchunk(bytes): |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
471 """Represents a chunk/segment in an indefinite length bytestring. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
472 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
473 This behaves like a ``bytes`` but in addition has the ``isfirst`` |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
474 and ``islast`` attributes indicating whether this chunk is the first |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
475 or last in an indefinite length bytestring. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
476 """ |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
477 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
478 def __new__(cls, v, first=False, last=False): |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
479 self = bytes.__new__(cls, v) |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
480 self.isfirst = first |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
481 self.islast = last |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
482 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
483 return self |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
484 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
485 |
48946
642e31cb55f0
py3: use class X: instead of class X(object):
Gregory Szorc <gregory.szorc@gmail.com>
parents:
48942
diff
changeset
|
486 class sansiodecoder: |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
487 """A CBOR decoder that doesn't perform its own I/O. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
488 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
489 To use, construct an instance and feed it segments containing |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
490 CBOR-encoded bytes via ``decode()``. The return value from ``decode()`` |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
491 indicates whether a fully-decoded value is available, how many bytes |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
492 were consumed, and offers a hint as to how many bytes should be fed |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
493 in next time to decode the next value. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
494 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
495 The decoder assumes it will decode N discrete CBOR values, not just |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
496 a single value. i.e. if the bytestream contains uints packed one after |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
497 the other, the decoder will decode them all, rather than just the initial |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
498 one. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
499 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
500 When ``decode()`` indicates a value is available, call ``getavailable()`` |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
501 to return all fully decoded values. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
502 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
503 ``decode()`` can partially decode input. It is up to the caller to keep |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
504 track of what data was consumed and to pass unconsumed data in on the |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
505 next invocation. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
506 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
507 The decoder decodes atomically at the *item* level. See ``decodeitem()``. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
508 If an *item* cannot be fully decoded, the decoder won't record it as |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
509 partially consumed. Instead, the caller will be instructed to pass in |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
510 the initial bytes of this item on the next invocation. This does result |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
511 in some redundant parsing. But the overhead should be minimal. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
512 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
513 This decoder only supports a subset of CBOR as required by Mercurial. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
514 It lacks support for: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
515 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
516 * Indefinite length arrays |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
517 * Indefinite length maps |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
518 * Use of indefinite length bytestrings as keys or values within |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
519 arrays, maps, or sets. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
520 * Nested arrays, maps, or sets within sets |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
521 * Any semantic tag that isn't a mathematical finite set |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
522 * Floating point numbers |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
523 * Undefined special value |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
524 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
525 CBOR types are decoded to Python types as follows: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
526 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
527 uint -> int |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
528 negint -> int |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
529 bytestring -> bytes |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
530 map -> dict |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
531 array -> list |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
532 True -> bool |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
533 False -> bool |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
534 null -> None |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
535 indefinite length bytestring chunk -> [bytestringchunk] |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
536 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
537 The only non-obvious mapping here is an indefinite length bytestring |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
538 to the ``bytestringchunk`` type. This is to facilitate streaming |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
539 indefinite length bytestrings out of the decoder and to differentiate |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
540 a regular bytestring from an indefinite length bytestring. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
541 """ |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
542 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
543 _STATE_NONE = 0 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
544 _STATE_WANT_MAP_KEY = 1 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
545 _STATE_WANT_MAP_VALUE = 2 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
546 _STATE_WANT_ARRAY_VALUE = 3 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
547 _STATE_WANT_SET_VALUE = 4 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
548 _STATE_WANT_BYTESTRING_CHUNK_FIRST = 5 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
549 _STATE_WANT_BYTESTRING_CHUNK_SUBSEQUENT = 6 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
550 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
551 def __init__(self): |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
552 # TODO add support for limiting size of bytestrings |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
553 # TODO add support for limiting number of keys / values in collections |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
554 # TODO add support for limiting size of buffered partial values |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
555 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
556 self.decodedbytecount = 0 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
557 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
558 self._state = self._STATE_NONE |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
559 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
560 # Stack of active nested collections. Each entry is a dict describing |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
561 # the collection. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
562 self._collectionstack = [] |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
563 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
564 # Fully decoded key to use for the current map. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
565 self._currentmapkey = None |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
566 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
567 # Fully decoded values available for retrieval. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
568 self._decodedvalues = [] |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
569 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
570 @property |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
571 def inprogress(self): |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
572 """Whether the decoder has partially decoded a value.""" |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
573 return self._state != self._STATE_NONE |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
574 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
575 def decode(self, b, offset=0): |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
576 """Attempt to decode bytes from an input buffer. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
577 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
578 ``b`` is a collection of bytes and ``offset`` is the byte |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
579 offset within that buffer from which to begin reading data. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
580 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
581 ``b`` must support ``len()`` and accessing bytes slices via |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
582 ``__slice__``. Typically ``bytes`` instances are used. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
583 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
584 Returns a tuple with the following fields: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
585 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
586 * Bool indicating whether values are available for retrieval. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
587 * Integer indicating the number of bytes that were fully consumed, |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
588 starting from ``offset``. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
589 * Integer indicating the number of bytes that are desired for the |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
590 next call in order to decode an item. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
591 """ |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
592 if not b: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
593 return bool(self._decodedvalues), 0, 0 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
594 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
595 initialoffset = offset |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
596 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
597 # We could easily split the body of this loop into a function. But |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
598 # Python performance is sensitive to function calls and collections |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
599 # are composed of many items. So leaving as a while loop could help |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
600 # with performance. One thing that may not help is the use of |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
601 # if..elif versus a lookup/dispatch table. There may be value |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
602 # in switching that. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
603 while offset < len(b): |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
604 # Attempt to decode an item. This could be a whole value or a |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
605 # special value indicating an event, such as start or end of a |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
606 # collection or indefinite length type. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
607 complete, value, readcount, special = decodeitem(b, offset) |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
608 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
609 if readcount > 0: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
610 self.decodedbytecount += readcount |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
611 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
612 if not complete: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
613 assert readcount < 0 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
614 return ( |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
615 bool(self._decodedvalues), |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
616 offset - initialoffset, |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
617 -readcount, |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
618 ) |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
619 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
620 offset += readcount |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
621 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
622 # No nested state. We either have a full value or beginning of a |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
623 # complex value to deal with. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
624 if self._state == self._STATE_NONE: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
625 # A normal value. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
626 if special == SPECIAL_NONE: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
627 self._decodedvalues.append(value) |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
628 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
629 elif special == SPECIAL_START_ARRAY: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
630 self._collectionstack.append( |
45942
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
43506
diff
changeset
|
631 { |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
43506
diff
changeset
|
632 b'remaining': value, |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
43506
diff
changeset
|
633 b'v': [], |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
43506
diff
changeset
|
634 } |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
635 ) |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
636 self._state = self._STATE_WANT_ARRAY_VALUE |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
637 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
638 elif special == SPECIAL_START_MAP: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
639 self._collectionstack.append( |
45942
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
43506
diff
changeset
|
640 { |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
43506
diff
changeset
|
641 b'remaining': value, |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
43506
diff
changeset
|
642 b'v': {}, |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
43506
diff
changeset
|
643 } |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
644 ) |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
645 self._state = self._STATE_WANT_MAP_KEY |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
646 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
647 elif special == SPECIAL_START_SET: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
648 self._collectionstack.append( |
45942
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
43506
diff
changeset
|
649 { |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
43506
diff
changeset
|
650 b'remaining': value, |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
43506
diff
changeset
|
651 b'v': set(), |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
43506
diff
changeset
|
652 } |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
653 ) |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
654 self._state = self._STATE_WANT_SET_VALUE |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
655 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
656 elif special == SPECIAL_START_INDEFINITE_BYTESTRING: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
657 self._state = self._STATE_WANT_BYTESTRING_CHUNK_FIRST |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
658 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
659 else: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
660 raise CBORDecodeError( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
661 b'unhandled special state: %d' % special |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
662 ) |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
663 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
664 # This value becomes an element of the current array. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
665 elif self._state == self._STATE_WANT_ARRAY_VALUE: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
666 # Simple values get appended. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
667 if special == SPECIAL_NONE: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
668 c = self._collectionstack[-1] |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
669 c[b'v'].append(value) |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
670 c[b'remaining'] -= 1 |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
671 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
672 # self._state doesn't need changed. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
673 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
674 # An array nested within an array. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
675 elif special == SPECIAL_START_ARRAY: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
676 lastc = self._collectionstack[-1] |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
677 newvalue = [] |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
678 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
679 lastc[b'v'].append(newvalue) |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
680 lastc[b'remaining'] -= 1 |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
681 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
682 self._collectionstack.append( |
45942
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
43506
diff
changeset
|
683 { |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
43506
diff
changeset
|
684 b'remaining': value, |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
43506
diff
changeset
|
685 b'v': newvalue, |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
43506
diff
changeset
|
686 } |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
687 ) |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
688 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
689 # self._state doesn't need changed. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
690 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
691 # A map nested within an array. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
692 elif special == SPECIAL_START_MAP: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
693 lastc = self._collectionstack[-1] |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
694 newvalue = {} |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
695 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
696 lastc[b'v'].append(newvalue) |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
697 lastc[b'remaining'] -= 1 |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
698 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
699 self._collectionstack.append( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
700 {b'remaining': value, b'v': newvalue} |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
701 ) |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
702 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
703 self._state = self._STATE_WANT_MAP_KEY |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
704 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
705 elif special == SPECIAL_START_SET: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
706 lastc = self._collectionstack[-1] |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
707 newvalue = set() |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
708 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
709 lastc[b'v'].append(newvalue) |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
710 lastc[b'remaining'] -= 1 |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
711 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
712 self._collectionstack.append( |
45942
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
43506
diff
changeset
|
713 { |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
43506
diff
changeset
|
714 b'remaining': value, |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
43506
diff
changeset
|
715 b'v': newvalue, |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
43506
diff
changeset
|
716 } |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
717 ) |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
718 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
719 self._state = self._STATE_WANT_SET_VALUE |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
720 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
721 elif special == SPECIAL_START_INDEFINITE_BYTESTRING: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
722 raise CBORDecodeError( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
723 b'indefinite length bytestrings ' |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
724 b'not allowed as array values' |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
725 ) |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
726 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
727 else: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
728 raise CBORDecodeError( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
729 b'unhandled special item when ' |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
730 b'expecting array value: %d' % special |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
731 ) |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
732 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
733 # This value becomes the key of the current map instance. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
734 elif self._state == self._STATE_WANT_MAP_KEY: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
735 if special == SPECIAL_NONE: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
736 self._currentmapkey = value |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
737 self._state = self._STATE_WANT_MAP_VALUE |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
738 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
739 elif special == SPECIAL_START_INDEFINITE_BYTESTRING: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
740 raise CBORDecodeError( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
741 b'indefinite length bytestrings ' |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
742 b'not allowed as map keys' |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
743 ) |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
744 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
745 elif special in ( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
746 SPECIAL_START_ARRAY, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
747 SPECIAL_START_MAP, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
748 SPECIAL_START_SET, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
749 ): |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
750 raise CBORDecodeError( |
43117
8ff1ecfadcd1
cleanup: join string literals that are already on one line
Martin von Zweigbergk <martinvonz@google.com>
parents:
43106
diff
changeset
|
751 b'collections not supported as map keys' |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
752 ) |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
753 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
754 # We do not allow special values to be used as map keys. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
755 else: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
756 raise CBORDecodeError( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
757 b'unhandled special item when ' |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
758 b'expecting map key: %d' % special |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
759 ) |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
760 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
761 # This value becomes the value of the current map key. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
762 elif self._state == self._STATE_WANT_MAP_VALUE: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
763 # Simple values simply get inserted into the map. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
764 if special == SPECIAL_NONE: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
765 lastc = self._collectionstack[-1] |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
766 lastc[b'v'][self._currentmapkey] = value |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
767 lastc[b'remaining'] -= 1 |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
768 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
769 self._state = self._STATE_WANT_MAP_KEY |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
770 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
771 # A new array is used as the map value. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
772 elif special == SPECIAL_START_ARRAY: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
773 lastc = self._collectionstack[-1] |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
774 newvalue = [] |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
775 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
776 lastc[b'v'][self._currentmapkey] = newvalue |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
777 lastc[b'remaining'] -= 1 |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
778 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
779 self._collectionstack.append( |
45942
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
43506
diff
changeset
|
780 { |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
43506
diff
changeset
|
781 b'remaining': value, |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
43506
diff
changeset
|
782 b'v': newvalue, |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
43506
diff
changeset
|
783 } |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
784 ) |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
785 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
786 self._state = self._STATE_WANT_ARRAY_VALUE |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
787 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
788 # A new map is used as the map value. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
789 elif special == SPECIAL_START_MAP: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
790 lastc = self._collectionstack[-1] |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
791 newvalue = {} |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
792 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
793 lastc[b'v'][self._currentmapkey] = newvalue |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
794 lastc[b'remaining'] -= 1 |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
795 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
796 self._collectionstack.append( |
45942
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
43506
diff
changeset
|
797 { |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
43506
diff
changeset
|
798 b'remaining': value, |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
43506
diff
changeset
|
799 b'v': newvalue, |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
43506
diff
changeset
|
800 } |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
801 ) |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
802 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
803 self._state = self._STATE_WANT_MAP_KEY |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
804 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
805 # A new set is used as the map value. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
806 elif special == SPECIAL_START_SET: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
807 lastc = self._collectionstack[-1] |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
808 newvalue = set() |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
809 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
810 lastc[b'v'][self._currentmapkey] = newvalue |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
811 lastc[b'remaining'] -= 1 |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
812 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
813 self._collectionstack.append( |
45942
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
43506
diff
changeset
|
814 { |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
43506
diff
changeset
|
815 b'remaining': value, |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
43506
diff
changeset
|
816 b'v': newvalue, |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
43506
diff
changeset
|
817 } |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
818 ) |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
819 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
820 self._state = self._STATE_WANT_SET_VALUE |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
821 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
822 elif special == SPECIAL_START_INDEFINITE_BYTESTRING: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
823 raise CBORDecodeError( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
824 b'indefinite length bytestrings not ' |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
825 b'allowed as map values' |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
826 ) |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
827 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
828 else: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
829 raise CBORDecodeError( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
830 b'unhandled special item when ' |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
831 b'expecting map value: %d' % special |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
832 ) |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
833 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
834 self._currentmapkey = None |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
835 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
836 # This value is added to the current set. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
837 elif self._state == self._STATE_WANT_SET_VALUE: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
838 if special == SPECIAL_NONE: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
839 lastc = self._collectionstack[-1] |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
840 lastc[b'v'].add(value) |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
841 lastc[b'remaining'] -= 1 |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
842 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
843 elif special == SPECIAL_START_INDEFINITE_BYTESTRING: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
844 raise CBORDecodeError( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
845 b'indefinite length bytestrings not ' |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
846 b'allowed as set values' |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
847 ) |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
848 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
849 elif special in ( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
850 SPECIAL_START_ARRAY, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
851 SPECIAL_START_MAP, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
852 SPECIAL_START_SET, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
853 ): |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
854 raise CBORDecodeError( |
43117
8ff1ecfadcd1
cleanup: join string literals that are already on one line
Martin von Zweigbergk <martinvonz@google.com>
parents:
43106
diff
changeset
|
855 b'collections not allowed as set values' |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
856 ) |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
857 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
858 # We don't allow non-trivial types to exist as set values. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
859 else: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
860 raise CBORDecodeError( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
861 b'unhandled special item when ' |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
862 b'expecting set value: %d' % special |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
863 ) |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
864 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
865 # This value represents the first chunk in an indefinite length |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
866 # bytestring. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
867 elif self._state == self._STATE_WANT_BYTESTRING_CHUNK_FIRST: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
868 # We received a full chunk. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
869 if special == SPECIAL_NONE: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
870 self._decodedvalues.append( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
871 bytestringchunk(value, first=True) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
872 ) |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
873 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
874 self._state = self._STATE_WANT_BYTESTRING_CHUNK_SUBSEQUENT |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
875 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
876 # The end of stream marker. This means it is an empty |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
877 # indefinite length bytestring. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
878 elif special == SPECIAL_INDEFINITE_BREAK: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
879 # We /could/ convert this to a b''. But we want to preserve |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
880 # the nature of the underlying data so consumers expecting |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
881 # an indefinite length bytestring get one. |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
882 self._decodedvalues.append( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
883 bytestringchunk(b'', first=True, last=True) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
884 ) |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
885 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
886 # Since indefinite length bytestrings can't be used in |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
887 # collections, we must be at the root level. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
888 assert not self._collectionstack |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
889 self._state = self._STATE_NONE |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
890 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
891 else: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
892 raise CBORDecodeError( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
893 b'unexpected special value when ' |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
894 b'expecting bytestring chunk: %d' % special |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
895 ) |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
896 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
897 # This value represents the non-initial chunk in an indefinite |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
898 # length bytestring. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
899 elif self._state == self._STATE_WANT_BYTESTRING_CHUNK_SUBSEQUENT: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
900 # We received a full chunk. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
901 if special == SPECIAL_NONE: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
902 self._decodedvalues.append(bytestringchunk(value)) |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
903 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
904 # The end of stream marker. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
905 elif special == SPECIAL_INDEFINITE_BREAK: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
906 self._decodedvalues.append(bytestringchunk(b'', last=True)) |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
907 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
908 # Since indefinite length bytestrings can't be used in |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
909 # collections, we must be at the root level. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
910 assert not self._collectionstack |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
911 self._state = self._STATE_NONE |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
912 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
913 else: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
914 raise CBORDecodeError( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
915 b'unexpected special value when ' |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
916 b'expecting bytestring chunk: %d' % special |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
917 ) |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
918 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
919 else: |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
920 raise CBORDecodeError( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
921 b'unhandled decoder state: %d' % self._state |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
922 ) |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
923 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
924 # We could have just added the final value in a collection. End |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
925 # all complete collections at the top of the stack. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
926 while True: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
927 # Bail if we're not waiting on a new collection item. |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
928 if self._state not in ( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
929 self._STATE_WANT_ARRAY_VALUE, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
930 self._STATE_WANT_MAP_KEY, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
931 self._STATE_WANT_SET_VALUE, |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
932 ): |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
933 break |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
934 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
935 # Or we are expecting more items for this collection. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
936 lastc = self._collectionstack[-1] |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
937 |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
938 if lastc[b'remaining']: |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
939 break |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
940 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
941 # The collection at the top of the stack is complete. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
942 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
943 # Discard it, as it isn't needed for future items. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
944 self._collectionstack.pop() |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
945 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
946 # If this is a nested collection, we don't emit it, since it |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
947 # will be emitted by its parent collection. But we do need to |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
948 # update state to reflect what the new top-most collection |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
949 # on the stack is. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
950 if self._collectionstack: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
951 self._state = { |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
952 list: self._STATE_WANT_ARRAY_VALUE, |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
953 dict: self._STATE_WANT_MAP_KEY, |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
954 set: self._STATE_WANT_SET_VALUE, |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
955 }[type(self._collectionstack[-1][b'v'])] |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
956 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
957 # If this is the root collection, emit it. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
958 else: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
959 self._decodedvalues.append(lastc[b'v']) |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
960 self._state = self._STATE_NONE |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
961 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
962 return ( |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
963 bool(self._decodedvalues), |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
964 offset - initialoffset, |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
965 0, |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
966 ) |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
967 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
968 def getavailable(self): |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
969 """Returns an iterator over fully decoded values. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
970 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
971 Once values are retrieved, they won't be available on the next call. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
972 """ |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
973 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
974 l = list(self._decodedvalues) |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
975 self._decodedvalues = [] |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
976 return l |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
977 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
978 |
48946
642e31cb55f0
py3: use class X: instead of class X(object):
Gregory Szorc <gregory.szorc@gmail.com>
parents:
48942
diff
changeset
|
979 class bufferingdecoder: |
39413
babad5ebaf0a
cborutil: add a buffering decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39412
diff
changeset
|
980 """A CBOR decoder that buffers undecoded input. |
babad5ebaf0a
cborutil: add a buffering decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39412
diff
changeset
|
981 |
babad5ebaf0a
cborutil: add a buffering decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39412
diff
changeset
|
982 This is a glorified wrapper around ``sansiodecoder`` that adds a buffering |
babad5ebaf0a
cborutil: add a buffering decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39412
diff
changeset
|
983 layer. All input that isn't consumed by ``sansiodecoder`` will be buffered |
babad5ebaf0a
cborutil: add a buffering decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39412
diff
changeset
|
984 and concatenated with any new input that arrives later. |
babad5ebaf0a
cborutil: add a buffering decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39412
diff
changeset
|
985 |
babad5ebaf0a
cborutil: add a buffering decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39412
diff
changeset
|
986 TODO consider adding limits as to the maximum amount of data that can |
babad5ebaf0a
cborutil: add a buffering decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39412
diff
changeset
|
987 be buffered. |
babad5ebaf0a
cborutil: add a buffering decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39412
diff
changeset
|
988 """ |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
989 |
39413
babad5ebaf0a
cborutil: add a buffering decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39412
diff
changeset
|
990 def __init__(self): |
babad5ebaf0a
cborutil: add a buffering decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39412
diff
changeset
|
991 self._decoder = sansiodecoder() |
40030
62160d3077cd
cborutil: change buffering strategy
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39456
diff
changeset
|
992 self._chunks = [] |
62160d3077cd
cborutil: change buffering strategy
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39456
diff
changeset
|
993 self._wanted = 0 |
39413
babad5ebaf0a
cborutil: add a buffering decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39412
diff
changeset
|
994 |
babad5ebaf0a
cborutil: add a buffering decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39412
diff
changeset
|
995 def decode(self, b): |
babad5ebaf0a
cborutil: add a buffering decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39412
diff
changeset
|
996 """Attempt to decode bytes to CBOR values. |
babad5ebaf0a
cborutil: add a buffering decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39412
diff
changeset
|
997 |
babad5ebaf0a
cborutil: add a buffering decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39412
diff
changeset
|
998 Returns a tuple with the following fields: |
babad5ebaf0a
cborutil: add a buffering decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39412
diff
changeset
|
999 |
babad5ebaf0a
cborutil: add a buffering decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39412
diff
changeset
|
1000 * Bool indicating whether new values are available for retrieval. |
babad5ebaf0a
cborutil: add a buffering decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39412
diff
changeset
|
1001 * Integer number of bytes decoded from the new input. |
babad5ebaf0a
cborutil: add a buffering decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39412
diff
changeset
|
1002 * Integer number of bytes wanted to decode the next value. |
babad5ebaf0a
cborutil: add a buffering decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39412
diff
changeset
|
1003 """ |
40124
b638219a23c3
cborutil: cast bytearray to bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40030
diff
changeset
|
1004 # We /might/ be able to support passing a bytearray all the |
b638219a23c3
cborutil: cast bytearray to bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40030
diff
changeset
|
1005 # way through. For now, let's cheat. |
b638219a23c3
cborutil: cast bytearray to bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40030
diff
changeset
|
1006 if isinstance(b, bytearray): |
b638219a23c3
cborutil: cast bytearray to bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40030
diff
changeset
|
1007 b = bytes(b) |
b638219a23c3
cborutil: cast bytearray to bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents:
40030
diff
changeset
|
1008 |
40030
62160d3077cd
cborutil: change buffering strategy
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39456
diff
changeset
|
1009 # Our strategy for buffering is to aggregate the incoming chunks in a |
62160d3077cd
cborutil: change buffering strategy
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39456
diff
changeset
|
1010 # list until we've received enough data to decode the next item. |
62160d3077cd
cborutil: change buffering strategy
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39456
diff
changeset
|
1011 # This is slightly more complicated than using an ``io.BytesIO`` |
62160d3077cd
cborutil: change buffering strategy
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39456
diff
changeset
|
1012 # or continuously concatenating incoming data. However, because it |
62160d3077cd
cborutil: change buffering strategy
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39456
diff
changeset
|
1013 # isn't constantly reallocating backing memory for a growing buffer, |
62160d3077cd
cborutil: change buffering strategy
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39456
diff
changeset
|
1014 # it prevents excessive memory thrashing and is significantly faster, |
62160d3077cd
cborutil: change buffering strategy
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39456
diff
changeset
|
1015 # especially in cases where the percentage of input chunks that don't |
62160d3077cd
cborutil: change buffering strategy
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39456
diff
changeset
|
1016 # decode into a full item is high. |
39413
babad5ebaf0a
cborutil: add a buffering decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39412
diff
changeset
|
1017 |
40030
62160d3077cd
cborutil: change buffering strategy
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39456
diff
changeset
|
1018 if self._chunks: |
62160d3077cd
cborutil: change buffering strategy
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39456
diff
changeset
|
1019 # A previous call said we needed N bytes to decode the next item. |
62160d3077cd
cborutil: change buffering strategy
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39456
diff
changeset
|
1020 # But this call doesn't provide enough data. We buffer the incoming |
62160d3077cd
cborutil: change buffering strategy
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39456
diff
changeset
|
1021 # chunk without attempting to decode. |
62160d3077cd
cborutil: change buffering strategy
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39456
diff
changeset
|
1022 if len(b) < self._wanted: |
62160d3077cd
cborutil: change buffering strategy
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39456
diff
changeset
|
1023 self._chunks.append(b) |
62160d3077cd
cborutil: change buffering strategy
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39456
diff
changeset
|
1024 self._wanted -= len(b) |
62160d3077cd
cborutil: change buffering strategy
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39456
diff
changeset
|
1025 return False, 0, self._wanted |
62160d3077cd
cborutil: change buffering strategy
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39456
diff
changeset
|
1026 |
62160d3077cd
cborutil: change buffering strategy
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39456
diff
changeset
|
1027 # Else we may have enough data to decode the next item. Aggregate |
62160d3077cd
cborutil: change buffering strategy
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39456
diff
changeset
|
1028 # old data with new and reset the buffer. |
62160d3077cd
cborutil: change buffering strategy
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39456
diff
changeset
|
1029 newlen = len(b) |
62160d3077cd
cborutil: change buffering strategy
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39456
diff
changeset
|
1030 self._chunks.append(b) |
62160d3077cd
cborutil: change buffering strategy
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39456
diff
changeset
|
1031 b = b''.join(self._chunks) |
62160d3077cd
cborutil: change buffering strategy
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39456
diff
changeset
|
1032 self._chunks = [] |
62160d3077cd
cborutil: change buffering strategy
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39456
diff
changeset
|
1033 oldlen = len(b) - newlen |
62160d3077cd
cborutil: change buffering strategy
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39456
diff
changeset
|
1034 |
39413
babad5ebaf0a
cborutil: add a buffering decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39412
diff
changeset
|
1035 else: |
babad5ebaf0a
cborutil: add a buffering decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39412
diff
changeset
|
1036 oldlen = 0 |
babad5ebaf0a
cborutil: add a buffering decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39412
diff
changeset
|
1037 |
babad5ebaf0a
cborutil: add a buffering decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39412
diff
changeset
|
1038 available, readcount, wanted = self._decoder.decode(b) |
40030
62160d3077cd
cborutil: change buffering strategy
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39456
diff
changeset
|
1039 self._wanted = wanted |
39413
babad5ebaf0a
cborutil: add a buffering decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39412
diff
changeset
|
1040 |
babad5ebaf0a
cborutil: add a buffering decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39412
diff
changeset
|
1041 if readcount < len(b): |
40030
62160d3077cd
cborutil: change buffering strategy
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39456
diff
changeset
|
1042 self._chunks.append(b[readcount:]) |
39413
babad5ebaf0a
cborutil: add a buffering decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39412
diff
changeset
|
1043 |
babad5ebaf0a
cborutil: add a buffering decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39412
diff
changeset
|
1044 return available, readcount - oldlen, wanted |
babad5ebaf0a
cborutil: add a buffering decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39412
diff
changeset
|
1045 |
babad5ebaf0a
cborutil: add a buffering decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39412
diff
changeset
|
1046 def getavailable(self): |
babad5ebaf0a
cborutil: add a buffering decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39412
diff
changeset
|
1047 return self._decoder.getavailable() |
babad5ebaf0a
cborutil: add a buffering decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39412
diff
changeset
|
1048 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42480
diff
changeset
|
1049 |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
1050 def decodeall(b): |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
1051 """Decode all CBOR items present in an iterable of bytes. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
1052 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
1053 In addition to regular decode errors, raises CBORDecodeError if the |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
1054 entirety of the passed buffer does not fully decode to complete CBOR |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
1055 values. This includes failure to decode any value, incomplete collection |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
1056 types, incomplete indefinite length items, and extra data at the end of |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
1057 the buffer. |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
1058 """ |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
1059 if not b: |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
1060 return [] |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
1061 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
1062 decoder = sansiodecoder() |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
1063 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
1064 havevalues, readcount, wantbytes = decoder.decode(b) |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
1065 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
1066 if readcount != len(b): |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
1067 raise CBORDecodeError(b'input data not fully consumed') |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
1068 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
1069 if decoder.inprogress: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
1070 raise CBORDecodeError(b'input data not complete') |
39411
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
1071 |
aeb551a3bb8a
cborutil: implement sans I/O decoder
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37898
diff
changeset
|
1072 return decoder.getavailable() |