--- a/mercurial/utils/cborutil.py Sat Oct 05 10:29:34 2019 -0400
+++ b/mercurial/utils/cborutil.py Sun Oct 06 09:45:02 2019 -0400
@@ -46,11 +46,14 @@
# Indefinite types begin with their major type ORd with information value 31.
BEGIN_INDEFINITE_BYTESTRING = struct.pack(
- r'>B', MAJOR_TYPE_BYTESTRING << 5 | SUBTYPE_INDEFINITE)
+ r'>B', MAJOR_TYPE_BYTESTRING << 5 | SUBTYPE_INDEFINITE
+)
BEGIN_INDEFINITE_ARRAY = struct.pack(
- r'>B', MAJOR_TYPE_ARRAY << 5 | SUBTYPE_INDEFINITE)
+ r'>B', MAJOR_TYPE_ARRAY << 5 | SUBTYPE_INDEFINITE
+)
BEGIN_INDEFINITE_MAP = struct.pack(
- r'>B', MAJOR_TYPE_MAP << 5 | SUBTYPE_INDEFINITE)
+ r'>B', MAJOR_TYPE_MAP << 5 | SUBTYPE_INDEFINITE
+)
ENCODED_LENGTH_1 = struct.Struct(r'>B')
ENCODED_LENGTH_2 = struct.Struct(r'>BB')
@@ -62,6 +65,7 @@
BREAK = b'\xff'
BREAK_INT = 255
+
def encodelength(majortype, length):
"""Obtain a value encoding the major type and its length."""
if length < 24:
@@ -75,10 +79,12 @@
else:
return ENCODED_LENGTH_5.pack(majortype << 5 | 27, length)
+
def streamencodebytestring(v):
yield encodelength(MAJOR_TYPE_BYTESTRING, len(v))
yield v
+
def streamencodebytestringfromiter(it):
"""Convert an iterator of chunks to an indefinite bytestring.
@@ -93,6 +99,7 @@
yield BREAK
+
def streamencodeindefinitebytestring(source, chunksize=65536):
"""Given a large source buffer, emit as an indefinite length bytestring.
@@ -104,7 +111,7 @@
l = len(source)
while True:
- chunk = source[i:i + chunksize]
+ chunk = source[i : i + chunksize]
i += len(chunk)
yield encodelength(MAJOR_TYPE_BYTESTRING, len(chunk))
@@ -115,6 +122,7 @@
yield BREAK
+
def streamencodeint(v):
if v >= 18446744073709551616 or v < -18446744073709551616:
raise ValueError('big integers not supported')
@@ -124,6 +132,7 @@
else:
yield encodelength(MAJOR_TYPE_NEGINT, abs(v) - 1)
+
def streamencodearray(l):
"""Encode a known size iterable to an array."""
@@ -133,6 +142,7 @@
for chunk in streamencode(i):
yield chunk
+
def streamencodearrayfromiter(it):
"""Encode an iterator of items to an indefinite length array."""
@@ -144,9 +154,11 @@
yield BREAK
+
def _mixedtypesortkey(v):
return type(v).__name__, v
+
def streamencodeset(s):
# https://www.iana.org/assignments/cbor-tags/cbor-tags.xhtml defines
# semantic tag 258 for finite sets.
@@ -155,6 +167,7 @@
for chunk in streamencodearray(sorted(s, key=_mixedtypesortkey)):
yield chunk
+
def streamencodemap(d):
"""Encode dictionary to a generator.
@@ -162,13 +175,15 @@
"""
yield encodelength(MAJOR_TYPE_MAP, len(d))
- for key, value in sorted(d.iteritems(),
- key=lambda x: _mixedtypesortkey(x[0])):
+ for key, value in sorted(
+ d.iteritems(), key=lambda x: _mixedtypesortkey(x[0])
+ ):
for chunk in streamencode(key):
yield chunk
for chunk in streamencode(value):
yield chunk
+
def streamencodemapfromiter(it):
"""Given an iterable of (key, value), encode to an indefinite length map."""
yield BEGIN_INDEFINITE_MAP
@@ -181,14 +196,17 @@
yield BREAK
+
def streamencodebool(b):
# major type 7, simple value 20 and 21.
yield b'\xf5' if b else b'\xf4'
+
def streamencodenone(v):
# major type 7, simple value 22.
yield b'\xf6'
+
STREAM_ENCODERS = {
bytes: streamencodebytestring,
int: streamencodeint,
@@ -201,6 +219,7 @@
type(None): streamencodenone,
}
+
def streamencode(v):
"""Encode a value in a streaming manner.
@@ -226,16 +245,23 @@
return fn(v)
+
class CBORDecodeError(Exception):
"""Represents an error decoding CBOR."""
+
if sys.version_info.major >= 3:
+
def _elementtointeger(b, i):
return b[i]
+
+
else:
+
def _elementtointeger(b, i):
return ord(b[i])
+
STRUCT_BIG_UBYTE = struct.Struct(r'>B')
STRUCT_BIG_USHORT = struct.Struct('>H')
STRUCT_BIG_ULONG = struct.Struct('>L')
@@ -248,6 +274,7 @@
SPECIAL_START_SET = 4
SPECIAL_INDEFINITE_BREAK = 5
+
def decodeitem(b, offset=0):
"""Decode a new CBOR value from a buffer at offset.
@@ -301,8 +328,9 @@
elif majortype == MAJOR_TYPE_BYTESTRING:
# Beginning of bytestrings are treated as uints in order to
# decode their length, which may be indefinite.
- complete, size, readcount = decodeuint(subtype, b, offset,
- allowindefinite=True)
+ complete, size, readcount = decodeuint(
+ subtype, b, offset, allowindefinite=True
+ )
# We don't know the size of the bytestring. It must be a definitive
# length since the indefinite subtype would be encoded in the initial
@@ -314,7 +342,7 @@
if size is not None:
# And the data is available in the buffer.
if offset + readcount + size <= len(b):
- value = b[offset + readcount:offset + readcount + size]
+ value = b[offset + readcount : offset + readcount + size]
return True, value, readcount + size + 1, SPECIAL_NONE
# And we need more data in order to return the bytestring.
@@ -367,15 +395,17 @@
if offset + readcount >= len(b):
return False, None, -1, SPECIAL_NONE
- complete, size, readcount2, special = decodeitem(b,
- offset + readcount)
+ complete, size, readcount2, special = decodeitem(
+ b, offset + readcount
+ )
if not complete:
return False, None, readcount2, SPECIAL_NONE
if special != SPECIAL_START_ARRAY:
- raise CBORDecodeError('expected array after finite set '
- 'semantic tag')
+ raise CBORDecodeError(
+ 'expected array after finite set ' 'semantic tag'
+ )
return True, size, readcount + readcount2 + 1, SPECIAL_START_SET
@@ -398,6 +428,7 @@
else:
assert False
+
def decodeuint(subtype, b, offset=0, allowindefinite=False):
"""Decode an unsigned integer.
@@ -428,8 +459,9 @@
else:
raise CBORDecodeError('indefinite length uint not allowed here')
elif subtype >= 28:
- raise CBORDecodeError('unsupported subtype on integer type: %d' %
- subtype)
+ raise CBORDecodeError(
+ 'unsupported subtype on integer type: %d' % subtype
+ )
if subtype == 24:
s = STRUCT_BIG_UBYTE
@@ -447,6 +479,7 @@
else:
return False, None, len(b) - offset - s.size
+
class bytestringchunk(bytes):
"""Represents a chunk/segment in an indefinite length bytestring.
@@ -462,6 +495,7 @@
return self
+
class sansiodecoder(object):
"""A CBOR decoder that doesn't perform its own I/O.
@@ -606,32 +640,30 @@
self._decodedvalues.append(value)
elif special == SPECIAL_START_ARRAY:
- self._collectionstack.append({
- 'remaining': value,
- 'v': [],
- })
+ self._collectionstack.append(
+ {'remaining': value, 'v': [],}
+ )
self._state = self._STATE_WANT_ARRAY_VALUE
elif special == SPECIAL_START_MAP:
- self._collectionstack.append({
- 'remaining': value,
- 'v': {},
- })
+ self._collectionstack.append(
+ {'remaining': value, 'v': {},}
+ )
self._state = self._STATE_WANT_MAP_KEY
elif special == SPECIAL_START_SET:
- self._collectionstack.append({
- 'remaining': value,
- 'v': set(),
- })
+ self._collectionstack.append(
+ {'remaining': value, 'v': set(),}
+ )
self._state = self._STATE_WANT_SET_VALUE
elif special == SPECIAL_START_INDEFINITE_BYTESTRING:
self._state = self._STATE_WANT_BYTESTRING_CHUNK_FIRST
else:
- raise CBORDecodeError('unhandled special state: %d' %
- special)
+ raise CBORDecodeError(
+ 'unhandled special state: %d' % special
+ )
# This value becomes an element of the current array.
elif self._state == self._STATE_WANT_ARRAY_VALUE:
@@ -651,10 +683,9 @@
lastc['v'].append(newvalue)
lastc['remaining'] -= 1
- self._collectionstack.append({
- 'remaining': value,
- 'v': newvalue,
- })
+ self._collectionstack.append(
+ {'remaining': value, 'v': newvalue,}
+ )
# self._state doesn't need changed.
@@ -666,10 +697,9 @@
lastc['v'].append(newvalue)
lastc['remaining'] -= 1
- self._collectionstack.append({
- 'remaining': value,
- 'v': newvalue
- })
+ self._collectionstack.append(
+ {'remaining': value, 'v': newvalue}
+ )
self._state = self._STATE_WANT_MAP_KEY
@@ -680,20 +710,23 @@
lastc['v'].append(newvalue)
lastc['remaining'] -= 1
- self._collectionstack.append({
- 'remaining': value,
- 'v': newvalue,
- })
+ self._collectionstack.append(
+ {'remaining': value, 'v': newvalue,}
+ )
self._state = self._STATE_WANT_SET_VALUE
elif special == SPECIAL_START_INDEFINITE_BYTESTRING:
- raise CBORDecodeError('indefinite length bytestrings '
- 'not allowed as array values')
+ raise CBORDecodeError(
+ 'indefinite length bytestrings '
+ 'not allowed as array values'
+ )
else:
- raise CBORDecodeError('unhandled special item when '
- 'expecting array value: %d' % special)
+ raise CBORDecodeError(
+ 'unhandled special item when '
+ 'expecting array value: %d' % special
+ )
# This value becomes the key of the current map instance.
elif self._state == self._STATE_WANT_MAP_KEY:
@@ -702,18 +735,26 @@
self._state = self._STATE_WANT_MAP_VALUE
elif special == SPECIAL_START_INDEFINITE_BYTESTRING:
- raise CBORDecodeError('indefinite length bytestrings '
- 'not allowed as map keys')
+ raise CBORDecodeError(
+ 'indefinite length bytestrings '
+ 'not allowed as map keys'
+ )
- elif special in (SPECIAL_START_ARRAY, SPECIAL_START_MAP,
- SPECIAL_START_SET):
- raise CBORDecodeError('collections not supported as map '
- 'keys')
+ elif special in (
+ SPECIAL_START_ARRAY,
+ SPECIAL_START_MAP,
+ SPECIAL_START_SET,
+ ):
+ raise CBORDecodeError(
+ 'collections not supported as map ' 'keys'
+ )
# We do not allow special values to be used as map keys.
else:
- raise CBORDecodeError('unhandled special item when '
- 'expecting map key: %d' % special)
+ raise CBORDecodeError(
+ 'unhandled special item when '
+ 'expecting map key: %d' % special
+ )
# This value becomes the value of the current map key.
elif self._state == self._STATE_WANT_MAP_VALUE:
@@ -733,10 +774,9 @@
lastc['v'][self._currentmapkey] = newvalue
lastc['remaining'] -= 1
- self._collectionstack.append({
- 'remaining': value,
- 'v': newvalue,
- })
+ self._collectionstack.append(
+ {'remaining': value, 'v': newvalue,}
+ )
self._state = self._STATE_WANT_ARRAY_VALUE
@@ -748,10 +788,9 @@
lastc['v'][self._currentmapkey] = newvalue
lastc['remaining'] -= 1
- self._collectionstack.append({
- 'remaining': value,
- 'v': newvalue,
- })
+ self._collectionstack.append(
+ {'remaining': value, 'v': newvalue,}
+ )
self._state = self._STATE_WANT_MAP_KEY
@@ -763,20 +802,23 @@
lastc['v'][self._currentmapkey] = newvalue
lastc['remaining'] -= 1
- self._collectionstack.append({
- 'remaining': value,
- 'v': newvalue,
- })
+ self._collectionstack.append(
+ {'remaining': value, 'v': newvalue,}
+ )
self._state = self._STATE_WANT_SET_VALUE
elif special == SPECIAL_START_INDEFINITE_BYTESTRING:
- raise CBORDecodeError('indefinite length bytestrings not '
- 'allowed as map values')
+ raise CBORDecodeError(
+ 'indefinite length bytestrings not '
+ 'allowed as map values'
+ )
else:
- raise CBORDecodeError('unhandled special item when '
- 'expecting map value: %d' % special)
+ raise CBORDecodeError(
+ 'unhandled special item when '
+ 'expecting map value: %d' % special
+ )
self._currentmapkey = None
@@ -788,27 +830,35 @@
lastc['remaining'] -= 1
elif special == SPECIAL_START_INDEFINITE_BYTESTRING:
- raise CBORDecodeError('indefinite length bytestrings not '
- 'allowed as set values')
+ raise CBORDecodeError(
+ 'indefinite length bytestrings not '
+ 'allowed as set values'
+ )
- elif special in (SPECIAL_START_ARRAY,
- SPECIAL_START_MAP,
- SPECIAL_START_SET):
- raise CBORDecodeError('collections not allowed as set '
- 'values')
+ elif special in (
+ SPECIAL_START_ARRAY,
+ SPECIAL_START_MAP,
+ SPECIAL_START_SET,
+ ):
+ raise CBORDecodeError(
+ 'collections not allowed as set ' 'values'
+ )
# We don't allow non-trivial types to exist as set values.
else:
- raise CBORDecodeError('unhandled special item when '
- 'expecting set value: %d' % special)
+ raise CBORDecodeError(
+ 'unhandled special item when '
+ 'expecting set value: %d' % special
+ )
# This value represents the first chunk in an indefinite length
# bytestring.
elif self._state == self._STATE_WANT_BYTESTRING_CHUNK_FIRST:
# We received a full chunk.
if special == SPECIAL_NONE:
- self._decodedvalues.append(bytestringchunk(value,
- first=True))
+ self._decodedvalues.append(
+ bytestringchunk(value, first=True)
+ )
self._state = self._STATE_WANT_BYTESTRING_CHUNK_SUBSEQUENT
@@ -818,9 +868,9 @@
# We /could/ convert this to a b''. But we want to preserve
# the nature of the underlying data so consumers expecting
# an indefinite length bytestring get one.
- self._decodedvalues.append(bytestringchunk(b'',
- first=True,
- last=True))
+ self._decodedvalues.append(
+ bytestringchunk(b'', first=True, last=True)
+ )
# Since indefinite length bytestrings can't be used in
# collections, we must be at the root level.
@@ -828,9 +878,10 @@
self._state = self._STATE_NONE
else:
- raise CBORDecodeError('unexpected special value when '
- 'expecting bytestring chunk: %d' %
- special)
+ raise CBORDecodeError(
+ 'unexpected special value when '
+ 'expecting bytestring chunk: %d' % special
+ )
# This value represents the non-initial chunk in an indefinite
# length bytestring.
@@ -849,21 +900,25 @@
self._state = self._STATE_NONE
else:
- raise CBORDecodeError('unexpected special value when '
- 'expecting bytestring chunk: %d' %
- special)
+ raise CBORDecodeError(
+ 'unexpected special value when '
+ 'expecting bytestring chunk: %d' % special
+ )
else:
- raise CBORDecodeError('unhandled decoder state: %d' %
- self._state)
+ raise CBORDecodeError(
+ 'unhandled decoder state: %d' % self._state
+ )
# We could have just added the final value in a collection. End
# all complete collections at the top of the stack.
while True:
# Bail if we're not waiting on a new collection item.
- if self._state not in (self._STATE_WANT_ARRAY_VALUE,
- self._STATE_WANT_MAP_KEY,
- self._STATE_WANT_SET_VALUE):
+ if self._state not in (
+ self._STATE_WANT_ARRAY_VALUE,
+ self._STATE_WANT_MAP_KEY,
+ self._STATE_WANT_SET_VALUE,
+ ):
break
# Or we are expecting more items for this collection.
@@ -909,6 +964,7 @@
self._decodedvalues = []
return l
+
class bufferingdecoder(object):
"""A CBOR decoder that buffers undecoded input.
@@ -919,6 +975,7 @@
TODO consider adding limits as to the maximum amount of data that can
be buffered.
"""
+
def __init__(self):
self._decoder = sansiodecoder()
self._chunks = []
@@ -978,6 +1035,7 @@
def getavailable(self):
return self._decoder.getavailable()
+
def decodeall(b):
"""Decode all CBOR items present in an iterable of bytes.