--- a/black.toml Tue Jan 21 17:15:34 2020 -0800
+++ b/black.toml Thu Jan 23 13:10:48 2020 -0800
@@ -9,7 +9,6 @@
| \.mypy_cache/
| \.venv/
| mercurial/thirdparty/
-| contrib/python-zstandard/
'''
skip-string-normalization = true
quiet = true
--- a/contrib/examples/fix.hgrc Tue Jan 21 17:15:34 2020 -0800
+++ b/contrib/examples/fix.hgrc Thu Jan 23 13:10:48 2020 -0800
@@ -6,7 +6,7 @@
rustfmt:pattern = set:**.rs
black:command = black --config=black.toml -
-black:pattern = set:**.py - mercurial/thirdparty/** - "contrib/python-zstandard/**"
+black:pattern = set:**.py - mercurial/thirdparty/**
# Mercurial doesn't have any Go code, but if we did this is how we
# would configure `hg fix` for Go:
--- a/contrib/packaging/inno/requirements.txt Tue Jan 21 17:15:34 2020 -0800
+++ b/contrib/packaging/inno/requirements.txt Thu Jan 23 13:10:48 2020 -0800
@@ -8,6 +8,10 @@
--hash=sha256:e4f3620cfea4f83eedc95b24abd9cd56f3c4b146dd0177e83a21b4eb49e21e50 \
--hash=sha256:fd7c7c74727ddcf00e9acd26bba8da604ffec95bf1c2144e67aff7a8b50e6cef \
# via dulwich
+configparser==4.0.2 \
+ --hash=sha256:254c1d9c79f60c45dfde850850883d5aaa7f19a23f13561243a050d5a7c3fe4c \
+ --hash=sha256:c7d282687a5308319bf3d2e7706e575c635b0a470342641c93bea0ea3b5331df \
+ # via entrypoints
docutils==0.15.2 \
--hash=sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0 \
--hash=sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827 \
@@ -34,8 +38,7 @@
--hash=sha256:881c4c157e45f30af185c1ffe8d549d48ac9127433f2c380c24b84572ad66297
pywin32-ctypes==0.2.0 \
--hash=sha256:24ffc3b341d457d48e8922352130cf2644024a4ff09762a2261fd34c36ee5942 \
- --hash=sha256:9dc2d991b3479cc2df15930958b674a48a227d5361d413827a4cfd0b5876fc98 \
- # via keyring
+ --hash=sha256:9dc2d991b3479cc2df15930958b674a48a227d5361d413827a4cfd0b5876fc98
urllib3==1.25.6 \
--hash=sha256:3de946ffbed6e6746608990594d08faac602528ac7015ac28d33cee6a45b7398 \
--hash=sha256:9a107b99a5393caf59c7aa3c1249c16e6879447533d0887f4336dde834c7be86 \
--- a/contrib/python-zstandard/make_cffi.py Tue Jan 21 17:15:34 2020 -0800
+++ b/contrib/python-zstandard/make_cffi.py Thu Jan 23 13:10:48 2020 -0800
@@ -52,7 +52,8 @@
# Headers whose preprocessed output will be fed into cdef().
HEADERS = [
- os.path.join(HERE, "zstd", *p) for p in (("zstd.h",), ("dictBuilder", "zdict.h"),)
+ os.path.join(HERE, "zstd", *p)
+ for p in (("zstd.h",), ("dictBuilder", "zdict.h"),)
]
INCLUDE_DIRS = [
@@ -139,7 +140,9 @@
env = dict(os.environ)
if getattr(compiler, "_paths", None):
env["PATH"] = compiler._paths
- process = subprocess.Popen(args + [input_file], stdout=subprocess.PIPE, env=env)
+ process = subprocess.Popen(
+ args + [input_file], stdout=subprocess.PIPE, env=env
+ )
output = process.communicate()[0]
ret = process.poll()
if ret:
--- a/contrib/python-zstandard/setup.py Tue Jan 21 17:15:34 2020 -0800
+++ b/contrib/python-zstandard/setup.py Thu Jan 23 13:10:48 2020 -0800
@@ -87,7 +87,9 @@
break
if not version:
- raise Exception("could not resolve package version; " "this should never happen")
+ raise Exception(
+ "could not resolve package version; " "this should never happen"
+ )
setup(
name="zstandard",
--- a/contrib/python-zstandard/setup_zstd.py Tue Jan 21 17:15:34 2020 -0800
+++ b/contrib/python-zstandard/setup_zstd.py Thu Jan 23 13:10:48 2020 -0800
@@ -138,12 +138,16 @@
if not system_zstd:
sources.update([os.path.join(actual_root, p) for p in zstd_sources])
if support_legacy:
- sources.update([os.path.join(actual_root, p) for p in zstd_sources_legacy])
+ sources.update(
+ [os.path.join(actual_root, p) for p in zstd_sources_legacy]
+ )
sources = list(sources)
include_dirs = set([os.path.join(actual_root, d) for d in ext_includes])
if not system_zstd:
- include_dirs.update([os.path.join(actual_root, d) for d in zstd_includes])
+ include_dirs.update(
+ [os.path.join(actual_root, d) for d in zstd_includes]
+ )
if support_legacy:
include_dirs.update(
[os.path.join(actual_root, d) for d in zstd_includes_legacy]
--- a/contrib/python-zstandard/tests/common.py Tue Jan 21 17:15:34 2020 -0800
+++ b/contrib/python-zstandard/tests/common.py Thu Jan 23 13:10:48 2020 -0800
@@ -50,7 +50,9 @@
os.environ.update(old_env)
if mod.backend != "cffi":
- raise Exception("got the zstandard %s backend instead of cffi" % mod.backend)
+ raise Exception(
+ "got the zstandard %s backend instead of cffi" % mod.backend
+ )
# If CFFI version is available, dynamically construct test methods
# that use it.
@@ -84,7 +86,9 @@
fn.__func__.func_defaults,
fn.__func__.func_closure,
)
- new_method = types.UnboundMethodType(new_fn, fn.im_self, fn.im_class)
+ new_method = types.UnboundMethodType(
+ new_fn, fn.im_self, fn.im_class
+ )
setattr(cls, name, new_method)
@@ -194,4 +198,6 @@
expensive_settings = hypothesis.settings(deadline=None, max_examples=10000)
hypothesis.settings.register_profile("expensive", expensive_settings)
- hypothesis.settings.load_profile(os.environ.get("HYPOTHESIS_PROFILE", "default"))
+ hypothesis.settings.load_profile(
+ os.environ.get("HYPOTHESIS_PROFILE", "default")
+ )
--- a/contrib/python-zstandard/tests/test_buffer_util.py Tue Jan 21 17:15:34 2020 -0800
+++ b/contrib/python-zstandard/tests/test_buffer_util.py Thu Jan 23 13:10:48 2020 -0800
@@ -67,7 +67,8 @@
self.skipTest("BufferWithSegments not available")
b = zstd.BufferWithSegments(
- b"foofooxfooxy", b"".join([ss.pack(0, 3), ss.pack(3, 4), ss.pack(7, 5)])
+ b"foofooxfooxy",
+ b"".join([ss.pack(0, 3), ss.pack(3, 4), ss.pack(7, 5)]),
)
self.assertEqual(len(b), 3)
self.assertEqual(b.size, 12)
@@ -83,17 +84,23 @@
if not hasattr(zstd, "BufferWithSegmentsCollection"):
self.skipTest("BufferWithSegmentsCollection not available")
- with self.assertRaisesRegex(ValueError, "must pass at least 1 argument"):
+ with self.assertRaisesRegex(
+ ValueError, "must pass at least 1 argument"
+ ):
zstd.BufferWithSegmentsCollection()
def test_argument_validation(self):
if not hasattr(zstd, "BufferWithSegmentsCollection"):
self.skipTest("BufferWithSegmentsCollection not available")
- with self.assertRaisesRegex(TypeError, "arguments must be BufferWithSegments"):
+ with self.assertRaisesRegex(
+ TypeError, "arguments must be BufferWithSegments"
+ ):
zstd.BufferWithSegmentsCollection(None)
- with self.assertRaisesRegex(TypeError, "arguments must be BufferWithSegments"):
+ with self.assertRaisesRegex(
+ TypeError, "arguments must be BufferWithSegments"
+ ):
zstd.BufferWithSegmentsCollection(
zstd.BufferWithSegments(b"foo", ss.pack(0, 3)), None
)
--- a/contrib/python-zstandard/tests/test_compressor.py Tue Jan 21 17:15:34 2020 -0800
+++ b/contrib/python-zstandard/tests/test_compressor.py Thu Jan 23 13:10:48 2020 -0800
@@ -24,7 +24,9 @@
def multithreaded_chunk_size(level, source_size=0):
- params = zstd.ZstdCompressionParameters.from_level(level, source_size=source_size)
+ params = zstd.ZstdCompressionParameters.from_level(
+ level, source_size=source_size
+ )
return 1 << (params.window_log + 2)
@@ -86,7 +88,9 @@
# This matches the test for read_to_iter() below.
cctx = zstd.ZstdCompressor(level=1, write_content_size=False)
- result = cctx.compress(b"f" * zstd.COMPRESSION_RECOMMENDED_INPUT_SIZE + b"o")
+ result = cctx.compress(
+ b"f" * zstd.COMPRESSION_RECOMMENDED_INPUT_SIZE + b"o"
+ )
self.assertEqual(
result,
b"\x28\xb5\x2f\xfd\x00\x40\x54\x00\x00"
@@ -99,7 +103,9 @@
result = cctx.compress(b"foo" * 256)
def test_no_magic(self):
- params = zstd.ZstdCompressionParameters.from_level(1, format=zstd.FORMAT_ZSTD1)
+ params = zstd.ZstdCompressionParameters.from_level(
+ 1, format=zstd.FORMAT_ZSTD1
+ )
cctx = zstd.ZstdCompressor(compression_params=params)
magic = cctx.compress(b"foobar")
@@ -223,7 +229,8 @@
self.assertEqual(
result,
- b"\x28\xb5\x2f\xfd\x23\x8f\x55\x0f\x70\x03\x19\x00\x00" b"\x66\x6f\x6f",
+ b"\x28\xb5\x2f\xfd\x23\x8f\x55\x0f\x70\x03\x19\x00\x00"
+ b"\x66\x6f\x6f",
)
def test_multithreaded_compression_params(self):
@@ -234,7 +241,9 @@
params = zstd.get_frame_parameters(result)
self.assertEqual(params.content_size, 3)
- self.assertEqual(result, b"\x28\xb5\x2f\xfd\x20\x03\x19\x00\x00\x66\x6f\x6f")
+ self.assertEqual(
+ result, b"\x28\xb5\x2f\xfd\x20\x03\x19\x00\x00\x66\x6f\x6f"
+ )
@make_cffi
@@ -347,7 +356,9 @@
)
self.assertEqual(cobj.compress(b"bar"), b"")
# 3 byte header plus content.
- self.assertEqual(cobj.flush(zstd.COMPRESSOBJ_FLUSH_BLOCK), b"\x18\x00\x00bar")
+ self.assertEqual(
+ cobj.flush(zstd.COMPRESSOBJ_FLUSH_BLOCK), b"\x18\x00\x00bar"
+ )
self.assertEqual(cobj.flush(), b"\x01\x00\x00")
def test_flush_empty_block(self):
@@ -445,7 +456,9 @@
self.assertEqual(int(r), 0)
self.assertEqual(w, 9)
- self.assertEqual(dest.getvalue(), b"\x28\xb5\x2f\xfd\x00\x48\x01\x00\x00")
+ self.assertEqual(
+ dest.getvalue(), b"\x28\xb5\x2f\xfd\x00\x48\x01\x00\x00"
+ )
def test_large_data(self):
source = io.BytesIO()
@@ -478,7 +491,9 @@
cctx = zstd.ZstdCompressor(level=1, write_checksum=True)
cctx.copy_stream(source, with_checksum)
- self.assertEqual(len(with_checksum.getvalue()), len(no_checksum.getvalue()) + 4)
+ self.assertEqual(
+ len(with_checksum.getvalue()), len(no_checksum.getvalue()) + 4
+ )
no_params = zstd.get_frame_parameters(no_checksum.getvalue())
with_params = zstd.get_frame_parameters(with_checksum.getvalue())
@@ -585,7 +600,9 @@
cctx = zstd.ZstdCompressor()
with cctx.stream_reader(b"foo") as reader:
- with self.assertRaisesRegex(ValueError, "cannot __enter__ multiple times"):
+ with self.assertRaisesRegex(
+ ValueError, "cannot __enter__ multiple times"
+ ):
with reader as reader2:
pass
@@ -744,7 +761,9 @@
source = io.BytesIO(b"foobar")
with cctx.stream_reader(source, size=2) as reader:
- with self.assertRaisesRegex(zstd.ZstdError, "Src size is incorrect"):
+ with self.assertRaisesRegex(
+ zstd.ZstdError, "Src size is incorrect"
+ ):
reader.read(10)
# Try another compression operation.
@@ -1126,7 +1145,9 @@
self.assertFalse(no_params.has_checksum)
self.assertTrue(with_params.has_checksum)
- self.assertEqual(len(with_checksum.getvalue()), len(no_checksum.getvalue()) + 4)
+ self.assertEqual(
+ len(with_checksum.getvalue()), len(no_checksum.getvalue()) + 4
+ )
def test_write_content_size(self):
no_size = NonClosingBytesIO()
@@ -1145,7 +1166,9 @@
# Declaring size will write the header.
with_size = NonClosingBytesIO()
- with cctx.stream_writer(with_size, size=len(b"foobar" * 256)) as compressor:
+ with cctx.stream_writer(
+ with_size, size=len(b"foobar" * 256)
+ ) as compressor:
self.assertEqual(compressor.write(b"foobar" * 256), 0)
no_params = zstd.get_frame_parameters(no_size.getvalue())
@@ -1191,7 +1214,9 @@
self.assertFalse(no_params.has_checksum)
self.assertFalse(with_params.has_checksum)
- self.assertEqual(len(with_dict_id.getvalue()), len(no_dict_id.getvalue()) + 4)
+ self.assertEqual(
+ len(with_dict_id.getvalue()), len(no_dict_id.getvalue()) + 4
+ )
def test_memory_size(self):
cctx = zstd.ZstdCompressor(level=3)
@@ -1337,7 +1362,9 @@
for chunk in cctx.read_to_iter(b"foobar"):
pass
- with self.assertRaisesRegex(ValueError, "must pass an object with a read"):
+ with self.assertRaisesRegex(
+ ValueError, "must pass an object with a read"
+ ):
for chunk in cctx.read_to_iter(True):
pass
@@ -1513,7 +1540,9 @@
dctx = zstd.ZstdDecompressor()
- self.assertEqual(dctx.decompress(b"".join(chunks)), (b"x" * 1000) + (b"y" * 24))
+ self.assertEqual(
+ dctx.decompress(b"".join(chunks)), (b"x" * 1000) + (b"y" * 24)
+ )
def test_small_chunk_size(self):
cctx = zstd.ZstdCompressor()
@@ -1533,7 +1562,8 @@
dctx = zstd.ZstdDecompressor()
self.assertEqual(
- dctx.decompress(b"".join(chunks), max_output_size=10000), b"foo" * 1024
+ dctx.decompress(b"".join(chunks), max_output_size=10000),
+ b"foo" * 1024,
)
def test_input_types(self):
@@ -1602,7 +1632,8 @@
list(chunker.finish())
with self.assertRaisesRegex(
- zstd.ZstdError, r"cannot call compress\(\) after compression finished"
+ zstd.ZstdError,
+ r"cannot call compress\(\) after compression finished",
):
list(chunker.compress(b"foo"))
@@ -1644,7 +1675,9 @@
with self.assertRaises(TypeError):
cctx.multi_compress_to_buffer((1, 2))
- with self.assertRaisesRegex(TypeError, "item 0 not a bytes like object"):
+ with self.assertRaisesRegex(
+ TypeError, "item 0 not a bytes like object"
+ ):
cctx.multi_compress_to_buffer([u"foo"])
def test_empty_input(self):
--- a/contrib/python-zstandard/tests/test_compressor_fuzzing.py Tue Jan 21 17:15:34 2020 -0800
+++ b/contrib/python-zstandard/tests/test_compressor_fuzzing.py Thu Jan 23 13:10:48 2020 -0800
@@ -28,9 +28,13 @@
original=strategies.sampled_from(random_input_data()),
level=strategies.integers(min_value=1, max_value=5),
source_read_size=strategies.integers(1, 16384),
- read_size=strategies.integers(-1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE),
+ read_size=strategies.integers(
+ -1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE
+ ),
)
- def test_stream_source_read(self, original, level, source_read_size, read_size):
+ def test_stream_source_read(
+ self, original, level, source_read_size, read_size
+ ):
if read_size == 0:
read_size = -1
@@ -58,9 +62,13 @@
original=strategies.sampled_from(random_input_data()),
level=strategies.integers(min_value=1, max_value=5),
source_read_size=strategies.integers(1, 16384),
- read_size=strategies.integers(-1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE),
+ read_size=strategies.integers(
+ -1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE
+ ),
)
- def test_buffer_source_read(self, original, level, source_read_size, read_size):
+ def test_buffer_source_read(
+ self, original, level, source_read_size, read_size
+ ):
if read_size == 0:
read_size = -1
@@ -155,9 +163,13 @@
original=strategies.sampled_from(random_input_data()),
level=strategies.integers(min_value=1, max_value=5),
source_read_size=strategies.integers(1, 16384),
- read_size=strategies.integers(1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE),
+ read_size=strategies.integers(
+ 1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE
+ ),
)
- def test_stream_source_readinto(self, original, level, source_read_size, read_size):
+ def test_stream_source_readinto(
+ self, original, level, source_read_size, read_size
+ ):
refctx = zstd.ZstdCompressor(level=level)
ref_frame = refctx.compress(original)
@@ -184,9 +196,13 @@
original=strategies.sampled_from(random_input_data()),
level=strategies.integers(min_value=1, max_value=5),
source_read_size=strategies.integers(1, 16384),
- read_size=strategies.integers(1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE),
+ read_size=strategies.integers(
+ 1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE
+ ),
)
- def test_buffer_source_readinto(self, original, level, source_read_size, read_size):
+ def test_buffer_source_readinto(
+ self, original, level, source_read_size, read_size
+ ):
refctx = zstd.ZstdCompressor(level=level)
ref_frame = refctx.compress(original)
@@ -285,9 +301,13 @@
original=strategies.sampled_from(random_input_data()),
level=strategies.integers(min_value=1, max_value=5),
source_read_size=strategies.integers(1, 16384),
- read_size=strategies.integers(-1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE),
+ read_size=strategies.integers(
+ -1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE
+ ),
)
- def test_stream_source_read1(self, original, level, source_read_size, read_size):
+ def test_stream_source_read1(
+ self, original, level, source_read_size, read_size
+ ):
if read_size == 0:
read_size = -1
@@ -315,9 +335,13 @@
original=strategies.sampled_from(random_input_data()),
level=strategies.integers(min_value=1, max_value=5),
source_read_size=strategies.integers(1, 16384),
- read_size=strategies.integers(-1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE),
+ read_size=strategies.integers(
+ -1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE
+ ),
)
- def test_buffer_source_read1(self, original, level, source_read_size, read_size):
+ def test_buffer_source_read1(
+ self, original, level, source_read_size, read_size
+ ):
if read_size == 0:
read_size = -1
@@ -412,7 +436,9 @@
original=strategies.sampled_from(random_input_data()),
level=strategies.integers(min_value=1, max_value=5),
source_read_size=strategies.integers(1, 16384),
- read_size=strategies.integers(1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE),
+ read_size=strategies.integers(
+ 1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE
+ ),
)
def test_stream_source_readinto1(
self, original, level, source_read_size, read_size
@@ -446,7 +472,9 @@
original=strategies.sampled_from(random_input_data()),
level=strategies.integers(min_value=1, max_value=5),
source_read_size=strategies.integers(1, 16384),
- read_size=strategies.integers(1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE),
+ read_size=strategies.integers(
+ 1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE
+ ),
)
def test_buffer_source_readinto1(
self, original, level, source_read_size, read_size
@@ -576,7 +604,9 @@
read_size=strategies.integers(min_value=1, max_value=1048576),
write_size=strategies.integers(min_value=1, max_value=1048576),
)
- def test_read_write_size_variance(self, original, level, read_size, write_size):
+ def test_read_write_size_variance(
+ self, original, level, read_size, write_size
+ ):
refctx = zstd.ZstdCompressor(level=level)
ref_frame = refctx.compress(original)
@@ -585,7 +615,11 @@
dest = io.BytesIO()
cctx.copy_stream(
- source, dest, size=len(original), read_size=read_size, write_size=write_size
+ source,
+ dest,
+ size=len(original),
+ read_size=read_size,
+ write_size=write_size,
)
self.assertEqual(dest.getvalue(), ref_frame)
@@ -675,7 +709,9 @@
decompressed_chunks.append(dobj.decompress(chunk))
self.assertEqual(
- dctx.decompress(b"".join(compressed_chunks), max_output_size=len(original)),
+ dctx.decompress(
+ b"".join(compressed_chunks), max_output_size=len(original)
+ ),
original,
)
self.assertEqual(b"".join(decompressed_chunks), original)
@@ -690,7 +726,9 @@
read_size=strategies.integers(min_value=1, max_value=4096),
write_size=strategies.integers(min_value=1, max_value=4096),
)
- def test_read_write_size_variance(self, original, level, read_size, write_size):
+ def test_read_write_size_variance(
+ self, original, level, read_size, write_size
+ ):
refcctx = zstd.ZstdCompressor(level=level)
ref_frame = refcctx.compress(original)
@@ -699,7 +737,10 @@
cctx = zstd.ZstdCompressor(level=level)
chunks = list(
cctx.read_to_iter(
- source, size=len(original), read_size=read_size, write_size=write_size
+ source,
+ size=len(original),
+ read_size=read_size,
+ write_size=write_size,
)
)
@@ -710,7 +751,9 @@
class TestCompressor_multi_compress_to_buffer_fuzzing(TestCase):
@hypothesis.given(
original=strategies.lists(
- strategies.sampled_from(random_input_data()), min_size=1, max_size=1024
+ strategies.sampled_from(random_input_data()),
+ min_size=1,
+ max_size=1024,
),
threads=strategies.integers(min_value=1, max_value=8),
use_dict=strategies.booleans(),
@@ -776,7 +819,8 @@
dctx = zstd.ZstdDecompressor()
self.assertEqual(
- dctx.decompress(b"".join(chunks), max_output_size=len(original)), original
+ dctx.decompress(b"".join(chunks), max_output_size=len(original)),
+ original,
)
self.assertTrue(all(len(chunk) == chunk_size for chunk in chunks[:-1]))
@@ -794,7 +838,9 @@
input_sizes=strategies.data(),
flushes=strategies.data(),
)
- def test_flush_block(self, original, level, chunk_size, input_sizes, flushes):
+ def test_flush_block(
+ self, original, level, chunk_size, input_sizes, flushes
+ ):
cctx = zstd.ZstdCompressor(level=level)
chunker = cctx.chunker(chunk_size=chunk_size)
@@ -830,7 +876,9 @@
decompressed_chunks.append(dobj.decompress(b"".join(chunks)))
self.assertEqual(
- dctx.decompress(b"".join(compressed_chunks), max_output_size=len(original)),
+ dctx.decompress(
+ b"".join(compressed_chunks), max_output_size=len(original)
+ ),
original,
)
self.assertEqual(b"".join(decompressed_chunks), original)
--- a/contrib/python-zstandard/tests/test_data_structures.py Tue Jan 21 17:15:34 2020 -0800
+++ b/contrib/python-zstandard/tests/test_data_structures.py Thu Jan 23 13:10:48 2020 -0800
@@ -65,7 +65,9 @@
p = zstd.ZstdCompressionParameters(threads=4)
self.assertEqual(p.threads, 4)
- p = zstd.ZstdCompressionParameters(threads=2, job_size=1048576, overlap_log=6)
+ p = zstd.ZstdCompressionParameters(
+ threads=2, job_size=1048576, overlap_log=6
+ )
self.assertEqual(p.threads, 2)
self.assertEqual(p.job_size, 1048576)
self.assertEqual(p.overlap_log, 6)
@@ -128,7 +130,9 @@
with self.assertRaisesRegex(
ValueError, "cannot specify both ldm_hash_rate_log"
):
- zstd.ZstdCompressionParameters(ldm_hash_rate_log=8, ldm_hash_every_log=4)
+ zstd.ZstdCompressionParameters(
+ ldm_hash_rate_log=8, ldm_hash_every_log=4
+ )
p = zstd.ZstdCompressionParameters(ldm_hash_rate_log=8)
self.assertEqual(p.ldm_hash_every_log, 8)
@@ -137,7 +141,9 @@
self.assertEqual(p.ldm_hash_every_log, 16)
def test_overlap_log(self):
- with self.assertRaisesRegex(ValueError, "cannot specify both overlap_log"):
+ with self.assertRaisesRegex(
+ ValueError, "cannot specify both overlap_log"
+ ):
zstd.ZstdCompressionParameters(overlap_log=1, overlap_size_log=9)
p = zstd.ZstdCompressionParameters(overlap_log=2)
@@ -169,10 +175,14 @@
zstd.get_frame_parameters(u"foobarbaz")
def test_invalid_input_sizes(self):
- with self.assertRaisesRegex(zstd.ZstdError, "not enough data for frame"):
+ with self.assertRaisesRegex(
+ zstd.ZstdError, "not enough data for frame"
+ ):
zstd.get_frame_parameters(b"")
- with self.assertRaisesRegex(zstd.ZstdError, "not enough data for frame"):
+ with self.assertRaisesRegex(
+ zstd.ZstdError, "not enough data for frame"
+ ):
zstd.get_frame_parameters(zstd.FRAME_HEADER)
def test_invalid_frame(self):
@@ -201,7 +211,9 @@
self.assertTrue(params.has_checksum)
# Upper 2 bits indicate content size.
- params = zstd.get_frame_parameters(zstd.FRAME_HEADER + b"\x40\x00\xff\x00")
+ params = zstd.get_frame_parameters(
+ zstd.FRAME_HEADER + b"\x40\x00\xff\x00"
+ )
self.assertEqual(params.content_size, 511)
self.assertEqual(params.window_size, 1024)
self.assertEqual(params.dict_id, 0)
@@ -215,7 +227,9 @@
self.assertFalse(params.has_checksum)
# Set multiple things.
- params = zstd.get_frame_parameters(zstd.FRAME_HEADER + b"\x45\x40\x0f\x10\x00")
+ params = zstd.get_frame_parameters(
+ zstd.FRAME_HEADER + b"\x45\x40\x0f\x10\x00"
+ )
self.assertEqual(params.content_size, 272)
self.assertEqual(params.window_size, 262144)
self.assertEqual(params.dict_id, 15)
--- a/contrib/python-zstandard/tests/test_data_structures_fuzzing.py Tue Jan 21 17:15:34 2020 -0800
+++ b/contrib/python-zstandard/tests/test_data_structures_fuzzing.py Thu Jan 23 13:10:48 2020 -0800
@@ -23,7 +23,9 @@
s_chainlog = strategies.integers(
min_value=zstd.CHAINLOG_MIN, max_value=zstd.CHAINLOG_MAX
)
-s_hashlog = strategies.integers(min_value=zstd.HASHLOG_MIN, max_value=zstd.HASHLOG_MAX)
+s_hashlog = strategies.integers(
+ min_value=zstd.HASHLOG_MIN, max_value=zstd.HASHLOG_MAX
+)
s_searchlog = strategies.integers(
min_value=zstd.SEARCHLOG_MIN, max_value=zstd.SEARCHLOG_MAX
)
@@ -61,7 +63,14 @@
s_strategy,
)
def test_valid_init(
- self, windowlog, chainlog, hashlog, searchlog, minmatch, targetlength, strategy
+ self,
+ windowlog,
+ chainlog,
+ hashlog,
+ searchlog,
+ minmatch,
+ targetlength,
+ strategy,
):
zstd.ZstdCompressionParameters(
window_log=windowlog,
@@ -83,7 +92,14 @@
s_strategy,
)
def test_estimated_compression_context_size(
- self, windowlog, chainlog, hashlog, searchlog, minmatch, targetlength, strategy
+ self,
+ windowlog,
+ chainlog,
+ hashlog,
+ searchlog,
+ minmatch,
+ targetlength,
+ strategy,
):
if minmatch == zstd.MINMATCH_MIN and strategy in (
zstd.STRATEGY_FAST,
--- a/contrib/python-zstandard/tests/test_decompressor.py Tue Jan 21 17:15:34 2020 -0800
+++ b/contrib/python-zstandard/tests/test_decompressor.py Thu Jan 23 13:10:48 2020 -0800
@@ -170,11 +170,15 @@
dctx.decompress(compressed, max_output_size=len(source) - 1)
# Input size + 1 works
- decompressed = dctx.decompress(compressed, max_output_size=len(source) + 1)
+ decompressed = dctx.decompress(
+ compressed, max_output_size=len(source) + 1
+ )
self.assertEqual(decompressed, source)
# A much larger buffer works.
- decompressed = dctx.decompress(compressed, max_output_size=len(source) * 64)
+ decompressed = dctx.decompress(
+ compressed, max_output_size=len(source) * 64
+ )
self.assertEqual(decompressed, source)
def test_stupidly_large_output_buffer(self):
@@ -237,7 +241,8 @@
dctx = zstd.ZstdDecompressor(max_window_size=2 ** zstd.WINDOWLOG_MIN)
with self.assertRaisesRegex(
- zstd.ZstdError, "decompression error: Frame requires too much memory"
+ zstd.ZstdError,
+ "decompression error: Frame requires too much memory",
):
dctx.decompress(frame, max_output_size=len(source))
@@ -291,7 +296,9 @@
self.assertEqual(w, len(source.getvalue()))
def test_read_write_size(self):
- source = OpCountingBytesIO(zstd.ZstdCompressor().compress(b"foobarfoobar"))
+ source = OpCountingBytesIO(
+ zstd.ZstdCompressor().compress(b"foobarfoobar")
+ )
dest = OpCountingBytesIO()
dctx = zstd.ZstdDecompressor()
@@ -309,7 +316,9 @@
dctx = zstd.ZstdDecompressor()
with dctx.stream_reader(b"foo") as reader:
- with self.assertRaisesRegex(ValueError, "cannot __enter__ multiple times"):
+ with self.assertRaisesRegex(
+ ValueError, "cannot __enter__ multiple times"
+ ):
with reader as reader2:
pass
@@ -474,7 +483,9 @@
dctx = zstd.ZstdDecompressor()
with dctx.stream_reader(frame) as reader:
- with self.assertRaisesRegex(ValueError, "cannot seek to negative position"):
+ with self.assertRaisesRegex(
+ ValueError, "cannot seek to negative position"
+ ):
reader.seek(-1, os.SEEK_SET)
reader.read(1)
@@ -490,7 +501,8 @@
reader.seek(-1, os.SEEK_CUR)
with self.assertRaisesRegex(
- ValueError, "zstd decompression streams cannot be seeked with SEEK_END"
+ ValueError,
+ "zstd decompression streams cannot be seeked with SEEK_END",
):
reader.seek(0, os.SEEK_END)
@@ -743,7 +755,9 @@
def test_read_lines(self):
cctx = zstd.ZstdCompressor()
- source = b"\n".join(("line %d" % i).encode("ascii") for i in range(1024))
+ source = b"\n".join(
+ ("line %d" % i).encode("ascii") for i in range(1024)
+ )
frame = cctx.compress(source)
@@ -821,7 +835,9 @@
dobj = dctx.decompressobj()
dobj.decompress(data)
- with self.assertRaisesRegex(zstd.ZstdError, "cannot use a decompressobj"):
+ with self.assertRaisesRegex(
+ zstd.ZstdError, "cannot use a decompressobj"
+ ):
dobj.decompress(data)
self.assertIsNone(dobj.flush())
@@ -1124,7 +1140,9 @@
# Buffer protocol works.
dctx.read_to_iter(b"foobar")
- with self.assertRaisesRegex(ValueError, "must pass an object with a read"):
+ with self.assertRaisesRegex(
+ ValueError, "must pass an object with a read"
+ ):
b"".join(dctx.read_to_iter(True))
def test_empty_input(self):
@@ -1226,7 +1244,9 @@
decompressed = b"".join(chunks)
self.assertEqual(decompressed, source.getvalue())
- @unittest.skipUnless("ZSTD_SLOW_TESTS" in os.environ, "ZSTD_SLOW_TESTS not set")
+ @unittest.skipUnless(
+ "ZSTD_SLOW_TESTS" in os.environ, "ZSTD_SLOW_TESTS not set"
+ )
def test_large_input(self):
bytes = list(struct.Struct(">B").pack(i) for i in range(256))
compressed = NonClosingBytesIO()
@@ -1241,13 +1261,16 @@
len(compressed.getvalue())
> zstd.DECOMPRESSION_RECOMMENDED_INPUT_SIZE
)
- have_raw = input_size > zstd.DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE * 2
+ have_raw = (
+ input_size > zstd.DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE * 2
+ )
if have_compressed and have_raw:
break
compressed = io.BytesIO(compressed.getvalue())
self.assertGreater(
- len(compressed.getvalue()), zstd.DECOMPRESSION_RECOMMENDED_INPUT_SIZE
+ len(compressed.getvalue()),
+ zstd.DECOMPRESSION_RECOMMENDED_INPUT_SIZE,
)
dctx = zstd.ZstdDecompressor()
@@ -1303,7 +1326,9 @@
self.assertEqual(streamed, source.getvalue())
def test_read_write_size(self):
- source = OpCountingBytesIO(zstd.ZstdCompressor().compress(b"foobarfoobar"))
+ source = OpCountingBytesIO(
+ zstd.ZstdCompressor().compress(b"foobarfoobar")
+ )
dctx = zstd.ZstdDecompressor()
for chunk in dctx.read_to_iter(source, read_size=1, write_size=1):
self.assertEqual(len(chunk), 1)
@@ -1355,10 +1380,14 @@
):
dctx.decompress_content_dict_chain([zstd.FRAME_HEADER])
- with self.assertRaisesRegex(ValueError, "chunk 0 is not a valid zstd frame"):
+ with self.assertRaisesRegex(
+ ValueError, "chunk 0 is not a valid zstd frame"
+ ):
dctx.decompress_content_dict_chain([b"foo" * 8])
- no_size = zstd.ZstdCompressor(write_content_size=False).compress(b"foo" * 64)
+ no_size = zstd.ZstdCompressor(write_content_size=False).compress(
+ b"foo" * 64
+ )
with self.assertRaisesRegex(
ValueError, "chunk 0 missing content size in frame"
@@ -1389,10 +1418,14 @@
):
dctx.decompress_content_dict_chain([initial, zstd.FRAME_HEADER])
- with self.assertRaisesRegex(ValueError, "chunk 1 is not a valid zstd frame"):
+ with self.assertRaisesRegex(
+ ValueError, "chunk 1 is not a valid zstd frame"
+ ):
dctx.decompress_content_dict_chain([initial, b"foo" * 8])
- no_size = zstd.ZstdCompressor(write_content_size=False).compress(b"foo" * 64)
+ no_size = zstd.ZstdCompressor(write_content_size=False).compress(
+ b"foo" * 64
+ )
with self.assertRaisesRegex(
ValueError, "chunk 1 missing content size in frame"
@@ -1400,7 +1433,9 @@
dctx.decompress_content_dict_chain([initial, no_size])
# Corrupt second frame.
- cctx = zstd.ZstdCompressor(dict_data=zstd.ZstdCompressionDict(b"foo" * 64))
+ cctx = zstd.ZstdCompressor(
+ dict_data=zstd.ZstdCompressionDict(b"foo" * 64)
+ )
frame = cctx.compress(b"bar" * 64)
frame = frame[0:12] + frame[15:]
@@ -1447,7 +1482,9 @@
with self.assertRaises(TypeError):
dctx.multi_decompress_to_buffer((1, 2))
- with self.assertRaisesRegex(TypeError, "item 0 not a bytes like object"):
+ with self.assertRaisesRegex(
+ TypeError, "item 0 not a bytes like object"
+ ):
dctx.multi_decompress_to_buffer([u"foo"])
with self.assertRaisesRegex(
@@ -1491,7 +1528,9 @@
if not hasattr(dctx, "multi_decompress_to_buffer"):
self.skipTest("multi_decompress_to_buffer not available")
- result = dctx.multi_decompress_to_buffer(frames, decompressed_sizes=sizes)
+ result = dctx.multi_decompress_to_buffer(
+ frames, decompressed_sizes=sizes
+ )
self.assertEqual(len(result), len(frames))
self.assertEqual(result.size(), sum(map(len, original)))
@@ -1582,10 +1621,15 @@
# And a manual mode.
b = b"".join([frames[0].tobytes(), frames[1].tobytes()])
b1 = zstd.BufferWithSegments(
- b, struct.pack("=QQQQ", 0, len(frames[0]), len(frames[0]), len(frames[1]))
+ b,
+ struct.pack(
+ "=QQQQ", 0, len(frames[0]), len(frames[0]), len(frames[1])
+ ),
)
- b = b"".join([frames[2].tobytes(), frames[3].tobytes(), frames[4].tobytes()])
+ b = b"".join(
+ [frames[2].tobytes(), frames[3].tobytes(), frames[4].tobytes()]
+ )
b2 = zstd.BufferWithSegments(
b,
struct.pack(
--- a/contrib/python-zstandard/tests/test_decompressor_fuzzing.py Tue Jan 21 17:15:34 2020 -0800
+++ b/contrib/python-zstandard/tests/test_decompressor_fuzzing.py Thu Jan 23 13:10:48 2020 -0800
@@ -196,7 +196,9 @@
streaming=strategies.booleans(),
source_read_size=strategies.integers(1, 1048576),
)
- def test_stream_source_readall(self, original, level, streaming, source_read_size):
+ def test_stream_source_readall(
+ self, original, level, streaming, source_read_size
+ ):
cctx = zstd.ZstdCompressor(level=level)
if streaming:
@@ -398,7 +400,9 @@
write_size=strategies.integers(min_value=1, max_value=8192),
input_sizes=strategies.data(),
)
- def test_write_size_variance(self, original, level, write_size, input_sizes):
+ def test_write_size_variance(
+ self, original, level, write_size, input_sizes
+ ):
cctx = zstd.ZstdCompressor(level=level)
frame = cctx.compress(original)
@@ -433,7 +437,9 @@
read_size=strategies.integers(min_value=1, max_value=8192),
write_size=strategies.integers(min_value=1, max_value=8192),
)
- def test_read_write_size_variance(self, original, level, read_size, write_size):
+ def test_read_write_size_variance(
+ self, original, level, read_size, write_size
+ ):
cctx = zstd.ZstdCompressor(level=level)
frame = cctx.compress(original)
@@ -441,7 +447,9 @@
dest = io.BytesIO()
dctx = zstd.ZstdDecompressor()
- dctx.copy_stream(source, dest, read_size=read_size, write_size=write_size)
+ dctx.copy_stream(
+ source, dest, read_size=read_size, write_size=write_size
+ )
self.assertEqual(dest.getvalue(), original)
@@ -490,11 +498,14 @@
original=strategies.sampled_from(random_input_data()),
level=strategies.integers(min_value=1, max_value=5),
write_size=strategies.integers(
- min_value=1, max_value=4 * zstd.DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE
+ min_value=1,
+ max_value=4 * zstd.DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE,
),
chunk_sizes=strategies.data(),
)
- def test_random_output_sizes(self, original, level, write_size, chunk_sizes):
+ def test_random_output_sizes(
+ self, original, level, write_size, chunk_sizes
+ ):
cctx = zstd.ZstdCompressor(level=level)
frame = cctx.compress(original)
@@ -524,7 +535,9 @@
read_size=strategies.integers(min_value=1, max_value=4096),
write_size=strategies.integers(min_value=1, max_value=4096),
)
- def test_read_write_size_variance(self, original, level, read_size, write_size):
+ def test_read_write_size_variance(
+ self, original, level, read_size, write_size
+ ):
cctx = zstd.ZstdCompressor(level=level)
frame = cctx.compress(original)
@@ -532,7 +545,9 @@
dctx = zstd.ZstdDecompressor()
chunks = list(
- dctx.read_to_iter(source, read_size=read_size, write_size=write_size)
+ dctx.read_to_iter(
+ source, read_size=read_size, write_size=write_size
+ )
)
self.assertEqual(b"".join(chunks), original)
@@ -542,7 +557,9 @@
class TestDecompressor_multi_decompress_to_buffer_fuzzing(TestCase):
@hypothesis.given(
original=strategies.lists(
- strategies.sampled_from(random_input_data()), min_size=1, max_size=1024
+ strategies.sampled_from(random_input_data()),
+ min_size=1,
+ max_size=1024,
),
threads=strategies.integers(min_value=1, max_value=8),
use_dict=strategies.booleans(),
--- a/contrib/python-zstandard/tests/test_train_dictionary.py Tue Jan 21 17:15:34 2020 -0800
+++ b/contrib/python-zstandard/tests/test_train_dictionary.py Thu Jan 23 13:10:48 2020 -0800
@@ -51,11 +51,15 @@
self.assertEqual(d.d, 16)
def test_set_dict_id(self):
- d = zstd.train_dictionary(8192, generate_samples(), k=64, d=16, dict_id=42)
+ d = zstd.train_dictionary(
+ 8192, generate_samples(), k=64, d=16, dict_id=42
+ )
self.assertEqual(d.dict_id(), 42)
def test_optimize(self):
- d = zstd.train_dictionary(8192, generate_samples(), threads=-1, steps=1, d=16)
+ d = zstd.train_dictionary(
+ 8192, generate_samples(), threads=-1, steps=1, d=16
+ )
# This varies by platform.
self.assertIn(d.k, (50, 2000))
@@ -71,10 +75,14 @@
def test_bad_precompute_compress(self):
d = zstd.train_dictionary(8192, generate_samples(), k=64, d=16)
- with self.assertRaisesRegex(ValueError, "must specify one of level or "):
+ with self.assertRaisesRegex(
+ ValueError, "must specify one of level or "
+ ):
d.precompute_compress()
- with self.assertRaisesRegex(ValueError, "must only specify one of level or "):
+ with self.assertRaisesRegex(
+ ValueError, "must only specify one of level or "
+ ):
d.precompute_compress(
level=3, compression_params=zstd.CompressionParameters()
)
@@ -88,5 +96,7 @@
d = zstd.ZstdCompressionDict(
b"dictcontent" * 64, dict_type=zstd.DICT_TYPE_FULLDICT
)
- with self.assertRaisesRegex(zstd.ZstdError, "unable to precompute dictionary"):
+ with self.assertRaisesRegex(
+ zstd.ZstdError, "unable to precompute dictionary"
+ ):
d.precompute_compress(level=1)
--- a/contrib/python-zstandard/zstandard/cffi.py Tue Jan 21 17:15:34 2020 -0800
+++ b/contrib/python-zstandard/zstandard/cffi.py Thu Jan 23 13:10:48 2020 -0800
@@ -299,10 +299,14 @@
_set_compression_parameter(params, lib.ZSTD_c_chainLog, chain_log)
_set_compression_parameter(params, lib.ZSTD_c_searchLog, search_log)
_set_compression_parameter(params, lib.ZSTD_c_minMatch, min_match)
- _set_compression_parameter(params, lib.ZSTD_c_targetLength, target_length)
+ _set_compression_parameter(
+ params, lib.ZSTD_c_targetLength, target_length
+ )
if strategy != -1 and compression_strategy != -1:
- raise ValueError("cannot specify both compression_strategy and strategy")
+ raise ValueError(
+ "cannot specify both compression_strategy and strategy"
+ )
if compression_strategy != -1:
strategy = compression_strategy
@@ -313,12 +317,16 @@
_set_compression_parameter(
params, lib.ZSTD_c_contentSizeFlag, write_content_size
)
- _set_compression_parameter(params, lib.ZSTD_c_checksumFlag, write_checksum)
+ _set_compression_parameter(
+ params, lib.ZSTD_c_checksumFlag, write_checksum
+ )
_set_compression_parameter(params, lib.ZSTD_c_dictIDFlag, write_dict_id)
_set_compression_parameter(params, lib.ZSTD_c_jobSize, job_size)
if overlap_log != -1 and overlap_size_log != -1:
- raise ValueError("cannot specify both overlap_log and overlap_size_log")
+ raise ValueError(
+ "cannot specify both overlap_log and overlap_size_log"
+ )
if overlap_size_log != -1:
overlap_log = overlap_size_log
@@ -326,12 +334,16 @@
overlap_log = 0
_set_compression_parameter(params, lib.ZSTD_c_overlapLog, overlap_log)
- _set_compression_parameter(params, lib.ZSTD_c_forceMaxWindow, force_max_window)
+ _set_compression_parameter(
+ params, lib.ZSTD_c_forceMaxWindow, force_max_window
+ )
_set_compression_parameter(
params, lib.ZSTD_c_enableLongDistanceMatching, enable_ldm
)
_set_compression_parameter(params, lib.ZSTD_c_ldmHashLog, ldm_hash_log)
- _set_compression_parameter(params, lib.ZSTD_c_ldmMinMatch, ldm_min_match)
+ _set_compression_parameter(
+ params, lib.ZSTD_c_ldmMinMatch, ldm_min_match
+ )
_set_compression_parameter(
params, lib.ZSTD_c_ldmBucketSizeLog, ldm_bucket_size_log
)
@@ -346,7 +358,9 @@
elif ldm_hash_rate_log == -1:
ldm_hash_rate_log = 0
- _set_compression_parameter(params, lib.ZSTD_c_ldmHashRateLog, ldm_hash_rate_log)
+ _set_compression_parameter(
+ params, lib.ZSTD_c_ldmHashRateLog, ldm_hash_rate_log
+ )
@property
def format(self):
@@ -354,7 +368,9 @@
@property
def compression_level(self):
- return _get_compression_parameter(self._params, lib.ZSTD_c_compressionLevel)
+ return _get_compression_parameter(
+ self._params, lib.ZSTD_c_compressionLevel
+ )
@property
def window_log(self):
@@ -386,7 +402,9 @@
@property
def write_content_size(self):
- return _get_compression_parameter(self._params, lib.ZSTD_c_contentSizeFlag)
+ return _get_compression_parameter(
+ self._params, lib.ZSTD_c_contentSizeFlag
+ )
@property
def write_checksum(self):
@@ -410,7 +428,9 @@
@property
def force_max_window(self):
- return _get_compression_parameter(self._params, lib.ZSTD_c_forceMaxWindow)
+ return _get_compression_parameter(
+ self._params, lib.ZSTD_c_forceMaxWindow
+ )
@property
def enable_ldm(self):
@@ -428,11 +448,15 @@
@property
def ldm_bucket_size_log(self):
- return _get_compression_parameter(self._params, lib.ZSTD_c_ldmBucketSizeLog)
+ return _get_compression_parameter(
+ self._params, lib.ZSTD_c_ldmBucketSizeLog
+ )
@property
def ldm_hash_rate_log(self):
- return _get_compression_parameter(self._params, lib.ZSTD_c_ldmHashRateLog)
+ return _get_compression_parameter(
+ self._params, lib.ZSTD_c_ldmHashRateLog
+ )
@property
def ldm_hash_every_log(self):
@@ -457,7 +481,8 @@
zresult = lib.ZSTD_CCtxParams_setParameter(params, param, value)
if lib.ZSTD_isError(zresult):
raise ZstdError(
- "unable to set compression context parameter: %s" % _zstd_error(zresult)
+ "unable to set compression context parameter: %s"
+ % _zstd_error(zresult)
)
@@ -467,14 +492,17 @@
zresult = lib.ZSTD_CCtxParams_getParameter(params, param, result)
if lib.ZSTD_isError(zresult):
raise ZstdError(
- "unable to get compression context parameter: %s" % _zstd_error(zresult)
+ "unable to get compression context parameter: %s"
+ % _zstd_error(zresult)
)
return result[0]
class ZstdCompressionWriter(object):
- def __init__(self, compressor, writer, source_size, write_size, write_return_read):
+ def __init__(
+ self, compressor, writer, source_size, write_size, write_return_read
+ ):
self._compressor = compressor
self._writer = writer
self._write_size = write_size
@@ -491,7 +519,9 @@
zresult = lib.ZSTD_CCtx_setPledgedSrcSize(compressor._cctx, source_size)
if lib.ZSTD_isError(zresult):
- raise ZstdError("error setting source size: %s" % _zstd_error(zresult))
+ raise ZstdError(
+ "error setting source size: %s" % _zstd_error(zresult)
+ )
def __enter__(self):
if self._closed:
@@ -595,13 +625,20 @@
while in_buffer.pos < in_buffer.size:
zresult = lib.ZSTD_compressStream2(
- self._compressor._cctx, out_buffer, in_buffer, lib.ZSTD_e_continue
+ self._compressor._cctx,
+ out_buffer,
+ in_buffer,
+ lib.ZSTD_e_continue,
)
if lib.ZSTD_isError(zresult):
- raise ZstdError("zstd compress error: %s" % _zstd_error(zresult))
+ raise ZstdError(
+ "zstd compress error: %s" % _zstd_error(zresult)
+ )
if out_buffer.pos:
- self._writer.write(ffi.buffer(out_buffer.dst, out_buffer.pos)[:])
+ self._writer.write(
+ ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
+ )
total_write += out_buffer.pos
self._bytes_compressed += out_buffer.pos
out_buffer.pos = 0
@@ -637,10 +674,14 @@
self._compressor._cctx, out_buffer, in_buffer, flush
)
if lib.ZSTD_isError(zresult):
- raise ZstdError("zstd compress error: %s" % _zstd_error(zresult))
+ raise ZstdError(
+ "zstd compress error: %s" % _zstd_error(zresult)
+ )
if out_buffer.pos:
- self._writer.write(ffi.buffer(out_buffer.dst, out_buffer.pos)[:])
+ self._writer.write(
+ ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
+ )
total_write += out_buffer.pos
self._bytes_compressed += out_buffer.pos
out_buffer.pos = 0
@@ -672,7 +713,9 @@
self._compressor._cctx, self._out, source, lib.ZSTD_e_continue
)
if lib.ZSTD_isError(zresult):
- raise ZstdError("zstd compress error: %s" % _zstd_error(zresult))
+ raise ZstdError(
+ "zstd compress error: %s" % _zstd_error(zresult)
+ )
if self._out.pos:
chunks.append(ffi.buffer(self._out.dst, self._out.pos)[:])
@@ -681,7 +724,10 @@
return b"".join(chunks)
def flush(self, flush_mode=COMPRESSOBJ_FLUSH_FINISH):
- if flush_mode not in (COMPRESSOBJ_FLUSH_FINISH, COMPRESSOBJ_FLUSH_BLOCK):
+ if flush_mode not in (
+ COMPRESSOBJ_FLUSH_FINISH,
+ COMPRESSOBJ_FLUSH_BLOCK,
+ ):
raise ValueError("flush mode not recognized")
if self._finished:
@@ -768,7 +814,9 @@
self._in.pos = 0
if lib.ZSTD_isError(zresult):
- raise ZstdError("zstd compress error: %s" % _zstd_error(zresult))
+ raise ZstdError(
+ "zstd compress error: %s" % _zstd_error(zresult)
+ )
if self._out.pos == self._out.size:
yield ffi.buffer(self._out.dst, self._out.pos)[:]
@@ -780,7 +828,8 @@
if self._in.src != ffi.NULL:
raise ZstdError(
- "cannot call flush() before consuming output from " "previous operation"
+ "cannot call flush() before consuming output from "
+ "previous operation"
)
while True:
@@ -788,7 +837,9 @@
self._compressor._cctx, self._out, self._in, lib.ZSTD_e_flush
)
if lib.ZSTD_isError(zresult):
- raise ZstdError("zstd compress error: %s" % _zstd_error(zresult))
+ raise ZstdError(
+ "zstd compress error: %s" % _zstd_error(zresult)
+ )
if self._out.pos:
yield ffi.buffer(self._out.dst, self._out.pos)[:]
@@ -812,7 +863,9 @@
self._compressor._cctx, self._out, self._in, lib.ZSTD_e_end
)
if lib.ZSTD_isError(zresult):
- raise ZstdError("zstd compress error: %s" % _zstd_error(zresult))
+ raise ZstdError(
+ "zstd compress error: %s" % _zstd_error(zresult)
+ )
if self._out.pos:
yield ffi.buffer(self._out.dst, self._out.pos)[:]
@@ -939,7 +992,10 @@
old_pos = out_buffer.pos
zresult = lib.ZSTD_compressStream2(
- self._compressor._cctx, out_buffer, self._in_buffer, lib.ZSTD_e_continue
+ self._compressor._cctx,
+ out_buffer,
+ self._in_buffer,
+ lib.ZSTD_e_continue,
)
self._bytes_compressed += out_buffer.pos - old_pos
@@ -997,7 +1053,9 @@
self._bytes_compressed += out_buffer.pos - old_pos
if lib.ZSTD_isError(zresult):
- raise ZstdError("error ending compression stream: %s", _zstd_error(zresult))
+ raise ZstdError(
+ "error ending compression stream: %s", _zstd_error(zresult)
+ )
if zresult == 0:
self._finished_output = True
@@ -1102,7 +1160,9 @@
self._bytes_compressed += out_buffer.pos - old_pos
if lib.ZSTD_isError(zresult):
- raise ZstdError("error ending compression stream: %s", _zstd_error(zresult))
+ raise ZstdError(
+ "error ending compression stream: %s", _zstd_error(zresult)
+ )
if zresult == 0:
self._finished_output = True
@@ -1170,13 +1230,17 @@
threads=0,
):
if level > lib.ZSTD_maxCLevel():
- raise ValueError("level must be less than %d" % lib.ZSTD_maxCLevel())
+ raise ValueError(
+ "level must be less than %d" % lib.ZSTD_maxCLevel()
+ )
if threads < 0:
threads = _cpu_count()
if compression_params and write_checksum is not None:
- raise ValueError("cannot define compression_params and " "write_checksum")
+ raise ValueError(
+ "cannot define compression_params and " "write_checksum"
+ )
if compression_params and write_content_size is not None:
raise ValueError(
@@ -1184,7 +1248,9 @@
)
if compression_params and write_dict_id is not None:
- raise ValueError("cannot define compression_params and " "write_dict_id")
+ raise ValueError(
+ "cannot define compression_params and " "write_dict_id"
+ )
if compression_params and threads:
raise ValueError("cannot define compression_params and threads")
@@ -1201,7 +1267,9 @@
self._params = ffi.gc(params, lib.ZSTD_freeCCtxParams)
- _set_compression_parameter(self._params, lib.ZSTD_c_compressionLevel, level)
+ _set_compression_parameter(
+ self._params, lib.ZSTD_c_compressionLevel, level
+ )
_set_compression_parameter(
self._params,
@@ -1210,7 +1278,9 @@
)
_set_compression_parameter(
- self._params, lib.ZSTD_c_checksumFlag, 1 if write_checksum else 0
+ self._params,
+ lib.ZSTD_c_checksumFlag,
+ 1 if write_checksum else 0,
)
_set_compression_parameter(
@@ -1218,7 +1288,9 @@
)
if threads:
- _set_compression_parameter(self._params, lib.ZSTD_c_nbWorkers, threads)
+ _set_compression_parameter(
+ self._params, lib.ZSTD_c_nbWorkers, threads
+ )
cctx = lib.ZSTD_createCCtx()
if cctx == ffi.NULL:
@@ -1237,10 +1309,13 @@
)
def _setup_cctx(self):
- zresult = lib.ZSTD_CCtx_setParametersUsingCCtxParams(self._cctx, self._params)
+ zresult = lib.ZSTD_CCtx_setParametersUsingCCtxParams(
+ self._cctx, self._params
+ )
if lib.ZSTD_isError(zresult):
raise ZstdError(
- "could not set compression parameters: %s" % _zstd_error(zresult)
+ "could not set compression parameters: %s"
+ % _zstd_error(zresult)
)
dict_data = self._dict_data
@@ -1259,7 +1334,8 @@
if lib.ZSTD_isError(zresult):
raise ZstdError(
- "could not load compression dictionary: %s" % _zstd_error(zresult)
+ "could not load compression dictionary: %s"
+ % _zstd_error(zresult)
)
def memory_size(self):
@@ -1275,7 +1351,9 @@
zresult = lib.ZSTD_CCtx_setPledgedSrcSize(self._cctx, len(data_buffer))
if lib.ZSTD_isError(zresult):
- raise ZstdError("error setting source size: %s" % _zstd_error(zresult))
+ raise ZstdError(
+ "error setting source size: %s" % _zstd_error(zresult)
+ )
out_buffer = ffi.new("ZSTD_outBuffer *")
in_buffer = ffi.new("ZSTD_inBuffer *")
@@ -1307,11 +1385,15 @@
zresult = lib.ZSTD_CCtx_setPledgedSrcSize(self._cctx, size)
if lib.ZSTD_isError(zresult):
- raise ZstdError("error setting source size: %s" % _zstd_error(zresult))
+ raise ZstdError(
+ "error setting source size: %s" % _zstd_error(zresult)
+ )
cobj = ZstdCompressionObj()
cobj._out = ffi.new("ZSTD_outBuffer *")
- cobj._dst_buffer = ffi.new("char[]", COMPRESSION_RECOMMENDED_OUTPUT_SIZE)
+ cobj._dst_buffer = ffi.new(
+ "char[]", COMPRESSION_RECOMMENDED_OUTPUT_SIZE
+ )
cobj._out.dst = cobj._dst_buffer
cobj._out.size = COMPRESSION_RECOMMENDED_OUTPUT_SIZE
cobj._out.pos = 0
@@ -1328,7 +1410,9 @@
zresult = lib.ZSTD_CCtx_setPledgedSrcSize(self._cctx, size)
if lib.ZSTD_isError(zresult):
- raise ZstdError("error setting source size: %s" % _zstd_error(zresult))
+ raise ZstdError(
+ "error setting source size: %s" % _zstd_error(zresult)
+ )
return ZstdCompressionChunker(self, chunk_size=chunk_size)
@@ -1353,7 +1437,9 @@
zresult = lib.ZSTD_CCtx_setPledgedSrcSize(self._cctx, size)
if lib.ZSTD_isError(zresult):
- raise ZstdError("error setting source size: %s" % _zstd_error(zresult))
+ raise ZstdError(
+ "error setting source size: %s" % _zstd_error(zresult)
+ )
in_buffer = ffi.new("ZSTD_inBuffer *")
out_buffer = ffi.new("ZSTD_outBuffer *")
@@ -1381,7 +1467,9 @@
self._cctx, out_buffer, in_buffer, lib.ZSTD_e_continue
)
if lib.ZSTD_isError(zresult):
- raise ZstdError("zstd compress error: %s" % _zstd_error(zresult))
+ raise ZstdError(
+ "zstd compress error: %s" % _zstd_error(zresult)
+ )
if out_buffer.pos:
ofh.write(ffi.buffer(out_buffer.dst, out_buffer.pos))
@@ -1423,7 +1511,9 @@
zresult = lib.ZSTD_CCtx_setPledgedSrcSize(self._cctx, size)
if lib.ZSTD_isError(zresult):
- raise ZstdError("error setting source size: %s" % _zstd_error(zresult))
+ raise ZstdError(
+ "error setting source size: %s" % _zstd_error(zresult)
+ )
return ZstdCompressionReader(self, source, read_size)
@@ -1443,7 +1533,9 @@
if size < 0:
size = lib.ZSTD_CONTENTSIZE_UNKNOWN
- return ZstdCompressionWriter(self, writer, size, write_size, write_return_read)
+ return ZstdCompressionWriter(
+ self, writer, size, write_size, write_return_read
+ )
write_to = stream_writer
@@ -1473,7 +1565,9 @@
zresult = lib.ZSTD_CCtx_setPledgedSrcSize(self._cctx, size)
if lib.ZSTD_isError(zresult):
- raise ZstdError("error setting source size: %s" % _zstd_error(zresult))
+ raise ZstdError(
+ "error setting source size: %s" % _zstd_error(zresult)
+ )
in_buffer = ffi.new("ZSTD_inBuffer *")
out_buffer = ffi.new("ZSTD_outBuffer *")
@@ -1517,7 +1611,9 @@
self._cctx, out_buffer, in_buffer, lib.ZSTD_e_continue
)
if lib.ZSTD_isError(zresult):
- raise ZstdError("zstd compress error: %s" % _zstd_error(zresult))
+ raise ZstdError(
+ "zstd compress error: %s" % _zstd_error(zresult)
+ )
if out_buffer.pos:
data = ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
@@ -1596,10 +1692,14 @@
data_buffer = ffi.from_buffer(data)
zresult = lib.ZSTD_getFrameHeader(params, data_buffer, len(data_buffer))
if lib.ZSTD_isError(zresult):
- raise ZstdError("cannot get frame parameters: %s" % _zstd_error(zresult))
+ raise ZstdError(
+ "cannot get frame parameters: %s" % _zstd_error(zresult)
+ )
if zresult:
- raise ZstdError("not enough data for frame parameters; need %d bytes" % zresult)
+ raise ZstdError(
+ "not enough data for frame parameters; need %d bytes" % zresult
+ )
return FrameParameters(params[0])
@@ -1611,9 +1711,14 @@
self.k = k
self.d = d
- if dict_type not in (DICT_TYPE_AUTO, DICT_TYPE_RAWCONTENT, DICT_TYPE_FULLDICT):
+ if dict_type not in (
+ DICT_TYPE_AUTO,
+ DICT_TYPE_RAWCONTENT,
+ DICT_TYPE_FULLDICT,
+ ):
raise ValueError(
- "invalid dictionary load mode: %d; must use " "DICT_TYPE_* constants"
+ "invalid dictionary load mode: %d; must use "
+ "DICT_TYPE_* constants"
)
self._dict_type = dict_type
@@ -1630,7 +1735,9 @@
def precompute_compress(self, level=0, compression_params=None):
if level and compression_params:
- raise ValueError("must only specify one of level or " "compression_params")
+ raise ValueError(
+ "must only specify one of level or " "compression_params"
+ )
if not level and not compression_params:
raise ValueError("must specify one of level or compression_params")
@@ -1675,7 +1782,9 @@
if ddict == ffi.NULL:
raise ZstdError("could not create decompression dict")
- ddict = ffi.gc(ddict, lib.ZSTD_freeDDict, size=lib.ZSTD_sizeof_DDict(ddict))
+ ddict = ffi.gc(
+ ddict, lib.ZSTD_freeDDict, size=lib.ZSTD_sizeof_DDict(ddict)
+ )
self.__dict__["_ddict"] = ddict
return ddict
@@ -1805,7 +1914,9 @@
self._decompressor._dctx, out_buffer, in_buffer
)
if lib.ZSTD_isError(zresult):
- raise ZstdError("zstd decompressor error: %s" % _zstd_error(zresult))
+ raise ZstdError(
+ "zstd decompressor error: %s" % _zstd_error(zresult)
+ )
if zresult == 0:
self._finished = True
@@ -2105,16 +2216,22 @@
if whence == os.SEEK_SET:
if pos < 0:
- raise ValueError("cannot seek to negative position with SEEK_SET")
+ raise ValueError(
+ "cannot seek to negative position with SEEK_SET"
+ )
if pos < self._bytes_decompressed:
- raise ValueError("cannot seek zstd decompression stream " "backwards")
+ raise ValueError(
+ "cannot seek zstd decompression stream " "backwards"
+ )
read_amount = pos - self._bytes_decompressed
elif whence == os.SEEK_CUR:
if pos < 0:
- raise ValueError("cannot seek zstd decompression stream " "backwards")
+ raise ValueError(
+ "cannot seek zstd decompression stream " "backwards"
+ )
read_amount = pos
elif whence == os.SEEK_END:
@@ -2123,7 +2240,9 @@
)
while read_amount:
- result = self.read(min(read_amount, DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE))
+ result = self.read(
+ min(read_amount, DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE)
+ )
if not result:
break
@@ -2257,10 +2376,14 @@
while in_buffer.pos < in_buffer.size:
zresult = lib.ZSTD_decompressStream(dctx, out_buffer, in_buffer)
if lib.ZSTD_isError(zresult):
- raise ZstdError("zstd decompress error: %s" % _zstd_error(zresult))
+ raise ZstdError(
+ "zstd decompress error: %s" % _zstd_error(zresult)
+ )
if out_buffer.pos:
- self._writer.write(ffi.buffer(out_buffer.dst, out_buffer.pos)[:])
+ self._writer.write(
+ ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
+ )
total_write += out_buffer.pos
out_buffer.pos = 0
@@ -2299,7 +2422,9 @@
data_buffer = ffi.from_buffer(data)
- output_size = lib.ZSTD_getFrameContentSize(data_buffer, len(data_buffer))
+ output_size = lib.ZSTD_getFrameContentSize(
+ data_buffer, len(data_buffer)
+ )
if output_size == lib.ZSTD_CONTENTSIZE_ERROR:
raise ZstdError("error determining content size from frame header")
@@ -2307,7 +2432,9 @@
return b""
elif output_size == lib.ZSTD_CONTENTSIZE_UNKNOWN:
if not max_output_size:
- raise ZstdError("could not determine content size in frame header")
+ raise ZstdError(
+ "could not determine content size in frame header"
+ )
result_buffer = ffi.new("char[]", max_output_size)
result_size = max_output_size
@@ -2330,7 +2457,9 @@
if lib.ZSTD_isError(zresult):
raise ZstdError("decompression error: %s" % _zstd_error(zresult))
elif zresult:
- raise ZstdError("decompression error: did not decompress full frame")
+ raise ZstdError(
+ "decompression error: did not decompress full frame"
+ )
elif output_size and out_buffer.pos != output_size:
raise ZstdError(
"decompression error: decompressed %d bytes; expected %d"
@@ -2346,7 +2475,9 @@
read_across_frames=False,
):
self._ensure_dctx()
- return ZstdDecompressionReader(self, source, read_size, read_across_frames)
+ return ZstdDecompressionReader(
+ self, source, read_size, read_across_frames
+ )
def decompressobj(self, write_size=DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE):
if write_size < 1:
@@ -2421,9 +2552,13 @@
while in_buffer.pos < in_buffer.size:
assert out_buffer.pos == 0
- zresult = lib.ZSTD_decompressStream(self._dctx, out_buffer, in_buffer)
+ zresult = lib.ZSTD_decompressStream(
+ self._dctx, out_buffer, in_buffer
+ )
if lib.ZSTD_isError(zresult):
- raise ZstdError("zstd decompress error: %s" % _zstd_error(zresult))
+ raise ZstdError(
+ "zstd decompress error: %s" % _zstd_error(zresult)
+ )
if out_buffer.pos:
data = ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
@@ -2449,7 +2584,9 @@
if not hasattr(writer, "write"):
raise ValueError("must pass an object with a write() method")
- return ZstdDecompressionWriter(self, writer, write_size, write_return_read)
+ return ZstdDecompressionWriter(
+ self, writer, write_size, write_return_read
+ )
write_to = stream_writer
@@ -2491,7 +2628,9 @@
# Flush all read data to output.
while in_buffer.pos < in_buffer.size:
- zresult = lib.ZSTD_decompressStream(self._dctx, out_buffer, in_buffer)
+ zresult = lib.ZSTD_decompressStream(
+ self._dctx, out_buffer, in_buffer
+ )
if lib.ZSTD_isError(zresult):
raise ZstdError(
"zstd decompressor error: %s" % _zstd_error(zresult)
@@ -2521,7 +2660,9 @@
# All chunks should be zstd frames and should have content size set.
chunk_buffer = ffi.from_buffer(chunk)
params = ffi.new("ZSTD_frameHeader *")
- zresult = lib.ZSTD_getFrameHeader(params, chunk_buffer, len(chunk_buffer))
+ zresult = lib.ZSTD_getFrameHeader(
+ params, chunk_buffer, len(chunk_buffer)
+ )
if lib.ZSTD_isError(zresult):
raise ValueError("chunk 0 is not a valid zstd frame")
elif zresult:
@@ -2546,7 +2687,9 @@
zresult = lib.ZSTD_decompressStream(self._dctx, out_buffer, in_buffer)
if lib.ZSTD_isError(zresult):
- raise ZstdError("could not decompress chunk 0: %s" % _zstd_error(zresult))
+ raise ZstdError(
+ "could not decompress chunk 0: %s" % _zstd_error(zresult)
+ )
elif zresult:
raise ZstdError("chunk 0 did not decompress full frame")
@@ -2561,11 +2704,15 @@
raise ValueError("chunk %d must be bytes" % i)
chunk_buffer = ffi.from_buffer(chunk)
- zresult = lib.ZSTD_getFrameHeader(params, chunk_buffer, len(chunk_buffer))
+ zresult = lib.ZSTD_getFrameHeader(
+ params, chunk_buffer, len(chunk_buffer)
+ )
if lib.ZSTD_isError(zresult):
raise ValueError("chunk %d is not a valid zstd frame" % i)
elif zresult:
- raise ValueError("chunk %d is too small to contain a zstd frame" % i)
+ raise ValueError(
+ "chunk %d is too small to contain a zstd frame" % i
+ )
if params.frameContentSize == lib.ZSTD_CONTENTSIZE_UNKNOWN:
raise ValueError("chunk %d missing content size in frame" % i)
@@ -2580,7 +2727,9 @@
in_buffer.size = len(chunk_buffer)
in_buffer.pos = 0
- zresult = lib.ZSTD_decompressStream(self._dctx, out_buffer, in_buffer)
+ zresult = lib.ZSTD_decompressStream(
+ self._dctx, out_buffer, in_buffer
+ )
if lib.ZSTD_isError(zresult):
raise ZstdError(
"could not decompress chunk %d: %s" % _zstd_error(zresult)
@@ -2597,7 +2746,9 @@
lib.ZSTD_DCtx_reset(self._dctx, lib.ZSTD_reset_session_only)
if self._max_window_size:
- zresult = lib.ZSTD_DCtx_setMaxWindowSize(self._dctx, self._max_window_size)
+ zresult = lib.ZSTD_DCtx_setMaxWindowSize(
+ self._dctx, self._max_window_size
+ )
if lib.ZSTD_isError(zresult):
raise ZstdError(
"unable to set max window size: %s" % _zstd_error(zresult)
@@ -2605,11 +2756,14 @@
zresult = lib.ZSTD_DCtx_setFormat(self._dctx, self._format)
if lib.ZSTD_isError(zresult):
- raise ZstdError("unable to set decoding format: %s" % _zstd_error(zresult))
+ raise ZstdError(
+ "unable to set decoding format: %s" % _zstd_error(zresult)
+ )
if self._dict_data and load_dict:
zresult = lib.ZSTD_DCtx_refDDict(self._dctx, self._dict_data._ddict)
if lib.ZSTD_isError(zresult):
raise ZstdError(
- "unable to reference prepared dictionary: %s" % _zstd_error(zresult)
+ "unable to reference prepared dictionary: %s"
+ % _zstd_error(zresult)
)
--- a/hgext/lfs/blobstore.py Tue Jan 21 17:15:34 2020 -0800
+++ b/hgext/lfs/blobstore.py Thu Jan 23 13:10:48 2020 -0800
@@ -94,15 +94,12 @@
pass
-class filewithprogress(object):
+class lfsuploadfile(object):
"""a file-like object that supports __len__ and read.
-
- Useful to provide progress information for how many bytes are read.
"""
- def __init__(self, fp, callback):
+ def __init__(self, fp):
self._fp = fp
- self._callback = callback # func(readsize)
fp.seek(0, os.SEEK_END)
self._len = fp.tell()
fp.seek(0)
@@ -113,14 +110,12 @@
def read(self, size):
if self._fp is None:
return b''
- data = self._fp.read(size)
- if data:
- if self._callback:
- self._callback(len(data))
- else:
+ return self._fp.read(size)
+
+ def close(self):
+ if self._fp is not None:
self._fp.close()
self._fp = None
- return data
class local(object):
@@ -495,15 +490,17 @@
_(b'detected corrupt lfs object: %s') % oid,
hint=_(b'run hg verify'),
)
- request.data = filewithprogress(localstore.open(oid), None)
- request.get_method = lambda: r'PUT'
- request.add_header('Content-Type', 'application/octet-stream')
- request.add_header('Content-Length', len(request.data))
for k, v in headers:
request.add_header(pycompat.strurl(k), pycompat.strurl(v))
try:
+ if action == b'upload':
+ request.data = lfsuploadfile(localstore.open(oid))
+ request.get_method = lambda: 'PUT'
+ request.add_header('Content-Type', 'application/octet-stream')
+ request.add_header('Content-Length', len(request.data))
+
with contextlib.closing(self.urlopener.open(request)) as res:
contentlength = res.info().get(b"content-length")
ui = self.ui # Shorten debug lines
@@ -545,6 +542,9 @@
raise LfsRemoteError(
_(b'LFS error: %s') % _urlerrorreason(ex), hint=hint
)
+ finally:
+ if request.data:
+ request.data.close()
def _batch(self, pointers, localstore, action):
if action not in [b'upload', b'download']:
--- a/mercurial/debugcommands.py Tue Jan 21 17:15:34 2020 -0800
+++ b/mercurial/debugcommands.py Thu Jan 23 13:10:48 2020 -0800
@@ -13,6 +13,7 @@
import errno
import operator
import os
+import platform
import random
import re
import socket
@@ -1487,6 +1488,11 @@
pycompat.sysexecutable or _(b"unknown"),
)
fm.write(
+ b'pythonimplementation',
+ _(b"checking Python implementation (%s)\n"),
+ pycompat.sysbytes(platform.python_implementation()),
+ )
+ fm.write(
b'pythonver',
_(b"checking Python version (%s)\n"),
(b"%d.%d.%d" % sys.version_info[:3]),
--- a/mercurial/match.py Tue Jan 21 17:15:34 2020 -0800
+++ b/mercurial/match.py Thu Jan 23 13:10:48 2020 -0800
@@ -24,7 +24,7 @@
)
from .utils import stringutil
-rustmod = policy.importrust('filepatterns')
+rustmod = policy.importrust('dirstate')
allpatternkinds = (
b're',
@@ -1273,15 +1273,6 @@
'''Convert a (normalized) pattern of any kind into a
regular expression.
globsuffix is appended to the regexp of globs.'''
-
- if rustmod is not None:
- try:
- return rustmod.build_single_regex(kind, pat, globsuffix)
- except rustmod.PatternError:
- raise error.ProgrammingError(
- b'not a regex pattern: %s:%s' % (kind, pat)
- )
-
if not pat and kind in (b'glob', b'relpath'):
return b''
if kind == b're':
@@ -1554,18 +1545,6 @@
This is useful to debug ignore patterns.
'''
- if rustmod is not None:
- result, warnings = rustmod.read_pattern_file(
- filepath, bool(warn), sourceinfo,
- )
-
- for warning_params in warnings:
- # Can't be easily emitted from Rust, because it would require
- # a mechanism for both gettext and calling the `warn` function.
- warn(_(b"%s: ignoring invalid syntax '%s'\n") % warning_params)
-
- return result
-
syntaxes = {
b're': b'relre:',
b'regexp': b'relre:',
--- a/rust/Cargo.lock Tue Jan 21 17:15:34 2020 -0800
+++ b/rust/Cargo.lock Thu Jan 23 13:10:48 2020 -0800
@@ -124,10 +124,16 @@
]
[[package]]
+name = "hex"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
name = "hg-core"
version = "0.1.0"
dependencies = [
"byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "hex 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -483,6 +489,7 @@
"checksum either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3"
"checksum fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba"
"checksum getrandom 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "473a1265acc8ff1e808cd0a1af8cee3c2ee5200916058a2ca113c29f2d903571"
+"checksum hex 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "023b39be39e3a2da62a94feb433e91e8bcd37676fbc8bea371daf52b7a769a3e"
"checksum lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
"checksum libc 0.2.64 (registry+https://github.com/rust-lang/crates.io-index)" = "74dfca3d9957906e8d1e6a0b641dc9a59848e793f1da2165889fd4f62d10d79c"
"checksum memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "88579771288728879b57485cc7d6b07d648c9f0141eb955f8ab7f9d45394468e"
--- a/rust/hg-core/Cargo.toml Tue Jan 21 17:15:34 2020 -0800
+++ b/rust/hg-core/Cargo.toml Thu Jan 23 13:10:48 2020 -0800
@@ -10,6 +10,7 @@
[dependencies]
byteorder = "1.3.1"
+hex = "0.4.0"
lazy_static = "1.3.0"
memchr = "2.2.0"
rand = "0.6.5"
--- a/rust/hg-core/src/revlog.rs Tue Jan 21 17:15:34 2020 -0800
+++ b/rust/hg-core/src/revlog.rs Thu Jan 23 13:10:48 2020 -0800
@@ -5,6 +5,10 @@
// GNU General Public License version 2 or any later version.
//! Mercurial concepts for handling revision history
+pub mod node;
+pub mod nodemap;
+pub use node::{Node, NodeError};
+
/// Mercurial revision numbers
///
/// As noted in revlog.c, revision numbers are actually encoded in
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/rust/hg-core/src/revlog/node.rs Thu Jan 23 13:10:48 2020 -0800
@@ -0,0 +1,191 @@
+// Copyright 2019-2020 Georges Racinet <georges.racinet@octobus.net>
+//
+// This software may be used and distributed according to the terms of the
+// GNU General Public License version 2 or any later version.
+
+//! Definitions and utilities for Revision nodes
+//!
+//! In Mercurial code base, it is customary to call "a node" the binary SHA
+//! of a revision.
+
+use hex::{self, FromHex, FromHexError};
+
+/// The length in bytes of a `Node`
+///
+/// This constant is meant to ease refactors of this module, and
+/// are private so that calling code does not expect all nodes have
+/// the same size, should we support several formats concurrently in
+/// the future.
+const NODE_BYTES_LENGTH: usize = 20;
+
+/// The length in bytes of a `Node`
+///
+/// see also `NODES_BYTES_LENGTH` about it being private.
+const NODE_NYBBLES_LENGTH: usize = 2 * NODE_BYTES_LENGTH;
+
+/// Private alias for readability and to ease future change
+type NodeData = [u8; NODE_BYTES_LENGTH];
+
+/// Binary revision SHA
+///
+/// ## Future changes of hash size
+///
+/// To accomodate future changes of hash size, Rust callers
+/// should use the conversion methods at the boundaries (FFI, actual
+/// computation of hashes and I/O) only, and only if required.
+///
+/// All other callers outside of unit tests should just handle `Node` values
+/// and never make any assumption on the actual length, using [`nybbles_len`]
+/// if they need a loop boundary.
+///
+/// All methods that create a `Node` either take a type that enforces
+/// the size or fail immediately at runtime with [`ExactLengthRequired`].
+///
+/// [`nybbles_len`]: #method.nybbles_len
+/// [`ExactLengthRequired`]: struct.NodeError#variant.ExactLengthRequired
+#[derive(Clone, Debug, PartialEq)]
+pub struct Node {
+ data: NodeData,
+}
+
+/// The node value for NULL_REVISION
+pub const NULL_NODE: Node = Node {
+ data: [0; NODE_BYTES_LENGTH],
+};
+
+impl From<NodeData> for Node {
+ fn from(data: NodeData) -> Node {
+ Node { data }
+ }
+}
+
+#[derive(Debug, PartialEq)]
+pub enum NodeError {
+ ExactLengthRequired(usize, String),
+ HexError(FromHexError, String),
+}
+
+/// Low level utility function, also for prefixes
+fn get_nybble(s: &[u8], i: usize) -> u8 {
+ if i % 2 == 0 {
+ s[i / 2] >> 4
+ } else {
+ s[i / 2] & 0x0f
+ }
+}
+
+impl Node {
+ /// Retrieve the `i`th half-byte of the binary data.
+ ///
+ /// This is also the `i`th hexadecimal digit in numeric form,
+ /// also called a [nybble](https://en.wikipedia.org/wiki/Nibble).
+ pub fn get_nybble(&self, i: usize) -> u8 {
+ get_nybble(&self.data, i)
+ }
+
+ /// Length of the data, in nybbles
+ pub fn nybbles_len(&self) -> usize {
+ // public exposure as an instance method only, so that we can
+ // easily support several sizes of hashes if needed in the future.
+ NODE_NYBBLES_LENGTH
+ }
+
+ /// Convert from hexadecimal string representation
+ ///
+ /// Exact length is required.
+ ///
+ /// To be used in FFI and I/O only, in order to facilitate future
+ /// changes of hash format.
+ pub fn from_hex(hex: &str) -> Result<Node, NodeError> {
+ Ok(NodeData::from_hex(hex)
+ .map_err(|e| NodeError::from((e, hex)))?
+ .into())
+ }
+
+ /// Convert to hexadecimal string representation
+ ///
+ /// To be used in FFI and I/O only, in order to facilitate future
+ /// changes of hash format.
+ pub fn encode_hex(&self) -> String {
+ hex::encode(self.data)
+ }
+
+ /// Provide access to binary data
+ ///
+ /// This is needed by FFI layers, for instance to return expected
+ /// binary values to Python.
+ pub fn as_bytes(&self) -> &[u8] {
+ &self.data
+ }
+}
+
+impl From<(FromHexError, &str)> for NodeError {
+ fn from(err_offender: (FromHexError, &str)) -> Self {
+ let (err, offender) = err_offender;
+ match err {
+ FromHexError::InvalidStringLength => {
+ NodeError::ExactLengthRequired(
+ NODE_NYBBLES_LENGTH,
+ offender.to_string(),
+ )
+ }
+ _ => NodeError::HexError(err, offender.to_string()),
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn sample_node() -> Node {
+ let mut data = [0; NODE_BYTES_LENGTH];
+ data.copy_from_slice(&[
+ 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, 0xfe, 0xdc, 0xba,
+ 0x98, 0x76, 0x54, 0x32, 0x10, 0xde, 0xad, 0xbe, 0xef,
+ ]);
+ data.into()
+ }
+
+ /// Pad an hexadecimal string to reach `NODE_NYBBLES_LENGTH`
+ ///
+ /// The padding is made with zeros
+ fn hex_pad_right(hex: &str) -> String {
+ let mut res = hex.to_string();
+ while res.len() < NODE_NYBBLES_LENGTH {
+ res.push('0');
+ }
+ res
+ }
+
+ fn sample_node_hex() -> String {
+ hex_pad_right("0123456789abcdeffedcba9876543210deadbeef")
+ }
+
+ #[test]
+ fn test_node_from_hex() {
+ assert_eq!(Node::from_hex(&sample_node_hex()), Ok(sample_node()));
+
+ let mut short = hex_pad_right("0123");
+ short.pop();
+ short.pop();
+ assert_eq!(
+ Node::from_hex(&short),
+ Err(NodeError::ExactLengthRequired(NODE_NYBBLES_LENGTH, short)),
+ );
+
+ let not_hex = hex_pad_right("012... oops");
+ assert_eq!(
+ Node::from_hex(¬_hex),
+ Err(NodeError::HexError(
+ FromHexError::InvalidHexCharacter { c: '.', index: 3 },
+ not_hex,
+ )),
+ );
+ }
+
+ #[test]
+ fn test_node_encode_hex() {
+ assert_eq!(sample_node().encode_hex(), sample_node_hex());
+ }
+}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/rust/hg-core/src/revlog/nodemap.rs Thu Jan 23 13:10:48 2020 -0800
@@ -0,0 +1,160 @@
+// Copyright 2018-2020 Georges Racinet <georges.racinet@octobus.net>
+// and Mercurial contributors
+//
+// This software may be used and distributed according to the terms of the
+// GNU General Public License version 2 or any later version.
+//! Indexing facilities for fast retrieval of `Revision` from `Node`
+//!
+//! This provides a variation on the 16-ary radix tree that is
+//! provided as "nodetree" in revlog.c, ready for append-only persistence
+//! on disk.
+//!
+//! Following existing implicit conventions, the "nodemap" terminology
+//! is used in a more abstract context.
+
+use super::Revision;
+use std::fmt;
+
+/// Low level NodeTree [`Blocks`] elements
+///
+/// These are exactly as for instance on persistent storage.
+type RawElement = i32;
+
+/// High level representation of values in NodeTree
+/// [`Blocks`](struct.Block.html)
+///
+/// This is the high level representation that most algorithms should
+/// use.
+#[derive(Clone, Debug, Eq, PartialEq)]
+enum Element {
+ Rev(Revision),
+ Block(usize),
+ None,
+}
+
+impl From<RawElement> for Element {
+ /// Conversion from low level representation, after endianness conversion.
+ ///
+ /// See [`Block`](struct.Block.html) for explanation about the encoding.
+ fn from(raw: RawElement) -> Element {
+ if raw >= 0 {
+ Element::Block(raw as usize)
+ } else if raw == -1 {
+ Element::None
+ } else {
+ Element::Rev(-raw - 2)
+ }
+ }
+}
+
+impl From<Element> for RawElement {
+ fn from(element: Element) -> RawElement {
+ match element {
+ Element::None => 0,
+ Element::Block(i) => i as RawElement,
+ Element::Rev(rev) => -rev - 2,
+ }
+ }
+}
+
+/// A logical block of the `NodeTree`, packed with a fixed size.
+///
+/// These are always used in container types implementing `Index<Block>`,
+/// such as `&Block`
+///
+/// As an array of integers, its ith element encodes that the
+/// ith potential edge from the block, representing the ith hexadecimal digit
+/// (nybble) `i` is either:
+///
+/// - absent (value -1)
+/// - another `Block` in the same indexable container (value ≥ 0)
+/// - a `Revision` leaf (value ≤ -2)
+///
+/// Endianness has to be fixed for consistency on shared storage across
+/// different architectures.
+///
+/// A key difference with the C `nodetree` is that we need to be
+/// able to represent the [`Block`] at index 0, hence -1 is the empty marker
+/// rather than 0 and the `Revision` range upper limit of -2 instead of -1.
+///
+/// Another related difference is that `NULL_REVISION` (-1) is not
+/// represented at all, because we want an immutable empty nodetree
+/// to be valid.
+
+#[derive(Clone, PartialEq)]
+pub struct Block([RawElement; 16]);
+
+impl Block {
+ fn new() -> Self {
+ Block([-1; 16])
+ }
+
+ fn get(&self, nybble: u8) -> Element {
+ Element::from(RawElement::from_be(self.0[nybble as usize]))
+ }
+
+ fn set(&mut self, nybble: u8, element: Element) {
+ self.0[nybble as usize] = RawElement::to_be(element.into())
+ }
+}
+
+impl fmt::Debug for Block {
+ /// sparse representation for testing and debugging purposes
+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+ f.debug_map()
+ .entries((0..16).filter_map(|i| match self.get(i) {
+ Element::None => None,
+ element => Some((i, element)),
+ }))
+ .finish()
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ /// Creates a `Block` using a syntax close to the `Debug` output
+ macro_rules! block {
+ {$($nybble:tt : $variant:ident($val:tt)),*} => (
+ {
+ let mut block = Block::new();
+ $(block.set($nybble, Element::$variant($val)));*;
+ block
+ }
+ )
+ }
+
+ #[test]
+ fn test_block_debug() {
+ let mut block = Block::new();
+ block.set(1, Element::Rev(3));
+ block.set(10, Element::Block(0));
+ assert_eq!(format!("{:?}", block), "{1: Rev(3), 10: Block(0)}");
+ }
+
+ #[test]
+ fn test_block_macro() {
+ let block = block! {5: Block(2)};
+ assert_eq!(format!("{:?}", block), "{5: Block(2)}");
+
+ let block = block! {13: Rev(15), 5: Block(2)};
+ assert_eq!(format!("{:?}", block), "{5: Block(2), 13: Rev(15)}");
+ }
+
+ #[test]
+ fn test_raw_block() {
+ let mut raw = [-1; 16];
+ raw[0] = 0;
+ raw[1] = RawElement::to_be(15);
+ raw[2] = RawElement::to_be(-2);
+ raw[3] = RawElement::to_be(-1);
+ raw[4] = RawElement::to_be(-3);
+ let block = Block(raw);
+ assert_eq!(block.get(0), Element::Block(0));
+ assert_eq!(block.get(1), Element::Block(15));
+ assert_eq!(block.get(3), Element::None);
+ assert_eq!(block.get(2), Element::Rev(0));
+ assert_eq!(block.get(4), Element::Rev(1));
+ }
+}
--- a/rust/hg-core/src/utils/hg_path.rs Tue Jan 21 17:15:34 2020 -0800
+++ b/rust/hg-core/src/utils/hg_path.rs Thu Jan 23 13:10:48 2020 -0800
@@ -138,6 +138,79 @@
None
}
}
+
+ #[cfg(windows)]
+ /// Copied from the Python stdlib's `os.path.splitdrive` implementation.
+ ///
+ /// Split a pathname into drive/UNC sharepoint and relative path specifiers.
+ /// Returns a 2-tuple (drive_or_unc, path); either part may be empty.
+ ///
+ /// If you assign
+ /// result = split_drive(p)
+ /// It is always true that:
+ /// result[0] + result[1] == p
+ ///
+ /// If the path contained a drive letter, drive_or_unc will contain everything
+ /// up to and including the colon.
+ /// e.g. split_drive("c:/dir") returns ("c:", "/dir")
+ ///
+ /// If the path contained a UNC path, the drive_or_unc will contain the host
+ /// name and share up to but not including the fourth directory separator
+ /// character.
+ /// e.g. split_drive("//host/computer/dir") returns ("//host/computer", "/dir")
+ ///
+ /// Paths cannot contain both a drive letter and a UNC path.
+ pub fn split_drive<'a>(&self) -> (&HgPath, &HgPath) {
+ let bytes = self.as_bytes();
+ let is_sep = |b| std::path::is_separator(b as char);
+
+ if self.len() < 2 {
+ (HgPath::new(b""), &self)
+ } else if is_sep(bytes[0])
+ && is_sep(bytes[1])
+ && (self.len() == 2 || !is_sep(bytes[2]))
+ {
+ // Is a UNC path:
+ // vvvvvvvvvvvvvvvvvvvv drive letter or UNC path
+ // \\machine\mountpoint\directory\etc\...
+ // directory ^^^^^^^^^^^^^^^
+
+ let machine_end_index = bytes[2..].iter().position(|b| is_sep(*b));
+ let mountpoint_start_index = if let Some(i) = machine_end_index {
+ i + 2
+ } else {
+ return (HgPath::new(b""), &self);
+ };
+
+ match bytes[mountpoint_start_index + 1..]
+ .iter()
+ .position(|b| is_sep(*b))
+ {
+ // A UNC path can't have two slashes in a row
+ // (after the initial two)
+ Some(0) => (HgPath::new(b""), &self),
+ Some(i) => {
+ let (a, b) =
+ bytes.split_at(mountpoint_start_index + 1 + i);
+ (HgPath::new(a), HgPath::new(b))
+ }
+ None => (&self, HgPath::new(b"")),
+ }
+ } else if bytes[1] == b':' {
+ // Drive path c:\directory
+ let (a, b) = bytes.split_at(2);
+ (HgPath::new(a), HgPath::new(b))
+ } else {
+ (HgPath::new(b""), &self)
+ }
+ }
+
+ #[cfg(unix)]
+ /// Split a pathname into drive and path. On Posix, drive is always empty.
+ pub fn split_drive(&self) -> (&HgPath, &HgPath) {
+ (HgPath::new(b""), &self)
+ }
+
/// Checks for errors in the path, short-circuiting at the first one.
/// This generates fine-grained errors useful for debugging.
/// To simply check if the path is valid during tests, use `is_valid`.
@@ -473,4 +546,101 @@
let base = HgPath::new(b"ends/");
assert_eq!(Some(HgPath::new(b"with/dir/")), path.relative_to(base));
}
+
+ #[test]
+ #[cfg(unix)]
+ fn test_split_drive() {
+ // Taken from the Python stdlib's tests
+ assert_eq!(
+ HgPath::new(br"/foo/bar").split_drive(),
+ (HgPath::new(b""), HgPath::new(br"/foo/bar"))
+ );
+ assert_eq!(
+ HgPath::new(br"foo:bar").split_drive(),
+ (HgPath::new(b""), HgPath::new(br"foo:bar"))
+ );
+ assert_eq!(
+ HgPath::new(br":foo:bar").split_drive(),
+ (HgPath::new(b""), HgPath::new(br":foo:bar"))
+ );
+ // Also try NT paths; should not split them
+ assert_eq!(
+ HgPath::new(br"c:\foo\bar").split_drive(),
+ (HgPath::new(b""), HgPath::new(br"c:\foo\bar"))
+ );
+ assert_eq!(
+ HgPath::new(b"c:/foo/bar").split_drive(),
+ (HgPath::new(b""), HgPath::new(br"c:/foo/bar"))
+ );
+ assert_eq!(
+ HgPath::new(br"\\conky\mountpoint\foo\bar").split_drive(),
+ (
+ HgPath::new(b""),
+ HgPath::new(br"\\conky\mountpoint\foo\bar")
+ )
+ );
+ }
+
+ #[test]
+ #[cfg(windows)]
+ fn test_split_drive() {
+ assert_eq!(
+ HgPath::new(br"c:\foo\bar").split_drive(),
+ (HgPath::new(br"c:"), HgPath::new(br"\foo\bar"))
+ );
+ assert_eq!(
+ HgPath::new(b"c:/foo/bar").split_drive(),
+ (HgPath::new(br"c:"), HgPath::new(br"/foo/bar"))
+ );
+ assert_eq!(
+ HgPath::new(br"\\conky\mountpoint\foo\bar").split_drive(),
+ (
+ HgPath::new(br"\\conky\mountpoint"),
+ HgPath::new(br"\foo\bar")
+ )
+ );
+ assert_eq!(
+ HgPath::new(br"//conky/mountpoint/foo/bar").split_drive(),
+ (
+ HgPath::new(br"//conky/mountpoint"),
+ HgPath::new(br"/foo/bar")
+ )
+ );
+ assert_eq!(
+ HgPath::new(br"\\\conky\mountpoint\foo\bar").split_drive(),
+ (
+ HgPath::new(br""),
+ HgPath::new(br"\\\conky\mountpoint\foo\bar")
+ )
+ );
+ assert_eq!(
+ HgPath::new(br"///conky/mountpoint/foo/bar").split_drive(),
+ (
+ HgPath::new(br""),
+ HgPath::new(br"///conky/mountpoint/foo/bar")
+ )
+ );
+ assert_eq!(
+ HgPath::new(br"\\conky\\mountpoint\foo\bar").split_drive(),
+ (
+ HgPath::new(br""),
+ HgPath::new(br"\\conky\\mountpoint\foo\bar")
+ )
+ );
+ assert_eq!(
+ HgPath::new(br"//conky//mountpoint/foo/bar").split_drive(),
+ (
+ HgPath::new(br""),
+ HgPath::new(br"//conky//mountpoint/foo/bar")
+ )
+ );
+ // UNC part containing U+0130
+ assert_eq!(
+ HgPath::new(b"//conky/MOUNTPO\xc4\xb0NT/foo/bar").split_drive(),
+ (
+ HgPath::new(b"//conky/MOUNTPO\xc4\xb0NT"),
+ HgPath::new(br"/foo/bar")
+ )
+ );
+ }
}
--- a/rust/hg-cpython/src/exceptions.rs Tue Jan 21 17:15:34 2020 -0800
+++ b/rust/hg-cpython/src/exceptions.rs Thu Jan 23 13:10:48 2020 -0800
@@ -13,7 +13,7 @@
//!
//! [`GraphError`]: struct.GraphError.html
use cpython::{
- exc::{IOError, RuntimeError, ValueError},
+ exc::{RuntimeError, ValueError},
py_exception, PyErr, Python,
};
use hg;
@@ -39,34 +39,6 @@
}
}
-py_exception!(rustext, PatternError, RuntimeError);
-py_exception!(rustext, PatternFileError, RuntimeError);
py_exception!(rustext, HgPathPyError, RuntimeError);
-impl PatternError {
- pub fn pynew(py: Python, inner: hg::PatternError) -> PyErr {
- match inner {
- hg::PatternError::UnsupportedSyntax(m) => {
- PatternError::new(py, ("PatternError", m))
- }
- }
- }
-}
-
-impl PatternFileError {
- pub fn pynew(py: Python, inner: hg::PatternFileError) -> PyErr {
- match inner {
- hg::PatternFileError::IO(e) => {
- let value = (e.raw_os_error().unwrap_or(2), e.to_string());
- PyErr::new::<IOError, _>(py, value)
- }
- hg::PatternFileError::Pattern(e, l) => match e {
- hg::PatternError::UnsupportedSyntax(m) => {
- PatternFileError::new(py, ("PatternFileError", m, l))
- }
- },
- }
- }
-}
-
py_exception!(shared_ref, AlreadyBorrowed, RuntimeError);
--- a/rust/hg-cpython/src/filepatterns.rs Tue Jan 21 17:15:34 2020 -0800
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,133 +0,0 @@
-// filepatterns.rs
-//
-// Copyright 2019, Georges Racinet <gracinet@anybox.fr>,
-// Raphaël Gomès <rgomes@octobus.net>
-//
-// This software may be used and distributed according to the terms of the
-// GNU General Public License version 2 or any later version.
-
-//! Bindings for the `hg::filepatterns` module provided by the
-//! `hg-core` crate. From Python, this will be seen as `rustext.filepatterns`
-//! and can be used as replacement for the the pure `filepatterns` Python
-//! module.
-use crate::exceptions::{PatternError, PatternFileError};
-use cpython::{
- PyBytes, PyDict, PyModule, PyObject, PyResult, PyTuple, Python, ToPyObject,
-};
-use hg::utils::files;
-use hg::{build_single_regex, read_pattern_file, LineNumber, PatternTuple};
-use std::path::PathBuf;
-
-/// Rust does not like functions with different return signatures.
-/// The 3-tuple version is always returned by the hg-core function,
-/// the (potential) conversion is handled at this level since it is not likely
-/// to have any measurable impact on performance.
-///
-/// The Python implementation passes a function reference for `warn` instead
-/// of a boolean that is used to emit warnings while parsing. The Rust
-/// implementation chooses to accumulate the warnings and propagate them to
-/// Python upon completion. See the `readpatternfile` function in `match.py`
-/// for more details.
-fn read_pattern_file_wrapper(
- py: Python,
- file_path: PyObject,
- warn: bool,
- source_info: bool,
-) -> PyResult<PyTuple> {
- let bytes = file_path.extract::<PyBytes>(py)?;
- let path = files::get_path_from_bytes(bytes.data(py));
- match read_pattern_file(path, warn) {
- Ok((patterns, warnings)) => {
- if source_info {
- let itemgetter = |x: &PatternTuple| {
- (PyBytes::new(py, &x.0), x.1, PyBytes::new(py, &x.2))
- };
- let results: Vec<(PyBytes, LineNumber, PyBytes)> =
- patterns.iter().map(itemgetter).collect();
- return Ok((results, warnings_to_py_bytes(py, &warnings))
- .to_py_object(py));
- }
- let itemgetter = |x: &PatternTuple| PyBytes::new(py, &x.0);
- let results: Vec<PyBytes> =
- patterns.iter().map(itemgetter).collect();
- Ok(
- (results, warnings_to_py_bytes(py, &warnings))
- .to_py_object(py),
- )
- }
- Err(e) => Err(PatternFileError::pynew(py, e)),
- }
-}
-
-fn warnings_to_py_bytes(
- py: Python,
- warnings: &[(PathBuf, Vec<u8>)],
-) -> Vec<(PyBytes, PyBytes)> {
- warnings
- .iter()
- .map(|(path, syn)| {
- (
- PyBytes::new(py, &files::get_bytes_from_path(path)),
- PyBytes::new(py, syn),
- )
- })
- .collect()
-}
-
-fn build_single_regex_wrapper(
- py: Python,
- kind: PyObject,
- pat: PyObject,
- globsuffix: PyObject,
-) -> PyResult<PyBytes> {
- match build_single_regex(
- kind.extract::<PyBytes>(py)?.data(py),
- pat.extract::<PyBytes>(py)?.data(py),
- globsuffix.extract::<PyBytes>(py)?.data(py),
- ) {
- Ok(regex) => Ok(PyBytes::new(py, ®ex)),
- Err(e) => Err(PatternError::pynew(py, e)),
- }
-}
-
-pub fn init_module(py: Python, package: &str) -> PyResult<PyModule> {
- let dotted_name = &format!("{}.filepatterns", package);
- let m = PyModule::new(py, dotted_name)?;
-
- m.add(py, "__package__", package)?;
- m.add(
- py,
- "__doc__",
- "Patterns files parsing - Rust implementation",
- )?;
- m.add(
- py,
- "build_single_regex",
- py_fn!(
- py,
- build_single_regex_wrapper(
- kind: PyObject,
- pat: PyObject,
- globsuffix: PyObject
- )
- ),
- )?;
- m.add(
- py,
- "read_pattern_file",
- py_fn!(
- py,
- read_pattern_file_wrapper(
- file_path: PyObject,
- warn: bool,
- source_info: bool
- )
- ),
- )?;
- m.add(py, "PatternError", py.get_type::<PatternError>())?;
- let sys = PyModule::import(py, "sys")?;
- let sys_modules: PyDict = sys.get(py, "modules")?.extract(py)?;
- sys_modules.set_item(py, dotted_name, &m)?;
-
- Ok(m)
-}
--- a/rust/hg-cpython/src/lib.rs Tue Jan 21 17:15:34 2020 -0800
+++ b/rust/hg-cpython/src/lib.rs Thu Jan 23 13:10:48 2020 -0800
@@ -33,7 +33,6 @@
pub mod dirstate;
pub mod discovery;
pub mod exceptions;
-pub mod filepatterns;
pub mod parsers;
pub mod revlog;
pub mod utils;
@@ -53,25 +52,10 @@
m.add(py, "revlog", revlog::init_module(py, &dotted_name)?)?;
m.add(
py,
- "filepatterns",
- filepatterns::init_module(py, &dotted_name)?,
- )?;
- m.add(
- py,
"parsers",
parsers::init_parsers_module(py, &dotted_name)?,
)?;
m.add(py, "GraphError", py.get_type::<exceptions::GraphError>())?;
- m.add(
- py,
- "PatternFileError",
- py.get_type::<exceptions::PatternFileError>(),
- )?;
- m.add(
- py,
- "PatternError",
- py.get_type::<exceptions::PatternError>(),
- )?;
Ok(())
});
--- a/tests/run-tests.py Tue Jan 21 17:15:34 2020 -0800
+++ b/tests/run-tests.py Thu Jan 23 13:10:48 2020 -0800
@@ -555,12 +555,6 @@
help="use pure Python code instead of C extensions",
)
hgconf.add_argument(
- "-3",
- "--py3-warnings",
- action="store_true",
- help="enable Py3k warnings on Python 2.7+",
- )
- hgconf.add_argument(
"--with-chg",
metavar="CHG",
help="use specified chg wrapper in place of hg",
@@ -748,9 +742,6 @@
)
options.timeout = 0
options.slowtimeout = 0
- if options.py3_warnings:
- if PYTHON3:
- parser.error('--py3-warnings can only be used on Python 2.7')
if options.blacklist:
options.blacklist = parselistfiles(options.blacklist, 'blacklist')
@@ -909,7 +900,6 @@
timeout=None,
startport=None,
extraconfigopts=None,
- py3warnings=False,
shell=None,
hgcommand=None,
slowtimeout=None,
@@ -942,8 +932,6 @@
must have the form "key=value" (something understood by hgrc). Values
of the form "foo.key=value" will result in "[foo] key=value".
- py3warnings enables Py3k warnings.
-
shell is the shell to execute tests in.
"""
if timeout is None:
@@ -968,7 +956,6 @@
self._slowtimeout = slowtimeout
self._startport = startport
self._extraconfigopts = extraconfigopts or []
- self._py3warnings = py3warnings
self._shell = _bytespath(shell)
self._hgcommand = hgcommand or b'hg'
self._usechg = usechg
@@ -1515,9 +1502,8 @@
return os.path.join(self._testdir, b'%s.out' % self.bname)
def _run(self, env):
- py3switch = self._py3warnings and b' -3' or b''
# Quote the python(3) executable for Windows
- cmd = b'"%s"%s "%s"' % (PYTHON, py3switch, self.path)
+ cmd = b'"%s" "%s"' % (PYTHON, self.path)
vlog("# Running", cmd.decode("utf-8"))
normalizenewlines = os.name == 'nt'
result = self._runcommand(cmd, env, normalizenewlines=normalizenewlines)
@@ -3366,7 +3352,6 @@
timeout=self.options.timeout,
startport=self._getport(count),
extraconfigopts=self.options.extra_config_opt,
- py3warnings=self.options.py3_warnings,
shell=self.options.shell,
hgcommand=self._hgcommand,
usechg=bool(self.options.with_chg or self.options.chg),
@@ -3512,15 +3497,6 @@
self._usecorrectpython()
- if self.options.py3_warnings and not self.options.anycoverage:
- vlog("# Updating hg command to enable Py3k Warnings switch")
- with open(os.path.join(self._bindir, 'hg'), 'rb') as f:
- lines = [line.rstrip() for line in f]
- lines[0] += ' -3'
- with open(os.path.join(self._bindir, 'hg'), 'wb') as f:
- for line in lines:
- f.write(line + '\n')
-
hgbat = os.path.join(self._bindir, b'hg.bat')
if os.path.isfile(hgbat):
# hg.bat expects to be put in bin/scripts while run-tests.py
--- a/tests/test-check-format.t Tue Jan 21 17:15:34 2020 -0800
+++ b/tests/test-check-format.t Thu Jan 23 13:10:48 2020 -0800
@@ -1,5 +1,5 @@
#require black
$ cd $RUNTESTDIR/..
- $ black --config=black.toml --check --diff `hg files 'set:(**.py + grep("^#!.*python")) - mercurial/thirdparty/** - "contrib/python-zstandard/**"'`
+ $ black --config=black.toml --check --diff `hg files 'set:(**.py + grep("^#!.*python")) - mercurial/thirdparty/**'`
--- a/tests/test-install.t Tue Jan 21 17:15:34 2020 -0800
+++ b/tests/test-install.t Thu Jan 23 13:10:48 2020 -0800
@@ -2,6 +2,7 @@
$ hg debuginstall
checking encoding (ascii)...
checking Python executable (*) (glob)
+ checking Python implementation (*) (glob)
checking Python version (2.*) (glob) (no-py3 !)
checking Python version (3.*) (glob) (py3 !)
checking Python lib (.*[Ll]ib.*)... (re)
@@ -43,6 +44,7 @@
"hgverextra": "*", (glob)
"problems": 0,
"pythonexe": "*", (glob)
+ "pythonimplementation": "*", (glob)
"pythonlib": "*", (glob)
"pythonsecurity": [*], (glob)
"pythonver": "*.*.*", (glob)
@@ -58,6 +60,7 @@
$ HGUSER= hg debuginstall
checking encoding (ascii)...
checking Python executable (*) (glob)
+ checking Python implementation (*) (glob)
checking Python version (2.*) (glob) (no-py3 !)
checking Python version (3.*) (glob) (py3 !)
checking Python lib (.*[Ll]ib.*)... (re)
@@ -103,6 +106,7 @@
$ HGEDITOR="~/tools/testeditor.exe" hg debuginstall
checking encoding (ascii)...
checking Python executable (*) (glob)
+ checking Python implementation (*) (glob)
checking Python version (2.*) (glob) (no-py3 !)
checking Python version (3.*) (glob) (py3 !)
checking Python lib (.*[Ll]ib.*)... (re)
@@ -128,6 +132,7 @@
$ HGEDITOR="c:\foo\bar\baz.exe -y -z" hg debuginstall
checking encoding (ascii)...
checking Python executable (*) (glob)
+ checking Python implementation (*) (glob)
checking Python version (2.*) (glob) (no-py3 !)
checking Python version (3.*) (glob) (py3 !)
checking Python lib (.*[Ll]ib.*)... (re)
@@ -185,6 +190,7 @@
$ ./installenv/*/hg debuginstall || cat pip.log
checking encoding (ascii)...
checking Python executable (*) (glob)
+ checking Python implementation (*) (glob)
checking Python version (3.*) (glob)
checking Python lib (*)... (glob)
checking Python security support (*) (glob)
@@ -221,6 +227,7 @@
$ ./installenv/*/hg debuginstall || cat pip.log
checking encoding (ascii)...
checking Python executable (*) (glob)
+ checking Python implementation (*) (glob)
checking Python version (2.*) (glob)
checking Python lib (*)... (glob)
checking Python security support (*) (glob)
--- a/tests/test-rebase-collapse.t Tue Jan 21 17:15:34 2020 -0800
+++ b/tests/test-rebase-collapse.t Thu Jan 23 13:10:48 2020 -0800
@@ -486,61 +486,6 @@
abort: cannot collapse multiple named branches
[255]
- $ repeatchange() {
- > hg checkout $1
- > hg cp d z
- > echo blah >> z
- > hg commit -Am "$2" --user "$3"
- > }
- $ repeatchange 3 "E" "user1"
- 0 files updated, 0 files merged, 0 files removed, 0 files unresolved
- $ repeatchange 3 "E" "user2"
- 0 files updated, 0 files merged, 1 files removed, 0 files unresolved
- created new head
- $ hg tglog
- @ 5: fbfb97b1089a 'E'
- |
- | o 4: f338eb3c2c7c 'E'
- |/
- o 3: 41acb9dca9eb 'D'
- |
- | o 2: 8ac4a08debf1 'C' two
- | |
- | o 1: 1ba175478953 'B' one
- |/
- o 0: 1994f17a630e 'A'
-
- $ hg rebase -s 5 -d 4
- rebasing 5:fbfb97b1089a "E" (tip)
- note: not rebasing 5:fbfb97b1089a "E" (tip), its destination already has all its changes
- saved backup bundle to $TESTTMP/e/.hg/strip-backup/fbfb97b1089a-553e1d85-rebase.hg
- $ hg tglog
- @ 4: f338eb3c2c7c 'E'
- |
- o 3: 41acb9dca9eb 'D'
- |
- | o 2: 8ac4a08debf1 'C' two
- | |
- | o 1: 1ba175478953 'B' one
- |/
- o 0: 1994f17a630e 'A'
-
- $ hg export tip
- # HG changeset patch
- # User user1
- # Date 0 0
- # Thu Jan 01 00:00:00 1970 +0000
- # Node ID f338eb3c2c7cc5b5915676a2376ba7ac558c5213
- # Parent 41acb9dca9eb976e84cd21fcb756b4afa5a35c09
- E
-
- diff -r 41acb9dca9eb -r f338eb3c2c7c z
- --- /dev/null Thu Jan 01 00:00:00 1970 +0000
- +++ b/z Thu Jan 01 00:00:00 1970 +0000
- @@ -0,0 +1,2 @@
- +d
- +blah
-
$ cd ..
Rebase, collapse and copies
--- a/tests/test-rebase-rename.t Tue Jan 21 17:15:34 2020 -0800
+++ b/tests/test-rebase-rename.t Thu Jan 23 13:10:48 2020 -0800
@@ -108,6 +108,62 @@
+ $ repeatchange() {
+ > hg checkout $1
+ > hg cp a z
+ > echo blah >> z
+ > hg commit -Am "$2" --user "$3"
+ > }
+ $ repeatchange 1 "E" "user1"
+ 2 files updated, 0 files merged, 3 files removed, 0 files unresolved
+ created new head
+ $ repeatchange 1 "E" "user2"
+ 0 files updated, 0 files merged, 1 files removed, 0 files unresolved
+ created new head
+ $ hg tglog
+ @ 5: af8ad1f97097 'E'
+ |
+ | o 4: 60f545c27784 'E'
+ |/
+ | o 3: 032a9b75e83b 'rename A'
+ | |
+ | o 2: 220d0626d185 'rename B'
+ |/
+ o 1: 3ab5da9a5c01 'B'
+ |
+ o 0: 1994f17a630e 'A'
+
+ $ hg rebase -s 5 -d 4
+ rebasing 5:af8ad1f97097 "E" (tip)
+ note: not rebasing 5:af8ad1f97097 "E" (tip), its destination already has all its changes
+ saved backup bundle to $TESTTMP/a/.hg/strip-backup/af8ad1f97097-c3e90708-rebase.hg
+ $ hg tglog
+ @ 4: 60f545c27784 'E'
+ |
+ | o 3: 032a9b75e83b 'rename A'
+ | |
+ | o 2: 220d0626d185 'rename B'
+ |/
+ o 1: 3ab5da9a5c01 'B'
+ |
+ o 0: 1994f17a630e 'A'
+
+ $ hg export tip
+ # HG changeset patch
+ # User user1
+ # Date 0 0
+ # Thu Jan 01 00:00:00 1970 +0000
+ # Node ID 60f545c277846e6bad309919bae3ae106f59cb39
+ # Parent 3ab5da9a5c01faa02c20f2ec4870a4f689c92da6
+ E
+
+ diff -r 3ab5da9a5c01 -r 60f545c27784 z
+ --- /dev/null Thu Jan 01 00:00:00 1970 +0000
+ +++ b/z Thu Jan 01 00:00:00 1970 +0000
+ @@ -0,0 +1,2 @@
+ +a
+ +blah
+
$ cd ..